diff --git a/flake.nix b/flake.nix --- a/flake.nix +++ b/flake.nix @@ -49,7 +49,7 @@ inherit (gitignore.lib) gitignoreSource; in { overlays.default = final: prev: let - goModHash = "sha256-SfyLSQa3g30PXCi/VQOm0cz372B6YdXE7xfBlhgCGec="; + goModHash = "sha256-EooM036KFlO4Zot5vDLX+HFU2GfjCtjVEfF7t+d4Avk="; buildCmdPackage = name: final.buildGoModule { pname = name; diff --git a/go.mod b/go.mod --- a/go.mod +++ b/go.mod @@ -16,7 +16,8 @@ github.com/dustin/go-humanize v1.0.1 github.com/gliderlabs/ssh v0.3.8 github.com/go-chi/chi/v5 v5.2.0 - github.com/go-git/go-git/v5 v5.0.0-00010101000000-000000000000 + github.com/go-enry/go-enry/v2 v2.9.2 + github.com/go-git/go-git/v5 v5.14.0 github.com/google/uuid v1.6.0 github.com/gorilla/sessions v1.4.0 github.com/haileyok/atproto-oauth-golang v0.0.2 @@ -49,6 +50,7 @@ github.com/dlclark/regexp2 v1.11.5 // indirect github.com/emirpasic/gods v1.18.1 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-enry/go-oniguruma v1.2.1 // indirect github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect github.com/go-git/go-billy/v5 v5.6.2 // indirect github.com/go-logr/logr v1.4.2 // indirect diff --git a/go.sum b/go.sum --- a/go.sum +++ b/go.sum @@ -69,6 +69,10 @@ github.com/gliderlabs/ssh v0.3.8/go.mod h1:xYoytBv1sV0aL3CavoDuJIQNURXkkfPA/wxQ1pL1fAU= github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= +github.com/go-enry/go-enry/v2 v2.9.2 h1:giOQAtCgBX08kosrX818DCQJTCNtKwoPBGu0qb6nKTY= +github.com/go-enry/go-enry/v2 v2.9.2/go.mod h1:9yrj4ES1YrbNb1Wb7/PWYr2bpaCXUGRt0uafN0ISyG8= +github.com/go-enry/go-oniguruma v1.2.1 h1:k8aAMuJfMrqm/56SG2lV9Cfti6tC4x8673aHCcBk+eo= +github.com/go-enry/go-oniguruma v1.2.1/go.mod h1:bWDhYP+S6xZQgiRL7wlTScFYBe023B6ilRZbCAD5Hf4= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 h1:+zs/tPmkDkHx3U66DAb0lQFJrpS6731Oaa12ikc+DiI= github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376/go.mod h1:an3vInlBmSxCcxctByoQdvwPiA7DTK7jaaFDBTtu0ic= github.com/go-git/go-billy/v5 v5.6.2 h1:6Q86EsPXMa7c3YZ3aLAQsMA0VlWmy43r6FHqa/UNbRM= @@ -263,6 +267,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= diff --git a/knotserver/handler.go b/knotserver/handler.go --- a/knotserver/handler.go +++ b/knotserver/handler.go @@ -126,6 +126,7 @@ r.Route("/repo", func(r chi.Router) { r.Use(h.VerifySignature) r.Put("/new", h.NewRepo) + r.Get("/languages/{branch}", h.RepoLanguages) r.Delete("/", h.RemoveRepo) r.Route("/fork", func(r chi.Router) { r.Post("/", h.RepoFork) diff --git a/knotserver/routes.go b/knotserver/routes.go --- a/knotserver/routes.go +++ b/knotserver/routes.go @@ -12,6 +12,7 @@ "net/http" "net/url" "os" + "path" "path/filepath" "strconv" "strings" @@ -19,6 +20,7 @@ securejoin "github.com/cyphar/filepath-securejoin" "github.com/gliderlabs/ssh" "github.com/go-chi/chi/v5" + "github.com/go-enry/go-enry/v2" gogit "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" @@ -710,6 +712,106 @@ w.Header().Set("Content-Type", "application/json") json.NewEncoder(w).Encode(types.AncestorCheckResponse{Status: status}) +} + +func (h *Handle) RepoLanguages(w http.ResponseWriter, r *http.Request) { + l := h.l.With("handler", "RepoForkSync") + + data := struct { + Did string `json:"did"` + Source string `json:"source"` + Name string `json:"name,omitempty"` + }{} + + if err := json.NewDecoder(r.Body).Decode(&data); err != nil { + writeError(w, "invalid request body", http.StatusBadRequest) + return + } + + did := data.Did + source := data.Source + + if did == "" || source == "" { + l.Error("invalid request body, empty did or name") + w.WriteHeader(http.StatusBadRequest) + return + } + + var name string + if data.Name != "" { + name = data.Name + } else { + name = filepath.Base(source) + } + + branch := chi.URLParam(r, "branch") + branch, _ = url.PathUnescape(branch) + + relativeRepoPath := filepath.Join(did, name) + repoPath, _ := securejoin.SecureJoin(h.c.Repo.ScanPath, relativeRepoPath) + + gr, err := git.Open(repoPath, branch) + if err != nil { + log.Println(err) + notFound(w) + return + } + + languageFileCount := make(map[string]int) + languagePercentage := make(map[string]float64) + + err = recurseEntireTree(gr, func(absPath string) { + lang, safe := enry.GetLanguageByExtension(absPath) + if len(lang) == 0 || !safe { + content, _ := gr.FileContentN(absPath, 1024) + if !safe { + lang = enry.GetLanguage(absPath, content) + } else { + lang, _ = enry.GetLanguageByContent(absPath, content) + if len(lang) == 0 { + return + } + } + } + + v, ok := languageFileCount[lang] + if ok { + languageFileCount[lang] = v + 1 + } else { + languageFileCount[lang] = 1 + } + }, "") + if err != nil { + log.Println(err) + writeError(w, err.Error(), http.StatusNoContent) + return + } + + for path, fileCount := range languageFileCount { + percentage := float64(fileCount) / float64(len(languageFileCount)) * 100.0 + languagePercentage[path] = percentage + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(types.RepoLanguageResponse{Languages: languagePercentage}) +} + +func recurseEntireTree(git *git.GitRepo, callback func(absPath string), filePath string) error { + files, err := git.FileTree(filePath) + if err != nil { + log.Println(err) + return err + } + + for _, file := range files { + absPath := path.Join(filePath, file.Name) + if !file.IsFile { + return recurseEntireTree(git, callback, absPath) + } + callback(absPath) + } + + return nil } func (h *Handle) RepoForkSync(w http.ResponseWriter, r *http.Request) { diff --git a/types/repo.go b/types/repo.go --- a/types/repo.go +++ b/types/repo.go @@ -109,3 +109,8 @@ type AncestorCheckResponse struct { Status ForkStatus `json:"status"` } + +type RepoLanguageResponse struct { + // Language: Percentage + Languages map[string]float64 `json:"languages"` +} diff --git a/types/tree.go b/types/tree.go --- a/types/tree.go +++ b/types/tree.go @@ -8,6 +8,7 @@ // A nicer git tree representation. type NiceTree struct { + // Relative path Name string `json:"name"` Mode string `json:"mode"` Size int64 `json:"size"` diff --git a/knotserver/git/git.go b/knotserver/git/git.go --- a/knotserver/git/git.go +++ b/knotserver/git/git.go @@ -169,6 +169,42 @@ return c, nil } +func (g *GitRepo) FileContentN(path string, cap int64) ([]byte, error) { + buf := []byte{} + + c, err := g.r.CommitObject(g.h) + if err != nil { + return nil, fmt.Errorf("commit object: %w", err) + } + + tree, err := c.Tree() + if err != nil { + return nil, fmt.Errorf("file tree: %w", err) + } + + file, err := tree.File(path) + if err != nil { + return nil, err + } + + isbin, _ := file.IsBinary() + + if !isbin { + reader, err := file.Reader() + if err != nil { + return nil, err + } + bufReader := io.LimitReader(reader, cap) + _, err = bufReader.Read(buf) + if err != nil { + return nil, err + } + return buf, nil + } else { + return nil, ErrBinaryFile + } +} + func (g *GitRepo) FileContent(path string) (string, error) { c, err := g.r.CommitObject(g.h) if err != nil {