From 1c43d5d16292140a5393064cd00e576e918a4a82 Mon Sep 17 00:00:00 2001 From: Anirudh Oppiliappan Date: Thu, 30 Jan 2025 14:09:42 +0000 Subject: [PATCH] knotserver: init --- go.mod | 1 + go.sum | 2 ++ knotserver/file.go | 74 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/git.go | 68 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/handler.go | 78 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/http_util.go | 26 ++++++++++++++++++++++++++ knotserver/routes.go | 415 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/util.go | 136 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ cmd/knotserver/main.go | 41 +++++++++++++++++++++++++++++++++++++++++ knotserver/git/diff.go | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/git/git.go | 344 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/git/repo.go | 33 +++++++++++++++++++++++++++++++++ knotserver/git/tree.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/git/service/service.go | 121 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/git/service/write_flusher.go | 25 +++++++++++++++++++++++++ 15 file(s) changed, 1549 insertion(s)(+), 0 deletion(s)(-) diff --git a/go.mod b/go.mod --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ github.com/bluekeyes/go-gitdiff v0.8.0 github.com/bluesky-social/indigo v0.0.0-20250123072624-9e3b84fdbb20 github.com/dustin/go-humanize v1.0.1 + github.com/go-chi/chi v1.5.5 github.com/go-chi/chi/v5 v5.2.0 github.com/go-git/go-git/v5 v5.12.0 github.com/google/uuid v1.6.0 diff --git a/go.sum b/go.sum --- a/go.sum +++ b/go.sum @@ -53,6 +53,8 @@ github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/gliderlabs/ssh v0.3.5 h1:OcaySEmAQJgyYcArR+gGGTHCyE7nvhEMTlYY+Dp8CpY= github.com/gliderlabs/ssh v0.3.5/go.mod h1:8XB4KraRrX39qHhT6yxPsHedjA08I/uBVwj4xC+/+z4= +github.com/go-chi/chi v1.5.5 h1:vOB/HbEMt9QqBqErz07QehcOKHaWFtuj87tTDVz2qXE= +github.com/go-chi/chi v1.5.5/go.mod h1:C9JqLr3tIYjDOZpzn+BCuxY8z8vmca43EeMgyZt7irw= github.com/go-chi/chi/v5 v5.2.0 h1:Aj1EtB0qR2Rdo2dG4O94RIU35w2lvQSj6BRA4+qwFL0= github.com/go-chi/chi/v5 v5.2.0/go.mod h1:DslCQbL2OYiznFReuXYUmQ2hGd1aDpCnlMNITLSKoi8= github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= diff --git a/knotserver/file.go b/knotserver/file.go new file mode 100644 --- /dev/null +++ b/knotserver/file.go @@ -0,0 +1,74 @@ +package knotserver + +import ( + "bytes" + "io" + "log" + "net/http" + "strings" + + "github.com/icyphox/bild/git" +) + +func (h *Handle) listFiles(files []git.NiceTree, data map[string]any, w http.ResponseWriter) { + data["files"] = files + data["meta"] = h.c.Meta + + writeJSON(w, data) + return +} + +func countLines(r io.Reader) (int, error) { + buf := make([]byte, 32*1024) + bufLen := 0 + count := 0 + nl := []byte{'\n'} + + for { + c, err := r.Read(buf) + if c > 0 { + bufLen += c + } + count += bytes.Count(buf[:c], nl) + + switch { + case err == io.EOF: + /* handle last line not having a newline at the end */ + if bufLen >= 1 && buf[(bufLen-1)%(32*1024)] != '\n' { + count++ + } + return count, nil + case err != nil: + return 0, err + } + } +} + +func (h *Handle) showFile(content string, data map[string]any, w http.ResponseWriter) { + lc, err := countLines(strings.NewReader(content)) + if err != nil { + // Non-fatal, we'll just skip showing line numbers in the template. + log.Printf("counting lines: %s", err) + } + + lines := make([]int, lc) + if lc > 0 { + for i := range lines { + lines[i] = i + 1 + } + } + + data["linecount"] = lines + data["content"] = content + data["meta"] = h.c.Meta + + writeJSON(w, data) + return +} + +func (h *Handle) showRaw(content string, w http.ResponseWriter) { + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "text/plain") + w.Write([]byte(content)) + return +} diff --git a/knotserver/git.go b/knotserver/git.go new file mode 100644 --- /dev/null +++ b/knotserver/git.go @@ -0,0 +1,68 @@ +package knotserver + +import ( + "compress/gzip" + "io" + "log" + "net/http" + "path/filepath" + + "github.com/icyphox/bild/knotserver/git/service" +) +func (d *Handle) InfoRefs(w http.ResponseWriter, r *http.Request) { + name := displayRepoName(r) + name = filepath.Clean(name) + + repo := filepath.Join(d.c.Repo.ScanPath, name) + + w.Header().Set("content-type", "application/x-git-upload-pack-advertisement") + w.WriteHeader(http.StatusOK) + + cmd := service.ServiceCommand{ + Dir: repo, + Stdout: w, + } + + if err := cmd.InfoRefs(); err != nil { + http.Error(w, err.Error(), 500) + log.Printf("git: failed to execute git-upload-pack (info/refs) %s", err) + return + } +} + +func (d *Handle) UploadPack(w http.ResponseWriter, r *http.Request) { + name := displayRepoName(r) + name = filepath.Clean(name) + + repo := filepath.Join(d.c.Repo.ScanPath, name) + + w.Header().Set("content-type", "application/x-git-upload-pack-result") + w.Header().Set("Connection", "Keep-Alive") + w.Header().Set("Transfer-Encoding", "chunked") + w.WriteHeader(http.StatusOK) + + cmd := service.ServiceCommand{ + Dir: repo, + Stdout: w, + } + + var reader io.ReadCloser + reader = r.Body + + if r.Header.Get("Content-Encoding") == "gzip" { + reader, err := gzip.NewReader(r.Body) + if err != nil { + http.Error(w, err.Error(), 500) + log.Printf("git: failed to create gzip reader: %s", err) + return + } + defer reader.Close() + } + + cmd.Stdin = reader + if err := cmd.UploadPack(); err != nil { + http.Error(w, err.Error(), 500) + log.Printf("git: failed to execute git-upload-pack %s", err) + return + } +} diff --git a/knotserver/handler.go b/knotserver/handler.go new file mode 100644 --- /dev/null +++ b/knotserver/handler.go @@ -0,0 +1,78 @@ +package knotserver + +import ( + "net/http" + + "github.com/go-chi/chi" + "github.com/icyphox/bild/config" + "github.com/icyphox/bild/db" +) + +func Setup(c *config.Config, db *db.DB) (http.Handler, error) { + r := chi.NewRouter() + + h := Handle{ + c: c, + db: db, + } + + // r.Route("/repo", func(r chi.Router) { + // r.Use(h.AuthMiddleware) + // r.Get("/new", h.NewRepo) + // r.Put("/new", h.NewRepo) + // }) + + r.Route("/{did}", func(r chi.Router) { + r.Get("/", h.Index) + + // Repo routes + r.Route("/{name}", func(r chi.Router) { + r.Get("/", h.Multiplex) + r.Post("/", h.Multiplex) + + r.Route("/tree/{ref}", func(r chi.Router) { + r.Get("/*", h.RepoTree) + }) + + r.Route("/blob/{ref}", func(r chi.Router) { + r.Get("/*", h.FileContent) + }) + + r.Get("/log/{ref}", h.Log) + r.Get("/archive/{file}", h.Archive) + r.Get("/commit/{ref}", h.Diff) + r.Get("/refs/", h.Refs) + + // Catch-all routes + r.Get("/*", h.Multiplex) + r.Post("/*", h.Multiplex) + }) + }) + + return r, nil +} + +type Handle struct { + c *config.Config + db *db.DB +} + +func (h *Handle) Multiplex(w http.ResponseWriter, r *http.Request) { + path := chi.URLParam(r, "*") + + if r.URL.RawQuery == "service=git-receive-pack" { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("no pushing allowed!")) + return + } + + if path == "info/refs" && + r.URL.RawQuery == "service=git-upload-pack" && + r.Method == "GET" { + h.InfoRefs(w, r) + } else if path == "git-upload-pack" && r.Method == "POST" { + h.UploadPack(w, r) + } else if r.Method == "GET" { + h.RepoIndex(w, r) + } +} diff --git a/knotserver/http_util.go b/knotserver/http_util.go new file mode 100644 --- /dev/null +++ b/knotserver/http_util.go @@ -0,0 +1,26 @@ +package knotserver + +import ( + "encoding/json" + "net/http" +) + +func writeJSON(w http.ResponseWriter, data interface{}) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(data) +} + +func writeError(w http.ResponseWriter, msg string, status int) { + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + json.NewEncoder(w).Encode(map[string]string{"error": msg}) +} + +func notFound(w http.ResponseWriter) { + writeError(w, "not found", http.StatusNotFound) +} + +func writeMsg(w http.ResponseWriter, msg string) { + writeJson(w, map[string]string{"msg": msg}) +} diff --git a/knotserver/routes.go b/knotserver/routes.go new file mode 100644 --- /dev/null +++ b/knotserver/routes.go @@ -0,0 +1,415 @@ +package knotserver + +import ( + "compress/gzip" + "errors" + "fmt" + "html/template" + "log" + "net/http" + "path/filepath" + "strconv" + "strings" + + "github.com/go-chi/chi/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/icyphox/bild/git" + "github.com/russross/blackfriday/v2" +) + +func (h *Handle) Index(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("This is a knot, part of the wider Tangle network: https://knots.sh")) +} + +func (h *Handle) RepoIndex(w http.ResponseWriter, r *http.Request) { + path := filepath.Join(h.c.Repo.ScanPath, didPath(r)) + + gr, err := git.Open(path, "") + if err != nil { + if errors.Is(err, plumbing.ErrReferenceNotFound) { + writeMsg(w, "repo empty") + return + } else { + notFound(w) + return + } + } + commits, err := gr.Commits() + if err != nil { + writeError(w, err.Error(), http.StatusInternalServerError) + log.Println(err) + return + } + + var readmeContent template.HTML + for _, readme := range h.c.Repo.Readme { + ext := filepath.Ext(readme) + content, _ := gr.FileContent(readme) + if len(content) > 0 { + switch ext { + case ".md", ".mkd", ".markdown": + unsafe := blackfriday.Run( + []byte(content), + blackfriday.WithExtensions(blackfriday.CommonExtensions), + ) + html := sanitize(unsafe) + readmeContent = template.HTML(html) + default: + safe := sanitize([]byte(content)) + readmeContent = template.HTML( + fmt.Sprintf(`
%s
`, safe), + ) + } + break + } + } + + if readmeContent == "" { + log.Printf("no readme found for %s", path) + } + + mainBranch, err := gr.FindMainBranch(h.c.Repo.MainBranch) + if err != nil { + writeError(w, err.Error(), http.StatusInternalServerError) + log.Println(err) + return + } + + if len(commits) >= 3 { + commits = commits[:3] + } + data := make(map[string]any) + data["ref"] = mainBranch + data["readme"] = readmeContent + data["commits"] = commits + data["desc"] = getDescription(path) + data["servername"] = h.c.Server.Name + data["meta"] = h.c.Meta + + writeJSON(w, data) + return +} + +func (h *Handle) RepoTree(w http.ResponseWriter, r *http.Request) { + treePath := chi.URLParam(r, "*") + ref := chi.URLParam(r, "ref") + + path := filepath.Join(h.c.Repo.ScanPath, didPath(r)) + gr, err := git.Open(path, ref) + if err != nil { + notFound(w) + return + } + + files, err := gr.FileTree(treePath) + if err != nil { + writeError(w, err.Error(), http.StatusInternalServerError) + log.Println(err) + return + } + + data := make(map[string]any) + data["ref"] = ref + data["parent"] = treePath + data["desc"] = getDescription(path) + data["dotdot"] = filepath.Dir(treePath) + + h.listFiles(files, data, w) + return +} + +func (h *Handle) FileContent(w http.ResponseWriter, r *http.Request) { + var raw bool + if rawParam, err := strconv.ParseBool(r.URL.Query().Get("raw")); err == nil { + raw = rawParam + } + + treePath := chi.URLParam(r, "*") + ref := chi.URLParam(r, "ref") + + path := filepath.Join(h.c.Repo.ScanPath, didPath(r)) + gr, err := git.Open(path, ref) + if err != nil { + notFound(w) + return + } + + contents, err := gr.FileContent(treePath) + if err != nil { + writeError(w, err.Error(), http.StatusInternalServerError) + return + } + data := make(map[string]any) + data["ref"] = ref + data["desc"] = getDescription(path) + data["path"] = treePath + + safe := sanitize([]byte(contents)) + + if raw { + h.showRaw(string(safe), w) + } else { + h.showFile(string(safe), data, w) + } +} + +func (h *Handle) Archive(w http.ResponseWriter, r *http.Request) { + name := displayRepoName(r) + + file := chi.URLParam(r, "file") + + // TODO: extend this to add more files compression (e.g.: xz) + if !strings.HasSuffix(file, ".tar.gz") { + notFound(w) + return + } + + ref := strings.TrimSuffix(file, ".tar.gz") + + // This allows the browser to use a proper name for the file when + // downloading + filename := fmt.Sprintf("%s-%s.tar.gz", name, ref) + setContentDisposition(w, filename) + setGZipMIME(w) + + path := filepath.Join(h.c.Repo.ScanPath, didPath(r)) + gr, err := git.Open(path, ref) + if err != nil { + notFound(w) + return + } + + gw := gzip.NewWriter(w) + defer gw.Close() + + prefix := fmt.Sprintf("%s-%s", name, ref) + err = gr.WriteTar(gw, prefix) + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with printing the error. + log.Println(err) + return + } + + err = gw.Flush() + if err != nil { + // once we start writing to the body we can't report error anymore + // so we are only left with printing the error. + log.Println(err) + return + } +} + +func (h *Handle) Log(w http.ResponseWriter, r *http.Request) { + ref := chi.URLParam(r, "ref") + + path := filepath.Join(h.c.Repo.ScanPath, didPath(r)) + gr, err := git.Open(path, ref) + if err != nil { + notFound(w) + return + } + + commits, err := gr.Commits() + if err != nil { + writeError(w, err.Error(), http.StatusInternalServerError) + log.Println(err) + return + } + + data := make(map[string]interface{}) + data["commits"] = commits + data["meta"] = h.c.Meta + data["ref"] = ref + data["desc"] = getDescription(path) + data["log"] = true + + writeJSON(w, data) + return +} + +func (h *Handle) Diff(w http.ResponseWriter, r *http.Request) { + name := displayRepoName(r) + if h.isIgnored(name) { + notFound(w) + return + } + ref := chi.URLParam(r, "ref") + + path := filepath.Join(h.c.Repo.ScanPath, didPath(r)) + gr, err := git.Open(path, ref) + if err != nil { + notFound(w) + return + } + + diff, err := gr.Diff() + if err != nil { + writeError(w, err.Error(), http.StatusInternalServerError) + log.Println(err) + return + } + + data := make(map[string]interface{}) + + data["commit"] = diff.Commit + data["stat"] = diff.Stat + data["diff"] = diff.Diff + data["meta"] = h.c.Meta + data["ref"] = ref + data["desc"] = getDescription(path) + + writeJSON(w, data) + return +} + +func (h *Handle) Refs(w http.ResponseWriter, r *http.Request) { + path := filepath.Join(h.c.Repo.ScanPath, didPath(r)) + gr, err := git.Open(path, "") + if err != nil { + notFound(w) + return + } + + tags, err := gr.Tags() + if err != nil { + // Non-fatal, we *should* have at least one branch to show. + log.Println(err) + } + + branches, err := gr.Branches() + if err != nil { + log.Println(err) + writeError(w, err.Error(), http.StatusInternalServerError) + return + } + + data := make(map[string]interface{}) + + data["meta"] = h.c.Meta + data["branches"] = branches + data["tags"] = tags + data["desc"] = getDescription(path) + + writeJSON(w, data) + return +} + +func (h *Handle) ServeStatic(w http.ResponseWriter, r *http.Request) { + f := chi.URLParam(r, "file") + f = filepath.Clean(filepath.Join(h.c.Dirs.Static, f)) + + http.ServeFile(w, r, f) +} + +// func (h *Handle) Keys(w http.ResponseWriter, r *http.Request) { +// session, _ := h.s.Get(r, "bild-session") +// did := session.Values["did"].(string) + +// switch r.Method { +// case http.MethodGet: +// keys, err := h.db.GetPublicKeys(did) +// if err != nil { +// h.WriteOOBNotice(w, "keys", "Failed to list keys. Try again later.") +// log.Println(err) +// return +// } + +// data := make(map[string]interface{}) +// data["keys"] = keys +// if err := h.t.ExecuteTemplate(w, "settings/keys", data); err != nil { +// log.Println(err) +// return +// } +// case http.MethodPut: +// key := r.FormValue("key") +// name := r.FormValue("name") +// client, _ := h.auth.AuthorizedClient(r) + +// _, _, _, _, err := ssh.ParseAuthorizedKey([]byte(key)) +// if err != nil { +// h.WriteOOBNotice(w, "keys", "Invalid public key. Check your formatting and try again.") +// log.Printf("parsing public key: %s", err) +// return +// } + +// if err := h.db.AddPublicKey(did, name, key); err != nil { +// h.WriteOOBNotice(w, "keys", "Failed to add key.") +// log.Printf("adding public key: %s", err) +// return +// } + +// // store in pds too +// resp, err := comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{ +// Collection: "sh.bild.publicKey", +// Repo: did, +// Rkey: uuid.New().String(), +// Record: &lexutil.LexiconTypeDecoder{Val: &shbild.PublicKey{ +// Created: time.Now().String(), +// Key: key, +// Name: name, +// }}, +// }) + +// // invalid record +// if err != nil { +// h.WriteOOBNotice(w, "keys", "Invalid inputs. Check your formatting and try again.") +// log.Printf("failed to create record: %s", err) +// return +// } + +// log.Println("created atproto record: ", resp.Uri) + +// h.WriteOOBNotice(w, "keys", "Key added!") +// return +// } +// } + +// func (h *Handle) NewRepo(w http.ResponseWriter, r *http.Request) { +// session, _ := h.s.Get(r, "bild-session") +// did := session.Values["did"].(string) +// handle := session.Values["handle"].(string) + +// switch r.Method { +// case http.MethodGet: +// if err := h.t.ExecuteTemplate(w, "repo/new", nil); err != nil { +// log.Println(err) +// return +// } +// case http.MethodPut: +// name := r.FormValue("name") +// description := r.FormValue("description") + +// repoPath := filepath.Join(h.c.Repo.ScanPath, did, name) +// err := git.InitBare(repoPath) +// if err != nil { +// h.WriteOOBNotice(w, "repo", "Error creating repo. Try again later.") +// return +// } + +// err = h.db.AddRepo(did, name, description) +// if err != nil { +// h.WriteOOBNotice(w, "repo", "Error creating repo. Try again later.") +// return +// } + +// w.Header().Set("HX-Redirect", fmt.Sprintf("/@%s/%s", handle, name)) +// w.WriteHeader(http.StatusOK) +// } +// } + +// func (h *Handle) Timeline(w http.ResponseWriter, r *http.Request) { +// session, err := h.s.Get(r, "bild-session") +// user := make(map[string]string) +// if err != nil || session.IsNew { +// // user is not logged in +// } else { +// user["handle"] = session.Values["handle"].(string) +// user["did"] = session.Values["did"].(string) +// } + +// if err := h.t.ExecuteTemplate(w, "timeline", user); err != nil { +// log.Println(err) +// return +// } +// } diff --git a/knotserver/util.go b/knotserver/util.go new file mode 100644 --- /dev/null +++ b/knotserver/util.go @@ -0,0 +1,136 @@ +package knotserver + +import ( + "fmt" + "io/fs" + "log" + "net/http" + "os" + "path/filepath" + "strings" + + "github.com/go-chi/chi/v5" + "github.com/icyphox/bild/auth" + "github.com/icyphox/bild/git" + "github.com/microcosm-cc/bluemonday" +) + +func sanitize(content []byte) []byte { + return bluemonday.UGCPolicy().SanitizeBytes([]byte(content)) +} + +func displayRepoName(r *http.Request) string { + user := r.Context().Value("did").(string) + name := chi.URLParam(r, "name") + + handle, err := auth.ResolveIdent(r.Context(), user) + if err != nil { + log.Printf("failed to resolve ident: %s: %s", user, err) + return fmt.Sprintf("%s/%s", user, name) + } + + return fmt.Sprintf("@%s/%s", handle.Handle.String(), name) +} + +func didPath(r *http.Request, did string) string { + path := filepath.Join(did, chi.URLParam(r, "name")) + filepath.Clean(path) + return path +} + +func getDescription(path string) (desc string) { + db, err := os.ReadFile(filepath.Join(path, "description")) + if err == nil { + desc = string(db) + } else { + desc = "" + } + return +} + +func (h *Handle) isUnlisted(name string) bool { + for _, i := range h.c.Repo.Unlisted { + if name == i { + return true + } + } + + return false +} + +func (h *Handle) isIgnored(name string) bool { + for _, i := range h.c.Repo.Ignore { + if name == i { + return true + } + } + + return false +} + +type repoInfo struct { + Git *git.GitRepo + Path string + Category string +} + +func (d *Handle) getAllRepos() ([]repoInfo, error) { + repos := []repoInfo{} + max := strings.Count(d.c.Repo.ScanPath, string(os.PathSeparator)) + 2 + + err := filepath.WalkDir(d.c.Repo.ScanPath, func(path string, de fs.DirEntry, err error) error { + if err != nil { + return err + } + + if de.IsDir() { + // Check if we've exceeded our recursion depth + if strings.Count(path, string(os.PathSeparator)) > max { + return fs.SkipDir + } + + if d.isIgnored(path) { + return fs.SkipDir + } + + // A bare repo should always have at least a HEAD file, if it + // doesn't we can continue recursing + if _, err := os.Lstat(filepath.Join(path, "HEAD")); err == nil { + repo, err := git.Open(path, "") + if err != nil { + log.Println(err) + } else { + relpath, _ := filepath.Rel(d.c.Repo.ScanPath, path) + repos = append(repos, repoInfo{ + Git: repo, + Path: relpath, + Category: d.category(path), + }) + // Since we found a Git repo, we don't want to recurse + // further + return fs.SkipDir + } + } + } + return nil + }) + + return repos, err +} + +func (d *Handle) category(path string) string { + return strings.TrimPrefix(filepath.Dir(strings.TrimPrefix(path, d.c.Repo.ScanPath)), string(os.PathSeparator)) +} + +func setContentDisposition(w http.ResponseWriter, name string) { + h := "inline; filename=\"" + name + "\"" + w.Header().Add("Content-Disposition", h) +} + +func setGZipMIME(w http.ResponseWriter) { + setMIME(w, "application/gzip") +} + +func setMIME(w http.ResponseWriter, mime string) { + w.Header().Add("Content-Type", mime) +} diff --git a/cmd/knotserver/main.go b/cmd/knotserver/main.go new file mode 100644 --- /dev/null +++ b/cmd/knotserver/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "flag" + "fmt" + "log" + "log/slog" + "net/http" + "os" + + "github.com/icyphox/bild/config" + "github.com/icyphox/bild/db" + "github.com/icyphox/bild/routes" +) + +func main() { + var cfg string + flag.StringVar(&cfg, "config", "./config.yaml", "path to config file") + flag.Parse() + + slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, nil))) + + c, err := config.Read(cfg) + if err != nil { + log.Fatal(err) + } + db, err := db.Setup(c.Server.DBPath) + if err != nil { + log.Fatalf("failed to setup db: %s", err) + } + + mux, err := routes.Setup(c, db) + if err != nil { + log.Fatal(err) + } + + addr := fmt.Sprintf("%s:%d", c.Server.Host, c.Server.Port) + + log.Println("starting main server on", addr) + go http.ListenAndServe(addr, mux) +} diff --git a/knotserver/git/diff.go b/knotserver/git/diff.go new file mode 100644 --- /dev/null +++ b/knotserver/git/diff.go @@ -0,0 +1,119 @@ +package git + +import ( + "fmt" + "log" + "strings" + + "github.com/bluekeyes/go-gitdiff/gitdiff" + "github.com/go-git/go-git/v5/plumbing/object" +) + +type TextFragment struct { + Header string + Lines []gitdiff.Line +} + +type Diff struct { + Name struct { + Old string + New string + } + TextFragments []TextFragment + IsBinary bool + IsNew bool + IsDelete bool +} + +// A nicer git diff representation. +type NiceDiff struct { + Commit struct { + Message string + Author object.Signature + This string + Parent string + } + Stat struct { + FilesChanged int + Insertions int + Deletions int + } + Diff []Diff +} + +func (g *GitRepo) Diff() (*NiceDiff, error) { + c, err := g.r.CommitObject(g.h) + if err != nil { + return nil, fmt.Errorf("commit object: %w", err) + } + + patch := &object.Patch{} + commitTree, err := c.Tree() + parent := &object.Commit{} + if err == nil { + parentTree := &object.Tree{} + if c.NumParents() != 0 { + parent, err = c.Parents().Next() + if err == nil { + parentTree, err = parent.Tree() + if err == nil { + patch, err = parentTree.Patch(commitTree) + if err != nil { + return nil, fmt.Errorf("patch: %w", err) + } + } + } + } else { + patch, err = parentTree.Patch(commitTree) + if err != nil { + return nil, fmt.Errorf("patch: %w", err) + } + } + } + + diffs, _, err := gitdiff.Parse(strings.NewReader(patch.String())) + if err != nil { + log.Println(err) + } + + nd := NiceDiff{} + nd.Commit.This = c.Hash.String() + + if parent.Hash.IsZero() { + nd.Commit.Parent = "" + } else { + nd.Commit.Parent = parent.Hash.String() + } + nd.Commit.Author = c.Author + nd.Commit.Message = c.Message + + for _, d := range diffs { + ndiff := Diff{} + ndiff.Name.New = d.NewName + ndiff.Name.Old = d.OldName + ndiff.IsBinary = d.IsBinary + ndiff.IsNew = d.IsNew + ndiff.IsDelete = d.IsDelete + + for _, tf := range d.TextFragments { + ndiff.TextFragments = append(ndiff.TextFragments, TextFragment{ + Header: tf.Header(), + Lines: tf.Lines, + }) + for _, l := range tf.Lines { + switch l.Op { + case gitdiff.OpAdd: + nd.Stat.Insertions += 1 + case gitdiff.OpDelete: + nd.Stat.Deletions += 1 + } + } + } + + nd.Diff = append(nd.Diff, ndiff) + } + + nd.Stat.FilesChanged = len(diffs) + + return &nd, nil +} diff --git a/knotserver/git/git.go b/knotserver/git/git.go new file mode 100644 --- /dev/null +++ b/knotserver/git/git.go @@ -0,0 +1,344 @@ +package git + +import ( + "archive/tar" + "fmt" + "io" + "io/fs" + "path" + "sort" + "time" + + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" +) + +type GitRepo struct { + r *git.Repository + h plumbing.Hash +} + +type TagList struct { + refs []*TagReference + r *git.Repository +} + +// TagReference is used to list both tag and non-annotated tags. +// Non-annotated tags should only contains a reference. +// Annotated tags should contain its reference and its tag information. +type TagReference struct { + ref *plumbing.Reference + tag *object.Tag +} + +// infoWrapper wraps the property of a TreeEntry so it can export fs.FileInfo +// to tar WriteHeader +type infoWrapper struct { + name string + size int64 + mode fs.FileMode + modTime time.Time + isDir bool +} + +func (self *TagList) Len() int { + return len(self.refs) +} + +func (self *TagList) Swap(i, j int) { + self.refs[i], self.refs[j] = self.refs[j], self.refs[i] +} + +// sorting tags in reverse chronological order +func (self *TagList) Less(i, j int) bool { + var dateI time.Time + var dateJ time.Time + + if self.refs[i].tag != nil { + dateI = self.refs[i].tag.Tagger.When + } else { + c, err := self.r.CommitObject(self.refs[i].ref.Hash()) + if err != nil { + dateI = time.Now() + } else { + dateI = c.Committer.When + } + } + + if self.refs[j].tag != nil { + dateJ = self.refs[j].tag.Tagger.When + } else { + c, err := self.r.CommitObject(self.refs[j].ref.Hash()) + if err != nil { + dateJ = time.Now() + } else { + dateJ = c.Committer.When + } + } + + return dateI.After(dateJ) +} + +func Open(path string, ref string) (*GitRepo, error) { + var err error + g := GitRepo{} + g.r, err = git.PlainOpen(path) + if err != nil { + return nil, fmt.Errorf("opening %s: %w", path, err) + } + + if ref == "" { + head, err := g.r.Head() + if err != nil { + return nil, fmt.Errorf("getting head of %s: %w", path, err) + } + g.h = head.Hash() + } else { + hash, err := g.r.ResolveRevision(plumbing.Revision(ref)) + if err != nil { + return nil, fmt.Errorf("resolving rev %s for %s: %w", ref, path, err) + } + g.h = *hash + } + return &g, nil +} + +func (g *GitRepo) Commits() ([]*object.Commit, error) { + ci, err := g.r.Log(&git.LogOptions{From: g.h}) + if err != nil { + return nil, fmt.Errorf("commits from ref: %w", err) + } + + commits := []*object.Commit{} + ci.ForEach(func(c *object.Commit) error { + commits = append(commits, c) + return nil + }) + + return commits, nil +} + +func (g *GitRepo) LastCommit() (*object.Commit, error) { + c, err := g.r.CommitObject(g.h) + if err != nil { + return nil, fmt.Errorf("last commit: %w", err) + } + return c, nil +} + +func (g *GitRepo) FileContent(path string) (string, error) { + c, err := g.r.CommitObject(g.h) + if err != nil { + return "", fmt.Errorf("commit object: %w", err) + } + + tree, err := c.Tree() + if err != nil { + return "", fmt.Errorf("file tree: %w", err) + } + + file, err := tree.File(path) + if err != nil { + return "", err + } + + isbin, _ := file.IsBinary() + + if !isbin { + return file.Contents() + } else { + return "Not displaying binary file", nil + } +} + +func (g *GitRepo) Tags() ([]*TagReference, error) { + iter, err := g.r.Tags() + if err != nil { + return nil, fmt.Errorf("tag objects: %w", err) + } + + tags := make([]*TagReference, 0) + + if err := iter.ForEach(func(ref *plumbing.Reference) error { + obj, err := g.r.TagObject(ref.Hash()) + switch err { + case nil: + tags = append(tags, &TagReference{ + ref: ref, + tag: obj, + }) + case plumbing.ErrObjectNotFound: + tags = append(tags, &TagReference{ + ref: ref, + }) + default: + return err + } + return nil + }); err != nil { + return nil, err + } + + tagList := &TagList{r: g.r, refs: tags} + sort.Sort(tagList) + return tags, nil +} + +func (g *GitRepo) Branches() ([]*plumbing.Reference, error) { + bi, err := g.r.Branches() + if err != nil { + return nil, fmt.Errorf("branchs: %w", err) + } + + branches := []*plumbing.Reference{} + + _ = bi.ForEach(func(ref *plumbing.Reference) error { + branches = append(branches, ref) + return nil + }) + + return branches, nil +} + +func (g *GitRepo) FindMainBranch(branches []string) (string, error) { + for _, b := range branches { + _, err := g.r.ResolveRevision(plumbing.Revision(b)) + if err == nil { + return b, nil + } + } + return "", fmt.Errorf("unable to find main branch") +} + +// WriteTar writes itself from a tree into a binary tar file format. +// prefix is root folder to be appended. +func (g *GitRepo) WriteTar(w io.Writer, prefix string) error { + tw := tar.NewWriter(w) + defer tw.Close() + + c, err := g.r.CommitObject(g.h) + if err != nil { + return fmt.Errorf("commit object: %w", err) + } + + tree, err := c.Tree() + if err != nil { + return err + } + + walker := object.NewTreeWalker(tree, true, nil) + defer walker.Close() + + name, entry, err := walker.Next() + for ; err == nil; name, entry, err = walker.Next() { + info, err := newInfoWrapper(name, prefix, &entry, tree) + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + + err = tw.WriteHeader(header) + if err != nil { + return err + } + + if !info.IsDir() { + file, err := tree.File(name) + if err != nil { + return err + } + + reader, err := file.Blob.Reader() + if err != nil { + return err + } + + _, err = io.Copy(tw, reader) + if err != nil { + reader.Close() + return err + } + reader.Close() + } + } + + return nil +} + +func newInfoWrapper( + name string, + prefix string, + entry *object.TreeEntry, + tree *object.Tree, +) (*infoWrapper, error) { + var ( + size int64 + mode fs.FileMode + isDir bool + ) + + if entry.Mode.IsFile() { + file, err := tree.TreeEntryFile(entry) + if err != nil { + return nil, err + } + mode = fs.FileMode(file.Mode) + + size, err = tree.Size(name) + if err != nil { + return nil, err + } + } else { + isDir = true + mode = fs.ModeDir | fs.ModePerm + } + + fullname := path.Join(prefix, name) + return &infoWrapper{ + name: fullname, + size: size, + mode: mode, + modTime: time.Unix(0, 0), + isDir: isDir, + }, nil +} + +func (i *infoWrapper) Name() string { + return i.name +} + +func (i *infoWrapper) Size() int64 { + return i.size +} + +func (i *infoWrapper) Mode() fs.FileMode { + return i.mode +} + +func (i *infoWrapper) ModTime() time.Time { + return i.modTime +} + +func (i *infoWrapper) IsDir() bool { + return i.isDir +} + +func (i *infoWrapper) Sys() any { + return nil +} + +func (t *TagReference) Name() string { + return t.ref.Name().Short() +} + +func (t *TagReference) Message() string { + if t.tag != nil { + return t.tag.Message + } + return "" +} diff --git a/knotserver/git/repo.go b/knotserver/git/repo.go new file mode 100644 --- /dev/null +++ b/knotserver/git/repo.go @@ -0,0 +1,33 @@ +package git + +import ( + "errors" + "fmt" + "os" + "path/filepath" + + gogit "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/config" +) + +func InitBare(path string) error { + parent := filepath.Dir(path) + + if err := os.MkdirAll(parent, 0755); errors.Is(err, os.ErrExist) { + return fmt.Errorf("error creating user directory: %w", err) + } + + repository, err := gogit.PlainInit(path, true) + if err != nil { + return err + } + + err = repository.CreateBranch(&config.Branch{ + Name: "main", + }) + if err != nil { + return fmt.Errorf("creating branch: %w", err) + } + + return nil +} diff --git a/knotserver/git/tree.go b/knotserver/git/tree.go new file mode 100644 --- /dev/null +++ b/knotserver/git/tree.go @@ -0,0 +1,66 @@ +package git + +import ( + "fmt" + + "github.com/go-git/go-git/v5/plumbing/object" +) + +func (g *GitRepo) FileTree(path string) ([]NiceTree, error) { + c, err := g.r.CommitObject(g.h) + if err != nil { + return nil, fmt.Errorf("commit object: %w", err) + } + + files := []NiceTree{} + tree, err := c.Tree() + if err != nil { + return nil, fmt.Errorf("file tree: %w", err) + } + + if path == "" { + files = makeNiceTree(tree) + } else { + o, err := tree.FindEntry(path) + if err != nil { + return nil, err + } + + if !o.Mode.IsFile() { + subtree, err := tree.Tree(path) + if err != nil { + return nil, err + } + + files = makeNiceTree(subtree) + } + } + + return files, nil +} + +// A nicer git tree representation. +type NiceTree struct { + Name string + Mode string + Size int64 + IsFile bool + IsSubtree bool +} + +func makeNiceTree(t *object.Tree) []NiceTree { + nts := []NiceTree{} + + for _, e := range t.Entries { + mode, _ := e.Mode.ToOSFileMode() + sz, _ := t.Size(e.Name) + nts = append(nts, NiceTree{ + Name: e.Name, + Mode: mode.String(), + IsFile: e.Mode.IsFile(), + Size: sz, + }) + } + + return nts +} diff --git a/knotserver/git/service/service.go b/knotserver/git/service/service.go new file mode 100644 --- /dev/null +++ b/knotserver/git/service/service.go @@ -0,0 +1,121 @@ +package service + +import ( + "bytes" + "fmt" + "io" + "log" + "net/http" + "os/exec" + "strings" + "syscall" +) + +// Mostly from charmbracelet/soft-serve and sosedoff/gitkit. + +type ServiceCommand struct { + Dir string + Stdin io.Reader + Stdout http.ResponseWriter +} + +func (c *ServiceCommand) InfoRefs() error { + cmd := exec.Command("git", []string{ + "upload-pack", + "--stateless-rpc", + "--advertise-refs", + ".", + }...) + + cmd.Dir = c.Dir + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + stdoutPipe, _ := cmd.StdoutPipe() + cmd.Stderr = cmd.Stdout + + if err := cmd.Start(); err != nil { + log.Printf("git: failed to start git-upload-pack (info/refs): %s", err) + return err + } + + if err := packLine(c.Stdout, "# service=git-upload-pack\n"); err != nil { + log.Printf("git: failed to write pack line: %s", err) + return err + } + + if err := packFlush(c.Stdout); err != nil { + log.Printf("git: failed to flush pack: %s", err) + return err + } + + buf := bytes.Buffer{} + if _, err := io.Copy(&buf, stdoutPipe); err != nil { + log.Printf("git: failed to copy stdout to tmp buffer: %s", err) + return err + } + + if err := cmd.Wait(); err != nil { + out := strings.Builder{} + _, _ = io.Copy(&out, &buf) + log.Printf("git: failed to run git-upload-pack; err: %s; output: %s", err, out.String()) + return err + } + + if _, err := io.Copy(c.Stdout, &buf); err != nil { + log.Printf("git: failed to copy stdout: %s", err) + } + + return nil +} + +func (c *ServiceCommand) UploadPack() error { + cmd := exec.Command("git", []string{ + "-c", "uploadpack.allowFilter=true", + "upload-pack", + "--stateless-rpc", + ".", + }...) + cmd.Dir = c.Dir + cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} + + stdoutPipe, _ := cmd.StdoutPipe() + cmd.Stderr = cmd.Stdout + defer stdoutPipe.Close() + + stdinPipe, err := cmd.StdinPipe() + if err != nil { + return err + } + defer stdinPipe.Close() + + if err := cmd.Start(); err != nil { + log.Printf("git: failed to start git-upload-pack: %s", err) + return err + } + + if _, err := io.Copy(stdinPipe, c.Stdin); err != nil { + log.Printf("git: failed to copy stdin: %s", err) + return err + } + stdinPipe.Close() + + if _, err := io.Copy(newWriteFlusher(c.Stdout), stdoutPipe); err != nil { + log.Printf("git: failed to copy stdout: %s", err) + return err + } + if err := cmd.Wait(); err != nil { + log.Printf("git: failed to wait for git-upload-pack: %s", err) + return err + } + + return nil +} + +func packLine(w io.Writer, s string) error { + _, err := fmt.Fprintf(w, "%04x%s", len(s)+4, s) + return err +} + +func packFlush(w io.Writer) error { + _, err := fmt.Fprint(w, "0000") + return err +} diff --git a/knotserver/git/service/write_flusher.go b/knotserver/git/service/write_flusher.go new file mode 100644 --- /dev/null +++ b/knotserver/git/service/write_flusher.go @@ -0,0 +1,25 @@ +package service + +import ( + "io" + "net/http" +) + +func newWriteFlusher(w http.ResponseWriter) io.Writer { + return writeFlusher{w.(interface { + io.Writer + http.Flusher + })} +} + +type writeFlusher struct { + wf interface { + io.Writer + http.Flusher + } +} + +func (w writeFlusher) Write(p []byte) (int, error) { + defer w.wf.Flush() + return w.wf.Write(p) +} -- tangled.sh