From d3d6a88814063ba1ee36d0c7f62cc526e9725db2 Mon Sep 17 00:00:00 2001 From: Lewis Date: Fri, 31 Jul 2026 17:14:56 +0300 Subject: [PATCH] gitutil,knotmirror/xrpc: match tree/branch responses that knot serves Lewis: May this revision serve well! --- appview/pages/markup/format.go | 17 --------- appview/pages/markup/format_test.go | 53 ---------------------------- appview/repo/index.go | 4 +-- appview/repo/tree.go | 4 +-- gitutil/readme.go | 22 ++++++++++++ gitutil/readme_test.go | 48 +++++++++++++++++++++++++ knotmirror/xrpc/git_get_tree.go | 44 +++++++++++++++++++++-- knotmirror/xrpc/git_list_branches.go | 2 ++ knotserver/xrpc/repo_tree.go | 4 +-- 9 files changed, 119 insertions(+), 79 deletions(-) create mode 100644 gitutil/readme.go create mode 100644 gitutil/readme_test.go diff --git a/appview/pages/markup/format.go b/appview/pages/markup/format.go index f4c87d8f..c38d326c 100644 --- a/appview/pages/markup/format.go +++ b/appview/pages/markup/format.go @@ -2,8 +2,6 @@ package markup import ( "regexp" - - "github.com/go-git/go-git/v5/plumbing/filemode" ) type Format string @@ -17,21 +15,6 @@ var FileTypePatterns = map[Format]*regexp.Regexp{ FormatMarkdown: regexp.MustCompile(`(?i)\.(md|markdown|mdown|mkdn|mkd)$`), } -var ReadmePattern = regexp.MustCompile(`(?i)^readme(?:\.[^.]+)?$`) - -// IsReadmeFile reports whether name/mode identifies a readme blob. The git -// mode is checked so directories or symlinks named "readme" are filtered out. -func IsReadmeFile(name, mode string) bool { - if !ReadmePattern.MatchString(name) { - return false - } - m, err := filemode.New(mode) - if err != nil { - return false - } - return m == filemode.Regular || m == filemode.Executable -} - // GetFormat returns the Format whose extension list matches filename, // falling back to FormatText. func GetFormat(filename string) Format { diff --git a/appview/pages/markup/format_test.go b/appview/pages/markup/format_test.go index b04ad01a..2a878d3d 100644 --- a/appview/pages/markup/format_test.go +++ b/appview/pages/markup/format_test.go @@ -2,59 +2,6 @@ package markup import "testing" -func TestIsReadmeFile(t *testing.T) { - const ( - fileMode = "100644" - execMode = "100755" - dirMode = "040000" - ) - - cases := []struct { - name string - mode string - want bool - }{ - {"README.md", fileMode, true}, - {"readme.md", fileMode, true}, - {"ReadMe.MD", fileMode, true}, - {"README.markdown", fileMode, true}, - {"README.mdown", fileMode, true}, - {"README.mkdn", fileMode, true}, - {"README.mkd", fileMode, true}, - {"README.txt", fileMode, true}, - {"readme", fileMode, true}, - {"README", execMode, true}, - - // regression: a directory named "readme" must not be picked up - // as the README blob; tree handlers used to fetch it and 503. - {"readme", dirMode, false}, - {"README.md", dirMode, false}, - - // readme is matched by convention, not by renderable format; - // unsupported markup falls through to plaintext in GetFormat. - {"README.rst", fileMode, true}, - {"README.org", fileMode, true}, - {"README.asciidoc", fileMode, true}, - {"README.foo", fileMode, true}, - - {"README.Music.md", fileMode, false}, - {"readme-old", fileMode, false}, - {"readme_legacy", fileMode, false}, - {"notreadme.md", fileMode, false}, - {"READMEISH", fileMode, false}, - {"README.md", "", false}, - {"README.md", "120000", false}, // symlink - } - - for _, c := range cases { - t.Run(c.name+"/"+c.mode, func(t *testing.T) { - if got := IsReadmeFile(c.name, c.mode); got != c.want { - t.Errorf("IsReadmeFile(%q, %q) = %v, want %v", c.name, c.mode, got, c.want) - } - }) - } -} - func TestFileTypePatterns(t *testing.T) { cases := []struct { format Format diff --git a/appview/repo/index.go b/appview/repo/index.go index 7aa03b29..95b51b6a 100644 --- a/appview/repo/index.go +++ b/appview/repo/index.go @@ -22,8 +22,8 @@ import ( "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/pages" - "tangled.org/core/appview/pages/markup" "tangled.org/core/appview/pipelines" + "tangled.org/core/gitutil" "tangled.org/core/types" "github.com/go-chi/chi/v5" @@ -322,7 +322,7 @@ func (rp *Repo) buildIndexResponse(ctx context.Context, repo *models.Repo, ref s treeResp = resp for _, file := range resp.Files { - if markup.IsReadmeFile(file.Name, file.Mode) { + if gitutil.IsReadmeFile(file.Name, file.Mode) { readmeFileName = file.Name break } diff --git a/appview/repo/tree.go b/appview/repo/tree.go index 16807c59..9c278494 100644 --- a/appview/repo/tree.go +++ b/appview/repo/tree.go @@ -11,8 +11,8 @@ import ( "tangled.org/core/api/tangled" "tangled.org/core/appview/db" "tangled.org/core/appview/pages" - "tangled.org/core/appview/pages/markup" "tangled.org/core/appview/reporesolver" + "tangled.org/core/gitutil" "tangled.org/core/types" xrpcclient "tangled.org/core/xrpc/xrpcclient" @@ -71,7 +71,7 @@ func (rp *Repo) Tree(w http.ResponseWriter, r *http.Request) { } } files[i] = file - if markup.IsReadmeFile(xrpcFile.Name, xrpcFile.Mode) { + if gitutil.IsReadmeFile(xrpcFile.Name, xrpcFile.Mode) { readmeFile = xrpcFile } } diff --git a/gitutil/readme.go b/gitutil/readme.go new file mode 100644 index 00000000..78dd5187 --- /dev/null +++ b/gitutil/readme.go @@ -0,0 +1,22 @@ +package gitutil + +import ( + "regexp" + + "github.com/go-git/go-git/v5/plumbing/filemode" +) + +var readmePattern = regexp.MustCompile(`(?i)^readme(?:\.[^.]+)?$`) + +// IsReadmeFile reports whether name/mode identifies a readme blob. The git +// mode is checked so directories or symlinks named "readme" are filtered out. +func IsReadmeFile(name, mode string) bool { + if !readmePattern.MatchString(name) { + return false + } + m, err := filemode.New(mode) + if err != nil { + return false + } + return m == filemode.Regular || m == filemode.Executable +} diff --git a/gitutil/readme_test.go b/gitutil/readme_test.go new file mode 100644 index 00000000..ea0d335d --- /dev/null +++ b/gitutil/readme_test.go @@ -0,0 +1,48 @@ +package gitutil + +import "testing" + +func TestIsReadmeFile(t *testing.T) { + const ( + fileMode = "100644" + execMode = "100755" + dirMode = "040000" + ) + + cases := []struct { + name string + mode string + want bool + }{ + {"README.md", fileMode, true}, + {"ReadMe.MD", fileMode, true}, + {"readme", fileMode, true}, + {"README", execMode, true}, + + // regression: a directory named "readme" must not be picked up + // as the README blob; tree handlers used to fetch it and 503. + {"readme", dirMode, false}, + {"README.md", dirMode, false}, + + // readme is matched by convention, not by renderable format; + // unsupported markup falls through to plaintext in GetFormat. + {"README.rst", fileMode, true}, + {"README.foo", fileMode, true}, + + {"README.Music.md", fileMode, false}, + {"readme-old", fileMode, false}, + {"readme_legacy", fileMode, false}, + {"notreadme.md", fileMode, false}, + {"READMEISH", fileMode, false}, + {"README.md", "", false}, + {"README.md", "120000", false}, // symlink + } + + for _, c := range cases { + t.Run(c.name+"/"+c.mode, func(t *testing.T) { + if got := IsReadmeFile(c.name, c.mode); got != c.want { + t.Errorf("IsReadmeFile(%q, %q) = %v, want %v", c.name, c.mode, got, c.want) + } + }) + } +} diff --git a/knotmirror/xrpc/git_get_tree.go b/knotmirror/xrpc/git_get_tree.go index f886f726..1b418db7 100644 --- a/knotmirror/xrpc/git_get_tree.go +++ b/knotmirror/xrpc/git_get_tree.go @@ -7,6 +7,7 @@ import ( "net/http" "path/filepath" "time" + "unicode/utf8" "github.com/bluesky-social/indigo/atproto/atclient" "github.com/bluesky-social/indigo/atproto/syntax" @@ -14,12 +15,14 @@ import ( "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/object" "tangled.org/core/api/tangled" + "tangled.org/core/gitutil" "tangled.org/core/knotmirror/xrpc/gitea" ) const ( LastCommitCache = "last_commit:%s:%s" LastCommitCacheTTL = 30 * 24 * time.Hour + MaxReadmeBytes = 1 << 20 ) func (x *Xrpc) GetTree(w http.ResponseWriter, r *http.Request) { @@ -170,6 +173,11 @@ func (x *Xrpc) getTree(ctx context.Context, repo syntax.DID, ref, treePath strin return nil, err } + sizes, err := gitea.EntrySizes(ctx, repoPath, subTree.Entries) + if err != nil { + x.logger.Warn("tree entry size read failed", "err", err) + } + outEntries := make([]*tangled.GitTempGetTree_TreeEntry, len(subTree.Entries)) for i, entry := range subTree.Entries { var entryLastCommit *tangled.GitTempGetTree_LastCommit @@ -187,6 +195,7 @@ func (x *Xrpc) getTree(ctx context.Context, repo syntax.DID, ref, treePath strin outEntries[i] = &tangled.GitTempGetTree_TreeEntry{ Name: entry.Name, Mode: entry.Mode.String(), + Size: sizes[i], Last_commit: entryLastCommit, } } @@ -195,7 +204,7 @@ func (x *Xrpc) getTree(ctx context.Context, repo syntax.DID, ref, treePath strin var dotdot *string if treePath != "" { parent = &treePath - if dir := filepath.Dir(treePath); dir != "" { + if dir := filepath.Dir(treePath); dir != "." { dotdot = &dir } } @@ -213,6 +222,8 @@ func (x *Xrpc) getTree(ctx context.Context, repo syntax.DID, ref, treePath strin } } + readmeName, readmeContents := x.readme(ctx, repoPath, subTree.Entries, sizes) + return &tangled.GitTempGetTree_Output{ Ref: ref, Parent: parent, @@ -221,8 +232,35 @@ func (x *Xrpc) getTree(ctx context.Context, repo syntax.DID, ref, treePath strin LastCommit: outLastCommit, // TODO: remove this field entirely Readme: &tangled.GitTempGetTree_Readme{ - Filename: "", - Contents: "", + Filename: readmeName, + Contents: readmeContents, }, }, nil } + +func (x *Xrpc) readme(ctx context.Context, repoPath string, entries []object.TreeEntry, sizes []int64) (string, string) { + for i, entry := range entries { + if !gitutil.IsReadmeFile(entry.Name, entry.Mode.String()) || sizes[i] > MaxReadmeBytes { + continue + } + size, reader, err := gitea.ReadBlob(ctx, repoPath, entry.Hash) + if err != nil { + x.logger.Warn("readme blob open failed", "file", entry.Name, "err", err) + continue + } + if size > MaxReadmeBytes { + reader.Close() + continue + } + contents, err := io.ReadAll(reader) + reader.Close() + if err != nil { + x.logger.Warn("readme blob read failed", "file", entry.Name, "err", err) + continue + } + if utf8.Valid(contents) { + return entry.Name, string(contents) + } + } + return "", "" +} diff --git a/knotmirror/xrpc/git_list_branches.go b/knotmirror/xrpc/git_list_branches.go index df60f153..ed5a7234 100644 --- a/knotmirror/xrpc/git_list_branches.go +++ b/knotmirror/xrpc/git_list_branches.go @@ -9,6 +9,7 @@ import ( "os" "os/exec" "path/filepath" + "slices" "strconv" "strings" @@ -163,6 +164,7 @@ func (x *Xrpc) listBranches(ctx context.Context, repo syntax.DID, limit int, cur if err != nil { return nil, fmt.Errorf("hydrating branch commits: %w", err) } + slices.Reverse(branches) // -> total total, err := func(repoPath string) (int, error) { diff --git a/knotserver/xrpc/repo_tree.go b/knotserver/xrpc/repo_tree.go index df773b92..72ed9022 100644 --- a/knotserver/xrpc/repo_tree.go +++ b/knotserver/xrpc/repo_tree.go @@ -7,7 +7,7 @@ import ( "unicode/utf8" "tangled.org/core/api/tangled" - "tangled.org/core/appview/pages/markup" + "tangled.org/core/gitutil" "tangled.org/core/knotserver/git" "tangled.org/core/types" xrpcerr "tangled.org/core/xrpc/errors" @@ -50,7 +50,7 @@ func (x *Xrpc) RepoTree(w http.ResponseWriter, r *http.Request) { var readmeFileName string var readmeContents string for _, file := range files { - if markup.IsReadmeFile(file.Name, file.Mode) { + if gitutil.IsReadmeFile(file.Name, file.Mode) { contents, err := gr.RawContent(filepath.Join(path, file.Name)) if err != nil { x.Logger.Error("failed to read contents of file", "path", path, "file", file.Name) -- 2.51.2