Something went wrong. Try again.
Monorepo for Tangled
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282package gitutil
import ( "archive/zip" "bytes" "context" "errors" "mime" "net/http" "net/http/httptest" "net/url" "os" "os/exec" "path/filepath" "strings" "testing" "unicode/utf8"
"github.com/go-git/go-git/v5/plumbing" "github.com/samber/lo" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require")
const testArchiveEndpoint = "https://knot.nel.pet/xrpc/sh.tangled.repo.archive"
func TestParseArchiveParams(t *testing.T) { cases := []struct { name string query url.Values repo RepoName want ArchiveParams wantStem ArchivePrefix wantFilename string wantType string wantErr string }{ {"all empty", url.Values{}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, {"tar.gz", url.Values{"format": {"tar.gz"}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, {"zip", url.Values{"format": {"zip"}}, "", ArchiveParams{Format: ArchiveZip}, "", "", "", ""}, { "every param", url.Values{"ref": {"refs/tags/v1.0.0"}, "format": {"zip"}, "prefix": {"/kelp/"}}, "", ArchiveParams{Rev: "refs/tags/v1.0.0", Format: ArchiveZip, Prefix: "kelp"}, "kelp", "kelp.zip", "application/zip", "", }, {"branch", url.Values{"ref": {"main"}}, "", ArchiveParams{Rev: "main", Format: ArchiveTarGz}, "squid-main", "squid-main.tar.gz", "application/gzip", ""}, {"nested prefix in the filename", url.Values{"ref": {"main"}, "prefix": {"kelp/uni"}}, "", ArchiveParams{Rev: "main", Format: ArchiveTarGz, Prefix: "kelp/uni"}, "kelp/uni", "kelp-uni.tar.gz", "application/gzip", ""}, {"full ref", url.Values{"ref": {"refs/heads/feat/uni"}}, "", ArchiveParams{Rev: "refs/heads/feat/uni", Format: ArchiveTarGz}, "squid-feat-uni", "squid-feat-uni.tar.gz", "application/gzip", ""}, {"head", url.Values{"ref": {"HEAD"}}, "", ArchiveParams{Rev: "HEAD", Format: ArchiveTarGz}, "squid-HEAD", "", "", ""}, {"trailing slash", url.Values{"ref": {"refs/heads/main/"}}, "", ArchiveParams{Rev: "refs/heads/main/", Format: ArchiveTarGz}, "squid-main-", "", "", ""}, {"traversal in a ref", url.Values{"ref": {"../../etc"}}, "", ArchiveParams{Rev: "../../etc", Format: ArchiveTarGz}, "squid-..-..-etc", "", "", ""}, {"windows separator in a ref", url.Values{"ref": {`feat\uni`}}, "", ArchiveParams{Rev: `feat\uni`, Format: ArchiveTarGz}, "squid-feat-uni", "", "", ""}, {"slash in a repo name", url.Values{"ref": {"main"}}, "kelp/limpet", ArchiveParams{Rev: "main", Format: ArchiveTarGz}, "kelp-limpet-main", "kelp-limpet-main.tar.gz", "application/gzip", ""}, {"quote in a repo name", url.Values{"ref": {"main"}}, `squid-a"b`, ArchiveParams{Rev: "main", Format: ArchiveTarGz}, `squid-a"b-main`, `squid-a"b-main.tar.gz`, "application/gzip", ""}, {"non-ascii repo name", url.Values{"ref": {"main"}, "format": {"zip"}}, "squid-über", ArchiveParams{Rev: "main", Format: ArchiveZip}, "squid-über-main", "squid-über-main.zip", "application/zip", ""},
{"bare slash prefix", url.Values{"prefix": {"/"}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, {"did prefix", url.Values{"prefix": {"did:plc:boltless"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "did:plc:boltless"}, "", "", "", ""}, {"nested prefix", url.Values{"prefix": {"squid/main"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid/main"}, "", "", "", ""}, {"prefix wrapped in slashes", url.Values{"prefix": {"/squid/main/"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid/main"}, "", "", "", ""}, {"space in a prefix", url.Values{"prefix": {"squid main"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "squid main"}, "", "", "", ""}, {"bare dot prefix", url.Values{"prefix": {"."}}, "", ArchiveParams{Format: ArchiveTarGz}, "", "", "", ""}, {"repeated separators and dot segments", url.Values{"prefix": {"/kelp//./uni/"}}, "", ArchiveParams{Format: ArchiveTarGz, Prefix: "kelp/uni"}, "", "", "", ""},
{"unsupported format", url.Values{"format": {"tar"}}, "", ArchiveParams{}, "", "", "", "only tar.gz and zip formats are supported"}, {"space in a ref", url.Values{"ref": {"refs/tags/a b"}}, "", ArchiveParams{}, "", "", "", "ref contains whitespace"}, {"control character in a ref", url.Values{"ref": {"refs/tags/a\nb"}}, "", ArchiveParams{}, "", "", "", "ref contains whitespace"}, {"ref that git would read as an option", url.Values{"ref": {"--output=/tmp/evil"}}, "", ArchiveParams{}, "", "", "", "ref starts with a dash"}, {"prefix escaping the root", url.Values{"prefix": {"../../evil"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"}, {"prefix escaping below the root", url.Values{"prefix": {"squid/../../evil"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"}, {"a dot-dot segment the knot would reject", url.Values{"prefix": {"squid/../limpet"}}, "", ArchiveParams{}, "", "", "", "prefix escapes the archive root"}, {"control character in a prefix", url.Values{"prefix": {"squid\nmain"}}, "", ArchiveParams{}, "", "", "", "prefix contains a control character"}, {"windows separator in a prefix", url.Values{"prefix": {`..\..\evil`}}, "", ArchiveParams{}, "", "", "", "prefix contains a backslash"}, {"prefix over the length limit", url.Values{"prefix": {strings.Repeat("a", MaxArchivePrefixLen+1)}}, "", ArchiveParams{}, "", "", "", "over the 255 byte limit"}, }
for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got, err := ParseArchiveParams(tc.query) rejected := err != nil if got != tc.want || rejected != (tc.wantErr != "") || (rejected && !strings.Contains(err.Error(), tc.wantErr)) { t.Fatalf("params = %+v with err %v, want %+v and an error mentioning %q", got, err, tc.want, tc.wantErr) } if rejected { return }
repo := RepoName("squid") if tc.repo != "" { repo = tc.repo } served := got.Serve(repo) query := served.Query("did:plc:limpet") if stem := ArchivePrefix(query.Get("prefix")); tc.wantStem != "" && stem != tc.wantStem { t.Errorf("served prefix = %q, want %q", stem, tc.wantStem) } if tc.wantFilename != "" { header := http.Header{} served.SetHeaders(header) mediatype, fields, err := mime.ParseMediaType(header.Get("Content-Disposition")) if err != nil || mediatype != "attachment" || fields["filename"] != tc.wantFilename { t.Errorf("Content-Disposition = %q (err %v), want an attachment with filename %q", header.Get("Content-Disposition"), err, tc.wantFilename) } if ct, sniff := header.Get("Content-Type"), header.Get("X-Content-Type-Options"); ct != tc.wantType || sniff != "nosniff" { t.Errorf("Content-Type = %q with X-Content-Type-Options %q, want %q and nosniff", ct, sniff, tc.wantType) } }
if back, err := ParseArchiveParams(query); err != nil || back.Serve(repo) != served { t.Errorf("query round trip = %+v (err %v), want the served archive %+v", back, err, served) } back, err := ParseImmutableLink(ImmutableLink(testArchiveEndpoint + "?" + query.Encode())) if got.Rev != "" && (err != nil || back != got.Rev) { t.Errorf("Link round trip = %q (err %v), want %q", back, err, got.Rev) } }) }}
func TestArchiveFallbacks(t *testing.T) { hash := RevFromHash(plumbing.NewHash("6f1d3a2b4c5d6e7f8091a2b3c4d5e6f708192a3b")) if kept, filled := Rev("refs/heads/main").Or(hash), Rev("").Or(hash); kept != "refs/heads/main" || filled != hash { t.Errorf("Or kept %q and filled %q, want refs/heads/main and the hash %q", kept, filled, hash) } if got := Rev("").Or(RevHead); got != RevHead { t.Errorf("empty rev = %q, want HEAD", got) } if _, err := ParseImmutableLink(""); err == nil { t.Error("ParseImmutableLink must reject an empty header") }
params := ArchiveParams{Rev: "main", Format: ArchiveZip, Prefix: "kelp"} if got := params.WithRev("6f1d3a2"); got != (ArchiveParams{Rev: "6f1d3a2", Format: ArchiveZip, Prefix: "kelp"}) || params.Rev != "main" { t.Errorf("WithRev = %+v leaving the receiver at %q, want only the rev replaced", got, params.Rev) }
served := params.Serve("squid") if got := served.WithRev("6f1d3a2"); got != (ArchiveParams{Rev: "6f1d3a2", Format: ArchiveZip, Prefix: "kelp"}).Serve("squid") || served.rev != "main" { t.Errorf("WithRev = %+v leaving the receiver at %q, want only the rev replaced", got, served.rev) }
for _, stem := range []ArchivePrefix{ archiveStem("squid", Rev("refs/heads/"+strings.Repeat("ü", 400))), archiveStem(RepoName(strings.Repeat("\xff", 300)), "main"), } { _, err := ParseArchivePrefix(stem.String()) if stem == "" || len(stem) > MaxArchivePrefixLen || !utf8.ValidString(stem.String()) || err != nil { t.Errorf("default prefix is %d bytes %q (err %v), want a non-empty valid prefix within %d bytes ending on a rune boundary", len(stem), stem, err, MaxArchivePrefixLen) } }}
func TestRevIsObjectID(t *testing.T) { sha1Hex, sha256Hex := "6f1d3a2b4c5d6e7f8091a2b3c4d5e6f708192a3b", strings.Repeat("6f1d3a2b", 8) for rev, want := range map[Rev]bool{ Rev(sha1Hex): true, Rev(sha256Hex): true, "": false, "main": false, "refs/heads/main": false, "6f1d3a2": false, RevHead: false, Rev(strings.ToUpper(sha1Hex)): false, Rev(sha1Hex + "b"): false, Rev(sha1Hex[:39] + "g"): false, Rev(strings.Repeat("6", 48)): false, } { assert.Equal(t, want, rev.IsObjectID(), "%q: only 40 or 64 characters of lowercase hex identify one commit forever", rev) }}
func TestArchiveResponseHeaders(t *testing.T) { upstream := http.Header{"Etag": {`"6f1d3a2b"`}, "Content-Length": {"4096"}, "Cache-Control": {"no-cache"}}
got := http.Header{"Etag": {`"limpet"`, `"conch"`}} ForwardHeaders(got, upstream, "Etag", "Content-Length", "Link") assert.Equal(t, http.Header{"Etag": {`"6f1d3a2b"`}, "Content-Length": {"4096"}}, got, "ForwardHeaders overwrites Etag, leaves the unlisted Cache-Control alone, and skips the Link that the knot never sent")
revalidation := http.Header{} ForwardHeaders(revalidation, upstream, "Etag") assert.Equal(t, http.Header{"Etag": {`"6f1d3a2b"`}}, revalidation, "the 304 path forwards the validator without a length")
validators := http.Header{} ForwardHeaders(validators, http.Header{"If-None-Match": {`"kelp"`, `"uni"`}}, "if-none-match") assert.Equal(t, []string{`"kelp"`, `"uni"`}, validators.Values("If-None-Match"), "ForwardHeaders canonicalizes a lowercase key and sends every validator that a client offered")
untouched := httptest.NewRecorder() ArchiveParams{Rev: "main", Format: ArchiveTarGz}.Serve("squid").SetHeaders(untouched.Header()) untouched.Header().Set("Content-Length", "4096") untouched.Header().Set("Etag", `"6f1d3a2b"`) untouched.Header().Set("Link", ImmutableLink(testArchiveEndpoint+"?ref=6f1d3a2b")) untouched.Header().Set("Cache-Control", "public, max-age=31536000") NewResponseBody(untouched).Fail() assert.Equal(t, http.StatusInternalServerError, untouched.Code) assert.Equal(t, http.Header{"Cache-Control": {"no-store"}, "X-Content-Type-Options": {"nosniff"}}, untouched.Header(), "Fail deletes the length, the type, the filename, the validator and the link, because a 500 doesn't describe the archive they came from")
body := NewResponseBody(httptest.NewRecorder()) written, err := body.Write([]byte("PK\x03\x04")) require.NoError(t, err) assert.Equal(t, 4, written) assert.PanicsWithError(t, http.ErrAbortHandler.Error(), body.Fail, "a truncated body has to abort the connection, since a clean return reads as a complete archive")
archive := ArchiveParams{Rev: "6f1d3a2b", Format: ArchiveTarGz}.Serve("squid") etag := archive.ETag("did:plc:limpet") assert.Regexp(t, `^"[0-9a-f]{64}"$`, etag) assert.NotContains(t, []string{ archive.ETag("did:plc:conch"), archive.WithRev("main").ETag("did:plc:limpet"), ArchiveParams{Rev: "6f1d3a2b", Format: ArchiveZip}.Serve("squid").ETag("did:plc:limpet"), ArchiveParams{Rev: "6f1d3a2b", Format: ArchiveTarGz}.Serve("kelp").ETag("did:plc:limpet"), }, etag, "a repo, rev, format or prefix that shapes different bytes gets a different validator")
for offered, want := range map[string]bool{ etag: true, "W/" + etag: true, "*": true, `"conch", ` + etag: true, `"conch"`: false, "": false, } { request := httptest.NewRequest(http.MethodGet, testArchiveEndpoint, nil) if offered != "" { request.Header.Set("If-None-Match", offered) } recorder := httptest.NewRecorder() assert.Equal(t, want, archive.ServeNotModified(recorder, request, "did:plc:limpet"), "If-None-Match %q", offered) assert.Equal(t, lo.Ternary(want, http.StatusNotModified, http.StatusOK), recorder.Code) assert.Equal(t, etag, recorder.Header().Get("Etag"), "the knot sets the validator on every answer, so a client with a stale copy learns it too") assert.Equal(t, "no-cache", recorder.Header().Get("Cache-Control")) }}
func TestWriteArchive(t *testing.T) { repoPath := t.TempDir() require.NoError(t, os.WriteFile(filepath.Join(repoPath, "README.md"), []byte("# squid\n"), 0644)) for _, args := range [][]string{ {"init", "-q", "-b", "main"}, {"add", "README.md"}, {"-c", "user.name=nel", "-c", "user.email=noreply@nel.pet", "commit", "-qm", "Initial commit"}, {"branch", "feat/uni"}, } { cmd := exec.Command("git", args...) cmd.Dir = repoPath require.NoError(t, cmd.Run(), "git %v", args) }
canceled, cancel := context.WithCancel(context.Background()) cancel()
head := ArchiveParams{Rev: RevHead, Format: ArchiveZip} cases := []struct { name string ctx context.Context archive ServedArchive want []string }{ {"the defaulted prefix begins every entry", context.Background(), ArchiveParams{Rev: "refs/heads/feat/uni", Format: ArchiveZip}.Serve("squid"), []string{"squid-feat-uni/", "squid-feat-uni/README.md"}}, {"a requested prefix replaces the default", context.Background(), ArchiveParams{Rev: RevHead, Format: ArchiveZip, Prefix: "kelp"}.Serve("squid"), []string{"kelp/", "kelp/README.md"}}, {"canceled context", canceled, head.Serve("squid"), nil}, {"a ref the repo doesn't have", context.Background(), ArchiveParams{Rev: "refs/heads/limpet", Format: ArchiveZip}.Serve("squid"), nil}, }
for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { var body bytes.Buffer err := WriteArchive(tc.ctx, &body, repoPath, tc.archive) if tc.want == nil { assert.Error(t, err) return } require.NoError(t, err)
entries, err := zip.NewReader(bytes.NewReader(body.Bytes()), int64(body.Len())) require.NoError(t, err) assert.Equal(t, tc.want, lo.Map(entries.File, func(f *zip.File, _ int) string { return f.Name })) }) }
err := WriteArchive(context.Background(), refusingWriter{}, repoPath, head.Serve("squid")) assert.ErrorIs(t, err, errClientGone, "WriteArchive returns the write error itself")}
var errClientGone = errors.New("the client stopped reading")
type refusingWriter struct{}
func (refusingWriter) Write(p []byte) (int, error) { return 0, errClientGone }