From cde4705021a07e3cb11322fb9ef78a6c786b41c0 Mon Sep 17 00:00:00 2001 From: oppiliappan Date: Mon, 19 Jan 2026 07:59:19 +0000 Subject: [PATCH] appview,knotserver: support lockable http tarball protocol MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit the lockable http tarball protocol is meant to serve tarball flakes, by emitting a stable `Link` header: Link: ; rel="immutable" this patch now supports the new header in two places, on the appview, at the `/archive/.tar.gz` endpoint: λ nix flake metadata -v --refresh --no-write-lock-file 'http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz' unpacking 'http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz' into the Git cache... warning: not writing modified lock file of flake 'http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz': • Added input 'nixpkgs': 'github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D' (2026-01-19) Resolved URL: http://127.0.0.1:3000/oppi.li/repo-19-01-26-08-04-14/archive/main.tar.gz Locked URL: http://127.0.0.1:3000/did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14/archive/a63d945ae97b84812e394207f3cc80f6525c2082.tar.gz?narHash=sha256-IdKT88RIWvWrgQFx6c%2BX3cC7JFene%2BQI9yo2rKSGoA4%3D Path: /nix/store/0k9pv83f0qn5cm0qy82j51plryk7szx7-source Fingerprint: 9512ee4857b31a76c1112f05161bda5280d8596b866c4f78986c6c01c1d2f419 Inputs: └───nixpkgs: github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D (2026-01-19 00:39:23) and on the knotserver, when using the `/xrpc/sh.tangled.repo.archive` endpoint: λ nix flake metadata -v --refresh --no-write-lock-file "http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did%3Aplc%3Aqfpnj4og54vl56wngdriaxug%2Frepo-19-01-26-08-04-14" unpacking 'http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14' into the Git cache... warning: not writing modified lock file of flake 'http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14': • Added input 'nixpkgs': 'github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D' (2026-01-19) Resolved URL: http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&prefix=&ref=main&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14 Locked URL: http://localhost:5555/xrpc/sh.tangled.repo.archive?format=tar.gz&narHash=sha256-IdKT88RIWvWrgQFx6c%2BX3cC7JFene%2BQI9yo2rKSGoA4%3D&prefix=&ref=a63d945ae97b84812e394207f3cc80f6525c2082&repo=did:plc:qfpnj4og54vl56wngdriaxug/repo-19-01-26-08-04-14 Path: /nix/store/0k9pv83f0qn5cm0qy82j51plryk7szx7-source Fingerprint: 9512ee4857b31a76c1112f05161bda5280d8596b866c4f78986c6c01c1d2f419 Inputs: └───nixpkgs: github:nixos/nixpkgs/bde09022887110deb780067364a0818e89258968?narHash=sha256-tLj4KcRDLakrlpvboTJDKsrp6z2XLwyQ4Zmo%2Bw8KsY4%3D (2026-01-19 00:39:23) note that the "Resolved URL" includes a hash of the commit. Co-authored-by: Seongmin Lee Signed-off-by: oppiliappan --- appview/repo/archive.go | 83 +++++++++++++++++++++++++-------- knotserver/git/git.go | 4 ++ knotserver/xrpc/repo_archive.go | 35 ++++++++++++++ 3 files changed, 103 insertions(+), 19 deletions(-) diff --git a/appview/repo/archive.go b/appview/repo/archive.go index a8644608..6ff16d75 100644 --- a/appview/repo/archive.go +++ b/appview/repo/archive.go @@ -2,16 +2,12 @@ package repo import ( "fmt" + "io" "net/http" "net/url" "strings" - "tangled.org/core/api/tangled" - xrpcclient "tangled.org/core/appview/xrpcclient" - - indigoxrpc "github.com/bluesky-social/indigo/xrpc" "github.com/go-chi/chi/v5" - "github.com/go-git/go-git/v5/plumbing" ) func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { @@ -29,22 +25,71 @@ func (rp *Repo) DownloadArchive(w http.ResponseWriter, r *http.Request) { scheme = "https" } host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } didSlashRepo := f.DidSlashRepo() - archiveBytes, err := tangled.RepoArchive(r.Context(), xrpcc, "tar.gz", "", ref, didSlashRepo) - if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { - l.Error("failed to call XRPC repo.archive", "err", xrpcerr) + + // build the xrpc url + u, err := url.Parse(host) + if err != nil { + l.Error("failed to parse host URL", "err", err) rp.pages.Error503(w) return } - // Set headers for file download, just pass along whatever the knot specifies - safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") - filename := fmt.Sprintf("%s-%s.tar.gz", f.Name, safeRefFilename) - w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) - w.Header().Set("Content-Type", "application/gzip") - w.Header().Set("Content-Length", fmt.Sprintf("%d", len(archiveBytes))) - // Write the archive data directly - w.Write(archiveBytes) + + u.Path = "/xrpc/sh.tangled.repo.archive" + query := url.Values{} + query.Set("format", "tar.gz") + query.Set("prefix", r.URL.Query().Get("prefix")) + query.Set("ref", ref) + query.Set("repo", didSlashRepo) + u.RawQuery = query.Encode() + + xrpcURL := u.String() + + // make the get request + resp, err := http.Get(xrpcURL) + if err != nil { + l.Error("failed to call XRPC repo.archive", "err", err) + rp.pages.Error503(w) + return + } + + // pass through headers from upstream response + if contentDisposition := resp.Header.Get("Content-Disposition"); contentDisposition != "" { + w.Header().Set("Content-Disposition", contentDisposition) + } + if contentType := resp.Header.Get("Content-Type"); contentType != "" { + w.Header().Set("Content-Type", contentType) + } + if contentLength := resp.Header.Get("Content-Length"); contentLength != "" { + w.Header().Set("Content-Length", contentLength) + } + if link := resp.Header.Get("Link"); link != "" { + if resolvedRef, err := extractImmutableLink(link); err == nil { + newLink := fmt.Sprintf("<%s/%s/archive/%s.tar.gz>; rel=\"immutable\"", + rp.config.Core.AppviewHost, f.DidSlashRepo(), resolvedRef) + w.Header().Set("Link", newLink) + } + } + + // stream the archive data directly + if _, err := io.Copy(w, resp.Body); err != nil { + l.Error("failed to write response", "err", err) + } +} + +func extractImmutableLink(linkHeader string) (string, error) { + trimmed := strings.TrimPrefix(linkHeader, "<") + trimmed = strings.TrimSuffix(trimmed, ">; rel=\"immutable\"") + + parsedLink, err := url.Parse(trimmed) + if err != nil { + return "", err + } + + resolvedRef := parsedLink.Query().Get("ref") + if resolvedRef == "" { + return "", fmt.Errorf("no ref found in link header") + } + + return resolvedRef, nil } diff --git a/knotserver/git/git.go b/knotserver/git/git.go index 6fbb3329..5d888852 100644 --- a/knotserver/git/git.go +++ b/knotserver/git/git.go @@ -76,6 +76,10 @@ func PlainOpen(path string) (*GitRepo, error) { return &g, nil } +func (g *GitRepo) Hash() plumbing.Hash { + return g.h +} + // re-open a repository and update references func (g *GitRepo) Refresh() error { refreshed, err := PlainOpen(g.path) diff --git a/knotserver/xrpc/repo_archive.go b/knotserver/xrpc/repo_archive.go index 42fe498e..4148ba62 100644 --- a/knotserver/xrpc/repo_archive.go +++ b/knotserver/xrpc/repo_archive.go @@ -4,10 +4,12 @@ import ( "compress/gzip" "fmt" "net/http" + "net/url" "strings" "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/api/tangled" "tangled.org/core/knotserver/git" xrpcerr "tangled.org/core/xrpc/errors" ) @@ -47,6 +49,18 @@ func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) { repoParts := strings.Split(repo, "/") repoName := repoParts[len(repoParts)-1] + immutableLink, err := x.buildImmutableLink(repo, format, gr.Hash().String(), prefix) + if err != nil { + x.Logger.Error( + "failed to build immutable link", + "err", err.Error(), + "repo", repo, + "format", format, + "ref", gr.Hash().String(), + "prefix", prefix, + ) + } + safeRefFilename := strings.ReplaceAll(plumbing.ReferenceName(ref).Short(), "/", "-") var archivePrefix string @@ -59,6 +73,7 @@ func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) { filename := fmt.Sprintf("%s-%s.tar.gz", repoName, safeRefFilename) w.Header().Set("Content-Disposition", fmt.Sprintf("attachment; filename=\"%s\"", filename)) w.Header().Set("Content-Type", "application/gzip") + w.Header().Set("Link", fmt.Sprintf("<%s>; rel=\"immutable\"", immutableLink)) gw := gzip.NewWriter(w) defer gw.Close() @@ -79,3 +94,23 @@ func (x *Xrpc) RepoArchive(w http.ResponseWriter, r *http.Request) { return } } + +func (x *Xrpc) buildImmutableLink(repo string, format string, ref string, prefix string) (string, error) { + scheme := "https" + if x.Config.Server.Dev { + scheme = "http" + } + + u, err := url.Parse(scheme + "://" + x.Config.Server.Hostname + "/xrpc/" + tangled.RepoArchiveNSID) + if err != nil { + return "", err + } + + params := url.Values{} + params.Set("repo", repo) + params.Set("format", format) + params.Set("ref", ref) + params.Set("prefix", prefix) + + return fmt.Sprintf("%s?%s", u.String(), params.Encode()), nil +} -- 2.51.2