diff --git a/api/tangled/repoblob.go b/api/tangled/repoblob.go --- a/api/tangled/repoblob.go +++ b/api/tangled/repoblob.go @@ -30,7 +30,8 @@ type RepoBlob_Output struct { // content: File content (base64 encoded for binary files) Content *string `json:"content,omitempty" cborgen:"content,omitempty"` // encoding: Content encoding - Encoding *string `json:"encoding,omitempty" cborgen:"encoding,omitempty"` + Encoding *string `json:"encoding,omitempty" cborgen:"encoding,omitempty"` + FileTooLarge *bool `json:"fileTooLarge,omitempty" cborgen:"fileTooLarge,omitempty"` // isBinary: Whether the file is binary IsBinary *bool `json:"isBinary,omitempty" cborgen:"isBinary,omitempty"` LastCommit *RepoBlob_LastCommit `json:"lastCommit,omitempty" cborgen:"lastCommit,omitempty"` diff --git a/appview/config/config.go b/appview/config/config.go --- a/appview/config/config.go +++ b/appview/config/config.go @@ -73,6 +73,10 @@ Host string `env:"HOST, default=https://camo.tangled.sh"` SharedSecret string `env:"SHARED_SECRET"` } +func (c *CamoConfig) Enabled() bool { + return c.SharedSecret != "" +} + type AvatarConfig struct { Host string `env:"HOST, default=https://avatar.tangled.sh"` SharedSecret string `env:"SHARED_SECRET"` diff --git a/appview/models/repo.go b/appview/models/repo.go --- a/appview/models/repo.go +++ b/appview/models/repo.go @@ -144,6 +144,7 @@ type BlobView struct { HasTextView bool // can show as code/text HasRenderedView bool // can show rendered (markup/image/video/submodule) HasRawView bool // can download raw (everything except submodule) + FileTooLarge bool // file too large (ignored for image files) // current display mode ShowingRendered bool // currently in rendered mode diff --git a/appview/pages/markup/markdown.go b/appview/pages/markup/markdown.go --- a/appview/pages/markup/markdown.go +++ b/appview/pages/markup/markdown.go @@ -25,7 +25,6 @@ callout "gitlab.com/staticnoise/goldmark-callout" "go.abhg.dev/goldmark/mermaid" htmlparse "golang.org/x/net/html" - "tangled.org/core/api/tangled" textension "tangled.org/core/appview/pages/markup/extension" "tangled.org/core/appview/pages/repoinfo" ) @@ -177,6 +176,8 @@ func visitNode(ctx *RenderContext, node *htmlparse.Node) { switch node.Type { case htmlparse.ElementNode: switch node.Data { + case "a": + // TODO: transform `./` or `/` links to tree link case "img", "source": for i, attr := range node.Attr { if attr.Key != "src" { @@ -185,8 +186,8 @@ } camoUrl, _ := url.Parse(ctx.CamoUrl) dstUrl, _ := url.Parse(attr.Val) - if dstUrl.Host != camoUrl.Host { - attr.Val = ctx.imageFromKnotTransformer(attr.Val) + if camoUrl != nil && dstUrl != nil && dstUrl.Host != camoUrl.Host { + attr.Val = ctx.imageToRawTransformer(attr.Val) attr.Val = ctx.camoImageLinkTransformer(attr.Val) node.Attr[i] = attr } @@ -224,18 +225,13 @@ switch n := n.(type) { case *ast.Heading: a.rctx.anchorHeadingTransformer(n) case *ast.Link: + // TODO: run this on HTML transformation instead a.rctx.relativeLinkTransformer(n) - case *ast.Image: - a.rctx.imageFromKnotAstTransformer(n) - a.rctx.camoImageLinkAstTransformer(n) } case RendererTypeDefault: switch n := n.(type) { case *ast.Heading: a.rctx.anchorHeadingTransformer(n) - case *ast.Image: - a.rctx.imageFromKnotAstTransformer(n) - a.rctx.camoImageLinkAstTransformer(n) } } @@ -257,36 +253,15 @@ newPath := path.Join("/", rctx.RepoInfo.FullName(), "tree", rctx.RepoInfo.Ref, actualPath) link.Destination = []byte(newPath) } -func (rctx *RenderContext) imageFromKnotTransformer(dst string) string { +func (rctx *RenderContext) imageToRawTransformer(dst string) string { if isAbsoluteUrl(dst) { return dst - } - - scheme := "https" - if rctx.IsDev { - scheme = "http" } actualPath := rctx.actualPath(dst) - repoName := fmt.Sprintf("%s/%s", rctx.RepoInfo.OwnerDid, rctx.RepoInfo.Name) - - query := fmt.Sprintf("repo=%s&ref=%s&path=%s&raw=true", - url.QueryEscape(repoName), url.QueryEscape(rctx.RepoInfo.Ref), actualPath) - - parsedURL := &url.URL{ - Scheme: scheme, - Host: rctx.Knot, - Path: path.Join("/xrpc", tangled.RepoBlobNSID), - RawQuery: query, - } - newPath := parsedURL.String() - return newPath -} - -func (rctx *RenderContext) imageFromKnotAstTransformer(img *ast.Image) { - dst := string(img.Destination) - img.Destination = []byte(rctx.imageFromKnotTransformer(dst)) + newDest := path.Join("/", rctx.RepoInfo.FullName(), "raw", rctx.RepoInfo.Ref, actualPath) + return newDest } func (rctx *RenderContext) anchorHeadingTransformer(h *ast.Heading) { diff --git a/appview/pages/templates/repo/blob.html b/appview/pages/templates/repo/blob.html --- a/appview/pages/templates/repo/blob.html +++ b/appview/pages/templates/repo/blob.html @@ -107,6 +107,10 @@ {{ else }}
{{ code .BlobView.Contents .Path | escapeHtml }}
{{ end }} + {{ else if .BlobView.FileTooLarge }} +

+ This file is too large to render. View raw.. +

{{ else if .BlobView.ContentType.IsMarkup }}
{{ if .BlobView.ShowingRendered }} diff --git a/appview/repo/blob.go b/appview/repo/blob.go --- a/appview/repo/blob.go +++ b/appview/repo/blob.go @@ -51,15 +51,8 @@ filePath := chi.URLParam(r, "*") filePath, _ = url.PathUnescape(filePath) - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - host := fmt.Sprintf("%s://%s", scheme, f.Knot) - xrpcc := &indigoxrpc.Client{ - Host: host, - } - resp, err := tangled.RepoBlob(r.Context(), xrpcc, filePath, false, ref, f.RepoIdentifier()) + xrpcc := &indigoxrpc.Client{Host: rp.config.KnotMirror.Url} + resp, err := tangled.RepoBlob(r.Context(), xrpcc, filePath, false, ref, f.RepoAt().String()) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.blob", "xrpcerr", xrpcerr, "err", err) rp.pages.Error503(w) @@ -135,23 +128,8 @@ filePath := chi.URLParam(r, "*") filePath, _ = url.PathUnescape(filePath) - scheme := "http" - if !rp.config.Core.Dev { - scheme = "https" - } - repo := f.RepoIdentifier() - baseURL := &url.URL{ - Scheme: scheme, - Host: f.Knot, - Path: "/xrpc/sh.tangled.repo.blob", - } - query := baseURL.Query() - query.Set("repo", repo) - query.Set("ref", ref) - query.Set("path", filePath) - query.Set("raw", "true") - baseURL.RawQuery = query.Encode() - blobURL := baseURL.String() + blobURL := generateBlobURL(rp.config, f, ref, filePath) + req, err := http.NewRequest("GET", blobURL, nil) if err != nil { l.Error("failed to create request", "err", err) @@ -187,12 +165,6 @@ return } contentType := resp.Header.Get("Content-Type") - body, err := io.ReadAll(resp.Body) - if err != nil { - l.Error("error reading response body from knotserver", "err", err) - w.WriteHeader(http.StatusInternalServerError) - return - } // Normalize to bare media type before classification; strips parameters // (e.g. "; charset=utf-8") and prevents bypass attempts like @@ -208,14 +180,18 @@ case strings.HasPrefix(mediaType, "text/") || isTextualMimeType(mediaType): // Serve all textual content as plain text so the browser never // interprets knot-supplied markup or scripts. w.Header().Set("Content-Type", "text/plain; charset=utf-8") - w.Write(body) - case safeBinaryMIMEType(mediaType): + case safeBinaryMIMEType(mediaType) || contentType == "application/octet-stream": // Use the normalized type, never the raw knot-supplied string. w.Header().Set("Content-Type", mediaType) - w.Write(body) default: w.WriteHeader(http.StatusUnsupportedMediaType) w.Write([]byte("unsupported content type")) + return + } + if _, err := io.Copy(w, resp.Body); err != nil { + l.Error("error streaming knotmirror response", "err", err) + w.WriteHeader(http.StatusInternalServerError) + return } } @@ -241,7 +217,7 @@ return view } // Determine if binary - if resp.IsBinary != nil && *resp.IsBinary { + if (resp.IsBinary != nil && *resp.IsBinary) || (resp.FileTooLarge != nil && *resp.FileTooLarge) { view.ContentSrc = generateBlobURL(config, repo, ref, filePath) ext := strings.ToLower(filepath.Ext(resp.Path)) @@ -295,26 +271,15 @@ return view } func generateBlobURL(config *config.Config, repo *models.Repo, ref, filePath string) string { - scheme := "http" - if !config.Core.Dev { - scheme = "https" - } - - repoName := repo.RepoIdentifier() - baseURL := &url.URL{ - Scheme: scheme, - Host: repo.Knot, - Path: "/xrpc/sh.tangled.repo.blob", - } - query := baseURL.Query() - query.Set("repo", repoName) + query := url.Values{} + query.Set("repo", string(repo.RepoAt())) query.Set("ref", ref) query.Set("path", filePath) query.Set("raw", "true") - baseURL.RawQuery = query.Encode() - blobURL := baseURL.String() - if !config.Core.Dev { + blobURL := fmt.Sprintf("%s/xrpc/%s?%s", config.KnotMirror.Url, tangled.GitTempGetBlobNSID, query.Encode()) + + if config.Camo.Enabled() { return markup.GenerateCamoURL(config.Camo.Host, config.Camo.SharedSecret, blobURL) } return blobURL diff --git a/knotmirror/xrpc/git_get_blob.go b/knotmirror/xrpc/git_get_blob.go --- a/knotmirror/xrpc/git_get_blob.go +++ b/knotmirror/xrpc/git_get_blob.go @@ -2,10 +2,13 @@ package xrpc import ( "context" + "crypto/sha256" "fmt" "io" "net/http" + "path/filepath" "slices" + "strings" "github.com/bluesky-social/indigo/atproto/atclient" "github.com/bluesky-social/indigo/atproto/syntax" @@ -51,10 +54,56 @@ return } defer reader.Close() - w.Header().Set("Content-Type", "application/octet-stream") - if _, err := io.Copy(w, reader); err != nil { - l.Error("failed to serve the blob", "err", err) + // default to octet-stream for large blobs + if file.Size > 1000*1000 { // 1MB + w.Header().Set("Content-Type", "application/octet-stream") + if _, err := io.Copy(w, reader); err != nil { + l.Error("failed to serve the blob", "err", err) + } + return + } + + contents, err := io.ReadAll(reader) + if err != nil { + l.Error("failed to read blob content", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to read the blob"}) + return + } + + mimeType := http.DetectContentType(contents) + // override MIME types for formats that http.DetectContentType does not recognize + switch filepath.Ext(path) { + case ".svg": + mimeType = "image/svg+xml" + case ".avif": + mimeType = "image/avif" + case ".jxl": + mimeType = "image/jxl" + case ".heic", ".heif": + mimeType = "image/heif" + } + + switch { + case strings.HasPrefix(mimeType, "image/"), strings.HasPrefix(mimeType, "video/"): + eTag := fmt.Sprintf("\"%x\"", sha256.Sum256(contents)) + if clientETag := r.Header.Get("If-None-Match"); clientETag == eTag { + w.WriteHeader(http.StatusNotModified) + return + } + w.Header().Set("ETag", eTag) + w.Header().Set("Content-Type", mimeType) + + case strings.HasPrefix(mimeType, "text/") || isTextualMimeType(mimeType): + w.Header().Set("Cache-Control", "public, no-cache") + // seve all text content as text/plain + w.Header().Set("Content-Type", "text/plain; charset=utf-8") + + default: + l.Error("attempted to serve disallowed file type", "mimetype", mimeType) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InvalidRequest", Message: "only image, video, and text files can be accessed directly"}) + return } + w.Write(contents) } func (x *Xrpc) getFile(ctx context.Context, repo syntax.ATURI, ref, path string) (*object.File, error) { diff --git a/knotmirror/xrpc/repo_blob.go b/knotmirror/xrpc/repo_blob.go new file mode 100644 --- /dev/null +++ b/knotmirror/xrpc/repo_blob.go @@ -0,0 +1,170 @@ +package xrpc + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "net/http" + "path/filepath" + "strings" + "time" + + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + "tangled.org/core/knotserver/git" +) + +// TODO(boltless): rewrite lexicon in new NSID +func (x *Xrpc) RepoBlob(w http.ResponseWriter, r *http.Request) { + var ( + repoQuery = r.URL.Query().Get("repo") + ref = r.URL.Query().Get("ref") // ref can be empty (git.Open handles this) + path = r.URL.Query().Get("path") + ) + + repo, err := syntax.ParseATURI(repoQuery) + if err != nil || repo.RecordKey() == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: fmt.Sprintf("repo parameter invalid: %s", repoQuery)}) + return + } + + l := x.logger.With("repo", repo, "ref", ref, "path", path) + + if path == "" { + writeJson(w, http.StatusBadRequest, atclient.ErrorBody{Name: "BadRequest", Message: "missing path parameter"}) + return + } + + gr, err := x.getRepo(r.Context(), repo, ref) + if err != nil { + l.Warn("local mirror failed, trying proxy", "err", err) + if x.proxyToKnot(w, r, repo) { + return + } + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get blob"}) + return + } + + // first check if this path is a submodule + submodule, err := gr.Submodule(path) + if err != nil { + // this is okay, continue and try to treat it as a regular file + } else { + writeJson(w, http.StatusOK, tangled.RepoBlob_Output{ + Ref: ref, + Path: path, + Submodule: &tangled.RepoBlob_Submodule{ + Name: submodule.Name, + Url: submodule.URL, + Branch: &submodule.Branch, + }, + }) + return + } + + file, err := x.getFile(r.Context(), repo, ref, path) + if err != nil { + l.Warn("local mirror failed, trying proxy", "err", err) + if x.proxyToKnot(w, r, repo) { + return + } + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to get blob"}) + return + } + + if file.Size > 1000*1000 { // 1MB + fileTooLarge := true + writeJson(w, http.StatusOK, tangled.RepoBlob_Output{ + Ref: ref, + Path: path, + Size: &file.Size, + FileTooLarge: &fileTooLarge, + }) + return + } + + reader, err := file.Reader() + if err != nil { + l.Error("failed to read blob", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to read the blob"}) + return + } + contents, err := io.ReadAll(reader) + if err != nil { + l.Error("failed to read blob content", "err", err) + writeJson(w, http.StatusInternalServerError, atclient.ErrorBody{Name: "InternalServerError", Message: "failed to read the blob"}) + return + } + + mimeType := http.DetectContentType(contents) + // override MIME types for formats that http.DetectContentType does not recognize + switch filepath.Ext(path) { + case ".svg": + mimeType = "image/svg+xml" + case ".avif": + mimeType = "image/avif" + case ".jxl": + mimeType = "image/jxl" + case ".heic", ".heif": + mimeType = "image/heif" + } + + isBinary := !(strings.HasPrefix(mimeType, "text/") || isTextualMimeType(mimeType)) + + // include content for text blob or svg + var content *string + if !isBinary { + content = new(string) + *content = string(contents) + } else if filepath.Ext(path) == ".svg" { + content = new(string) + *content = base64.StdEncoding.EncodeToString(contents) + } + + response := tangled.RepoBlob_Output{ + Ref: ref, + Path: path, + Size: &file.Size, + IsBinary: &isBinary, + Content: content, + } + + ctx, cancel := context.WithTimeout(r.Context(), 2*time.Second) + defer cancel() + + lastCommit, err := gr.LastCommitFile(ctx, path) + if err == nil && lastCommit != nil { + response.LastCommit = &tangled.RepoBlob_LastCommit{ + Hash: lastCommit.Hash.String(), + Message: lastCommit.Message, + When: lastCommit.When.Format(time.RFC3339), + } + + // try to get author information + commit, err := gr.Commit(lastCommit.Hash) + if err == nil { + response.LastCommit.Author = &tangled.RepoBlob_Signature{ + Name: commit.Author.Name, + Email: commit.Author.Email, + } + } + } + + writeJson(w, http.StatusOK, response) +} + +func (x *Xrpc) getRepo(ctx context.Context, repo syntax.ATURI, ref string) (*git.GitRepo, error) { + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + return nil, fmt.Errorf("resolving repo at-uri: %w", err) + } + + gr, err := git.Open(repoPath, ref) + if err != nil { + return nil, fmt.Errorf("opening git repo: %w", err) + } + + return gr, nil +} diff --git a/knotmirror/xrpc/xrpc.go b/knotmirror/xrpc/xrpc.go --- a/knotmirror/xrpc/xrpc.go +++ b/knotmirror/xrpc/xrpc.go @@ -55,6 +55,7 @@ r.Get("/"+tangled.GitTempListBranchesNSID, x.ListBranches) // wip, unknown output r.Get("/"+tangled.GitTempListCommitsNSID, x.ListCommits) r.Get("/"+tangled.GitTempListLanguagesNSID, x.ListLanguages) r.Get("/"+tangled.GitTempListTagsNSID, x.ListTags) + r.Get("/"+tangled.RepoBlobNSID, x.RepoBlob) r.Post("/"+tangled.SyncRequestCrawlNSID, x.RequestCrawl) return r diff --git a/lexicons/repo/blob.json b/lexicons/repo/blob.json --- a/lexicons/repo/blob.json +++ b/lexicons/repo/blob.json @@ -80,6 +80,9 @@ }, "lastCommit": { "type": "ref", "ref": "#lastCommit" + }, + "fileTooLarge": { + "type": "boolean" } } }