From 549f5fddbfb7dcce55e442d552009bdac6a8f89d Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Sat, 23 May 2026 17:16:48 +0900 Subject: [PATCH] wip: appview,blobstore: v2 markdown renderer New renderer that is more pure and receive ownerDid to generate blob URLs including the user DID. Issue: PDS doesn't serve pre-uploaded blobs, so preview fails. We need extra blobCache just for pre-uploaded blobs or just do markdown rendering on client side. NOTE: considering we can't implement markdown normalizer with goldmark, maybe it's better to split this as separate, embedable service Signed-off-by: Seongmin Lee --- appview/pages/compose_parse_test.go | 8 +- appview/pages/funcmap.go | 10 + appview/pages/funcmap_test.go | 2 +- appview/pages/markup/markdown2.go | 215 ++++++++++++++++++ appview/pages/markup/sanitizer.go | 3 + appview/pages/pages.go | 16 +- .../fragments/comment/commentBody.html | 4 +- .../templates/fragments/markdownPreview.html | 2 +- appview/state/markup.go | 12 +- appview/state/state.go | 16 +- blobstore/blobstore.go | 1 + blobstore/pds.go | 17 +- blobstore/porxie.go | 8 +- cmd/blog/main.go | 2 +- 14 files changed, 288 insertions(+), 28 deletions(-) create mode 100644 appview/pages/markup/markdown2.go diff --git a/appview/pages/compose_parse_test.go b/appview/pages/compose_parse_test.go index ad99eda0..78b7960f 100644 --- a/appview/pages/compose_parse_test.go +++ b/appview/pages/compose_parse_test.go @@ -16,7 +16,7 @@ import ( func TestPullComposeTemplatesParse(t *testing.T) { cfg := &config.Config{} - p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) cases := []struct { name string @@ -43,7 +43,7 @@ func TestPullComposeTemplatesParse(t *testing.T) { func TestPullComposeHostRender(t *testing.T) { cfg := &config.Config{} - p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) base := RepoNewPullParams{ RepoInfo: repoinfo.RepoInfo{ @@ -78,7 +78,7 @@ func TestPullComposeHostRender(t *testing.T) { func TestPullComposeHostRenderWithData(t *testing.T) { cfg := &config.Config{} - p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) sampleBranches := []types.Branch{ {Reference: types.Reference{Name: "feature"}}, @@ -207,7 +207,7 @@ index 0000000..1111111 100644 func TestPullComposeLabelStateRoundTrip(t *testing.T) { cfg := &config.Config{} - p := NewPages(cfg, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) + p := NewPages(cfg, nil, nil, nil, nil, slog.New(slog.NewTextHandler(io.Discard, nil))) sampleBranches := []types.Branch{ {Reference: types.Reference{Name: "feature"}}, diff --git a/appview/pages/funcmap.go b/appview/pages/funcmap.go index 8232deb4..cb8e533a 100644 --- a/appview/pages/funcmap.go +++ b/appview/pages/funcmap.go @@ -334,6 +334,16 @@ func (p *Pages) funcMap() template.FuncMap { sanitized := rctx.SanitizeDefault(htmlString) return template.HTML(sanitized) }, + // render user-owned markdown content which can include blob attachments + "markdownBody": func(owner syntax.DID, preview bool, text string) template.HTML { + rctx := &markup.RenderMarkdownBodyParams{ + OwnerDid: owner, + IsPreview: preview, + } + htmlString := p.markdown.RenderMarkdownBody(rctx, text) + sanitized := p.rctx.SanitizeDefault(htmlString) + return template.HTML(sanitized) + }, "code": func(content, path string) string { var style *chroma.Style = styles.Get("catpuccin-latte") formatter := chromahtml.New( diff --git a/appview/pages/funcmap_test.go b/appview/pages/funcmap_test.go index f35f9808..1e8e6bff 100644 --- a/appview/pages/funcmap_test.go +++ b/appview/pages/funcmap_test.go @@ -22,7 +22,7 @@ func TestPages_funcMap(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - p := NewPages(tt.config, tt.res, nil, nil, tt.l) + p := NewPages(tt.config, tt.res, nil, nil, nil, tt.l) got := p.funcMap() // TODO: update the condition below to compare got with tt.want. if true { diff --git a/appview/pages/markup/markdown2.go b/appview/pages/markup/markdown2.go new file mode 100644 index 00000000..2e795e33 --- /dev/null +++ b/appview/pages/markup/markdown2.go @@ -0,0 +1,215 @@ +package markup + +import ( + "bytes" + "context" + "fmt" + "io" + "net/url" + "strings" + + chromahtml "github.com/alecthomas/chroma/v2/formatters/html" + "github.com/alecthomas/chroma/v2/styles" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/ipfs/go-cid" + "github.com/yuin/goldmark" + emoji "github.com/yuin/goldmark-emoji" + highlighting "github.com/yuin/goldmark-highlighting/v2" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/renderer/html" + callout "gitlab.com/staticnoise/goldmark-callout" + "go.abhg.dev/goldmark/mermaid" + htmlparse "golang.org/x/net/html" + "tangled.org/core/appview/config" + textension "tangled.org/core/appview/pages/markup/extension" + "tangled.org/core/blobstore" +) + +// make more pure markdown renderer. + +type RenderGitBlobContext struct { + RepoDid syntax.DID + CurrentRev string + CurrentDir string +} + +// MarkdownRenderer can render user-owned markdown bodies like issue title/body. +type MarkdownRenderer struct { + CamoUrl string + CamoSecret string + Hostname string + blobStore blobstore.BlobStore // to generate blob url + markdown goldmark.Markdown +} + +func NewMarkdownRenderer(cfg *config.Config, blobStore blobstore.BlobStore) *MarkdownRenderer { + markdown := goldmark.New( + goldmark.WithExtensions( + extension.GFM, + &mermaid.Extender{ + RenderMode: mermaid.RenderModeClient, + NoScript: true, + }, + highlighting.NewHighlighting( + highlighting.WithFormatOptions( + chromahtml.Standalone(false), + chromahtml.WithClasses(true), + ), + highlighting.WithCustomStyle(styles.Get("catppuccin-latte")), + ), + extension.NewFootnote( + extension.WithFootnoteIDPrefix([]byte("footnote")), + ), + callout.CalloutExtention, + textension.AtExt, + textension.NewTangledLinkExt(cfg.Core.AppviewHost), + emoji.Emoji, + ), + goldmark.WithParserOptions( + parser.WithAutoHeadingID(), + ), + goldmark.WithRendererOptions(html.WithUnsafe()), + ) + return &MarkdownRenderer{ + CamoUrl: cfg.Camo.Host, + CamoSecret: cfg.Camo.SharedSecret, + Hostname: cfg.Core.AppviewHost, + blobStore: blobStore, + markdown: markdown, + } +} + +type RenderMarkdownBodyParams struct { + OwnerDid syntax.DID // Content owner DID. Used to make blob links + IsPreview bool +} + +func (m *MarkdownRenderer) RenderMarkdownBody(params *RenderMarkdownBodyParams, source string) string { + var buf bytes.Buffer + if err := m.markdown.Convert([]byte(source), &buf); err != nil { + return source + } + + var processed strings.Builder + if err := postProcessHTML( + strings.NewReader(buf.String()), + &processed, + func(node *htmlparse.Node) { m.visitNode(params, node) }, + ); err != nil { + return source + } + + return processed.String() +} + +func (m *MarkdownRenderer) visitNode(params *RenderMarkdownBodyParams, node *htmlparse.Node) { + switch node.Type { + case htmlparse.ElementNode: + switch node.Data { + case "img", "source": + isBlob := false + for i, attr := range node.Attr { + if attr.Key != "src" { + continue + } + + if rawCid, found := strings.CutPrefix(attr.Val, "blob://"); found { + isBlob = true + cid, err := cid.Parse(rawCid) + if err != nil { + continue // skip invalid cid + } + blobUrl, err := m.blobStore.MakeBlobUrl(context.TODO(), params.OwnerDid, cid) + if err != nil { + continue + } + attr.Val = blobUrl + node.Attr[i] = attr + continue + } + + src, err := url.Parse(attr.Val) + if err != nil { + continue // skip invalid url + } + + if src.IsAbs() { + camoUrl, _ := url.Parse(m.CamoUrl) + if camoUrl != nil && src.Host != m.Hostname && src.Host != camoUrl.Host { + attr.Val = GenerateCamoURL(m.CamoUrl, m.CamoSecret, attr.Val) + node.Attr[i] = attr + } + } + } + if params.IsPreview && isBlob { + setAttr(node, "onerror", fmt.Sprintf("this.onerror=null;this.src=%q", "/static/blobpreviewplaceholder.png")) + } + } + + for n := node.FirstChild; n != nil; n = n.NextSibling { + m.visitNode(params, n) + } + default: + } +} + +func setAttr(node *htmlparse.Node, key, val string) { + for i, attr := range node.Attr { + if attr.Key != key { + continue + } + attr.Val = val + node.Attr[i] = attr + return + } + node.Attr = append(node.Attr, htmlparse.Attribute{ + Key: key, + Val: val, + }) +} + +func postProcessHTML(input io.Reader, output io.Writer, processFn func(*htmlparse.Node)) error { + node, err := htmlparse.Parse(io.MultiReader( + strings.NewReader(""), + input, + strings.NewReader(""), + )) + if err != nil { + return fmt.Errorf("failed to parse html: %w", err) + } + + if node.Type == htmlparse.DocumentNode { + node = node.FirstChild + } + + processFn(node) + + newNodes := make([]*htmlparse.Node, 0, 5) + + if node.Data == "html" { + node = node.FirstChild + for node != nil && node.Data != "body" { + node = node.NextSibling + } + } + if node != nil { + if node.Data == "body" { + child := node.FirstChild + for child != nil { + newNodes = append(newNodes, child) + child = child.NextSibling + } + } else { + newNodes = append(newNodes, node) + } + } + + for _, node := range newNodes { + if err := htmlparse.Render(output, node); err != nil { + return fmt.Errorf("failed to render processed html: %w", err) + } + } + + return nil +} diff --git a/appview/pages/markup/sanitizer.go b/appview/pages/markup/sanitizer.go index 61b0117b..b40e66ea 100644 --- a/appview/pages/markup/sanitizer.go +++ b/appview/pages/markup/sanitizer.go @@ -83,6 +83,9 @@ func buildDefaultPolicy() *bluemonday.Policy { policy.AllowAttrs(generalSafeAttrs...).OnElements(generalSafeElements...) + // image + policy.AllowAttrs("onerror").OnElements("img") + // video policy.AllowAttrs("src", "autoplay", "controls").OnElements("video") diff --git a/appview/pages/pages.go b/appview/pages/pages.go index e817c995..3a9abb2d 100644 --- a/appview/pages/pages.go +++ b/appview/pages/pages.go @@ -27,6 +27,7 @@ import ( "tangled.org/core/appview/pages/markup" "tangled.org/core/appview/pages/repoinfo" "tangled.org/core/appview/pagination" + "tangled.org/core/blobstore" "tangled.org/core/idresolver" "tangled.org/core/types" @@ -79,9 +80,11 @@ type Pages struct { templateDir string // Path to templates on disk for dev mode rctx *markup.RenderContext logger *slog.Logger + + markdown *markup.MarkdownRenderer } -func NewPages(config *config.Config, res *idresolver.Resolver, database *db.DB, rdb *cache.Cache, logger *slog.Logger) *Pages { +func NewPages(config *config.Config, res *idresolver.Resolver, database *db.DB, rdb *cache.Cache, blobStore blobstore.BlobStore, logger *slog.Logger) *Pages { // initialized with safe defaults, can be overridden per use rctx := &markup.RenderContext{ IsDev: config.Core.Dev, @@ -89,7 +92,6 @@ func NewPages(config *config.Config, res *idresolver.Resolver, database *db.DB, CamoUrl: config.Camo.Host, CamoSecret: config.Camo.SharedSecret, Sanitizer: markup.NewSanitizer(), - Files: Files, } p := &Pages{ @@ -104,6 +106,7 @@ func NewPages(config *config.Config, res *idresolver.Resolver, database *db.DB, rdb: rdb, templateDir: "appview/pages", logger: logger, + markdown: markup.NewMarkdownRenderer(config, blobStore), } if p.dev { @@ -1379,8 +1382,13 @@ func (p *Pages) PullComposeHostFragment(w io.Writer, params RepoNewPullParams) e return p.executePlain("repo/pulls/fragments/pullComposeHost", w, params) } -func (p *Pages) MarkdownPreviewFragment(w io.Writer, body string) error { - return p.executePlain("fragments/markdownPreview", w, body) +type MarkdownPreviewFragmentParams struct { + LoggedInUser *oauth.MultiAccountUser + Content string +} + +func (p *Pages) MarkdownPreviewFragment(w io.Writer, params MarkdownPreviewFragmentParams) error { + return p.executePlain("fragments/markdownPreview", w, params) } type EditPullParams struct { diff --git a/appview/pages/templates/fragments/comment/commentBody.html b/appview/pages/templates/fragments/comment/commentBody.html index 3bd7b8c1..a86c81f5 100644 --- a/appview/pages/templates/fragments/comment/commentBody.html +++ b/appview/pages/templates/fragments/comment/commentBody.html @@ -1,7 +1,7 @@ {{ define "fragments/comment/commentBody" }} -
+
{{ if not .Comment.Deleted }} -
{{ .Comment.Body.Text | markdown }}
+
{{ markdownBody .Comment.Did false .Comment.Body.Text }}
{{ template "repo/fragments/reactions" (dict "Reactions" .Reactions "UserReacted" .UserReacted diff --git a/appview/pages/templates/fragments/markdownPreview.html b/appview/pages/templates/fragments/markdownPreview.html index b65f13b3..dc094001 100644 --- a/appview/pages/templates/fragments/markdownPreview.html +++ b/appview/pages/templates/fragments/markdownPreview.html @@ -1,7 +1,7 @@ {{ define "fragments/markdownPreview" }} {{ if . }}
- {{ . | markdown }} + {{ markdownBody (did .LoggedInUser.Did) true .Content }}
{{ else }}
Nothing to preview.
diff --git a/appview/state/markup.go b/appview/state/markup.go index 21e068c9..42db6405 100644 --- a/appview/state/markup.go +++ b/appview/state/markup.go @@ -1,8 +1,14 @@ package state -import "net/http" +import ( + "net/http" + + "tangled.org/core/appview/pages" +) func (s *State) MarkdownPreview(w http.ResponseWriter, r *http.Request) { - body := r.FormValue("body") - s.pages.MarkdownPreviewFragment(w, body) + s.pages.MarkdownPreviewFragment(w, pages.MarkdownPreviewFragmentParams{ + LoggedInUser: s.oauth.GetMultiAccountUser(r), + Content: r.FormValue("body"), + }) } diff --git a/appview/state/state.go b/appview/state/state.go index cf9d22af..1a4f748c 100644 --- a/appview/state/state.go +++ b/appview/state/state.go @@ -115,7 +115,14 @@ func Make(ctx context.Context, config *config.Config) (*State, error) { return nil, fmt.Errorf("failed to create posthog client: %w", err) } - pages := pages.NewPages(config, res, d, rdb, log.SubLogger(logger, "pages")) + var blobStore blobstore.BlobStore + if config.Porxie.Url != "" { + blobStore = blobstore.NewPorxieBlobStore(config.Porxie.Url) + } else { + blobStore = blobstore.NewPdsBlobStore(res.Directory()) + } + + pages := pages.NewPages(config, res, d, rdb, blobStore, log.SubLogger(logger, "pages")) knotcompat.UseNativeLatch(knotacl.NewLatch(d, log.SubLogger(logger, "knotacl-latch"))) aclService := knotacl.NewService(enforcer, d, config.Core.Dev, log.SubLogger(logger, "knotacl")) oauth, err := oauth.New(config, posthog, d, enforcer, aclService, res, log.SubLogger(logger, "oauth")) @@ -129,13 +136,6 @@ func Make(ctx context.Context, config *config.Config) (*State, error) { mentionsResolver := mentions.New(config, res, d, log.SubLogger(logger, "mentionsResolver")) - var blobStore blobstore.BlobStore - if config.Porxie.Url != "" { - blobStore = blobstore.NewPorxieBlobStore(config.Porxie.Url) - } else { - blobStore = blobstore.NewPdsBlobStore(res.Directory()) - } - jc, err := jetstream.NewJetstreamClient( config.Jetstream.Endpoint, "appview", diff --git a/blobstore/blobstore.go b/blobstore/blobstore.go index fe1691c2..f9bc15f3 100644 --- a/blobstore/blobstore.go +++ b/blobstore/blobstore.go @@ -9,5 +9,6 @@ import ( ) type BlobStore interface { + MakeBlobUrl(ctx context.Context, did syntax.DID, cid cid.Cid) (string, error) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.ReadCloser, error) } diff --git a/blobstore/pds.go b/blobstore/pds.go index 3d413f14..f6fe51ec 100644 --- a/blobstore/pds.go +++ b/blobstore/pds.go @@ -20,10 +20,12 @@ func NewPdsBlobStore(dir identity.Directory) *Pds { return &Pds{dir} } -func (s *Pds) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.ReadCloser, error) { +var _ BlobStore = (*Pds)(nil) + +func (s *Pds) MakeBlobUrl(ctx context.Context, did syntax.DID, cid cid.Cid) (string, error) { id, err := s.dir.LookupDID(ctx, did) if err != nil { - return nil, err + return "", err } url, _ := url.Parse(fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob", id.PDSEndpoint())) @@ -32,7 +34,16 @@ func (s *Pds) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.Read q.Set("cid", cid.String()) url.RawQuery = q.Encode() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), nil) + return url.String(), nil +} + +func (s *Pds) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.ReadCloser, error) { + url, err := s.MakeBlobUrl(ctx, did, cid) + if err != nil { + return nil, err + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) resp, err := http.DefaultClient.Do(req) if err != nil { return nil, err diff --git a/blobstore/porxie.go b/blobstore/porxie.go index 9f867480..7a0ed011 100644 --- a/blobstore/porxie.go +++ b/blobstore/porxie.go @@ -18,8 +18,14 @@ func NewPorxieBlobStore(url string) *Porxie { return &Porxie{url} } +var _ BlobStore = (*Porxie)(nil) + +func (s *Porxie) MakeBlobUrl(ctx context.Context, did syntax.DID, cid cid.Cid) (string, error) { + return fmt.Sprintf("%s/%s/%s", s.url, did, cid), nil +} + func (s *Porxie) GetBlob(ctx context.Context, did syntax.DID, cid cid.Cid) (io.ReadCloser, error) { - url := fmt.Sprintf("%s/%s/%s", s.url, did, cid) + url, _ := s.MakeBlobUrl(ctx, did, cid) resp, err := http.Get(url) if err != nil { return nil, err diff --git a/cmd/blog/main.go b/cmd/blog/main.go index 3541c098..46309914 100644 --- a/cmd/blog/main.go +++ b/cmd/blog/main.go @@ -54,7 +54,7 @@ func main() { func makePages(ctx context.Context, cfg *config.Config, logger *slog.Logger) (*pages.Pages, error) { resolver := idresolver.DefaultResolver(cfg.Plc.PLCURL) - return pages.NewPages(cfg, resolver, nil, nil, logger), nil + return pages.NewPages(cfg, resolver, nil, nil, nil, logger), nil } func runBuild(ctx context.Context, logger *slog.Logger) error { -- 2.54.0