From 67efa863c298e708eb544cc95320c69228db7a9b Mon Sep 17 00:00:00 2001 From: Seongmin Lee Date: Fri, 14 Nov 2025 13:24:53 +0900 Subject: [PATCH] wip: appview/pages: markdown normalizer markdown normalizer will _normalize_ mentions & references into persistent decentralized identifiers. ``` @alice.com -> [did:plc:alice] https://tangled.org/tangled.org/core/pulls/123 -> [at://...] ``` normalized content will be stored under `.text` in `sh.tangled.markup.markdown` type while orininal content will be kept under `.original`. This will allow understanding references while ingesting from third-party appviews Signed-off-by: Seongmin Lee --- appview/pages/markup/extension/atlink.go | 11 +- appview/pages/markup/extension/reflink.go | 75 ++++++ appview/pages/markup/markdown.go | 1 - appview/pages/markup/normalizer/normalizer.go | 224 ++++++++++++++++++ .../markup/normalizer/normalizer_test.go | 33 +++ appview/pages/markup/parser/parser.go | 45 ++++ appview/pages/markup/parser/reflink.go | 84 +++++++ appview/pages/pages.go | 1 - 8 files changed, 467 insertions(+), 7 deletions(-) create mode 100644 appview/pages/markup/extension/reflink.go create mode 100644 appview/pages/markup/normalizer/normalizer.go create mode 100644 appview/pages/markup/normalizer/normalizer_test.go create mode 100644 appview/pages/markup/parser/parser.go create mode 100644 appview/pages/markup/parser/reflink.go diff --git a/appview/pages/markup/extension/atlink.go b/appview/pages/markup/extension/atlink.go index 005f4955..57d98bb0 100644 --- a/appview/pages/markup/extension/atlink.go +++ b/appview/pages/markup/extension/atlink.go @@ -16,8 +16,9 @@ import ( // An AtNode struct represents an AtNode type AtNode struct { - Handle string ast.BaseInline + Segment text.Segment + Handle string } var _ ast.Node = &AtNode{} @@ -67,10 +68,10 @@ func (s *atParser) Parse(parent ast.Node, block text.Reader, pc parser.Context) atSegment := text.NewSegment(segment.Start, segment.Start+m[1]) block.Advance(m[1]) - node := &AtNode{} - node.AppendChild(node, ast.NewTextSegment(atSegment)) - node.Handle = string(atSegment.Value(block.Source())[1:]) - return node + return &AtNode{ + Segment: atSegment, + Handle: string(atSegment.Value(block.Source())[1:]), + } } // atHtmlRenderer is a renderer.NodeRenderer implementation that diff --git a/appview/pages/markup/extension/reflink.go b/appview/pages/markup/extension/reflink.go new file mode 100644 index 00000000..020f8a8a --- /dev/null +++ b/appview/pages/markup/extension/reflink.go @@ -0,0 +1,75 @@ +package extension + +import ( + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "tangled.org/core/appview/models" +) + +// KindCustomLink is a NodeKind of the [CustomLinkNode]. +var KindCustomLink = ast.NewNodeKind("CustomLink") + +type CustomLinkNode struct { + ast.BaseInline + + // union type + Commit *TangledCommitLink + Issue *TangledIssueLink + Pull *TangledPullLink +} + +type TangledIssueLink struct { + Id int + Title string + IsOpen bool +} + +type TangledPullLink struct { + Id int + Title string + State models.PullState +} + +var _ ast.Node = (*CustomLinkNode)(nil) + +// Dump implements [ast.Node]. +func (n *CustomLinkNode) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// Kind implements [ast.Node]. +func (n *CustomLinkNode) Kind() ast.NodeKind { + return KindTangledLink +} + +type refLinkTransformer struct{} + +var _ parser.ASTTransformer = (*refLinkTransformer)(nil) + +// Transform implements [parser.ASTTransformer]. +func (t *refLinkTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + switch n.Kind() { + case ast.KindLink: + node := n.(*ast.Link) + dest := node.Destination + if did, err := syntax.ParseDID(string(dest)); err != nil { + // pc.Get( + } + } + + // 1. find links that has DID + // 2. check if pc has the resolved object + // 3. replace to renderable AtUriLink nodes (PR id, title, state etc) + + return ast.WalkContinue, nil + }) +} + +// TODO: renderer diff --git a/appview/pages/markup/markdown.go b/appview/pages/markup/markdown.go index 890aed7d..a70ea539 100644 --- a/appview/pages/markup/markdown.go +++ b/appview/pages/markup/markdown.go @@ -49,7 +49,6 @@ type RenderContext struct { Hostname string RendererType RendererType Sanitizer Sanitizer - Files fs.FS } func NewMarkdown(hostname string, extra ...goldmark.Extender) goldmark.Markdown { diff --git a/appview/pages/markup/normalizer/normalizer.go b/appview/pages/markup/normalizer/normalizer.go new file mode 100644 index 00000000..b4337434 --- /dev/null +++ b/appview/pages/markup/normalizer/normalizer.go @@ -0,0 +1,224 @@ +package normalizer + +import ( + "context" + "net/url" + "path" + "slices" + "strconv" + "strings" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "tangled.org/core/appview/db" + "tangled.org/core/appview/models" + textension "tangled.org/core/appview/pages/markup/extension" + "tangled.org/core/sets" +) + +type MarkdownNormalizer struct { + hostname string + dir identity.Directory + db db.Execer +} + +func NewMarkdownNormalizer(hostname string, dir identity.Directory) *MarkdownNormalizer { + return &MarkdownNormalizer{ + hostname: hostname, + dir: dir, + } +} + +var markdownParser parser.Parser + +func init() { + markdownParser = goldmark.New( + goldmark.WithExtensions( + extension.GFM, + extension.NewFootnote( + extension.WithFootnoteIDPrefix([]byte("footnote")), + ), + textension.AtExt, + ), + ).Parser() +} + +// 1. parse markdown with AtExt +// 2. walk AST +// - normalize AtExt segments -> [did:...] (when resolvable) +// @alice.com -> [did:plc:alice] +// - normalize Tangled-Link segments +// -> `[at://...]` + +// from render step: +// 1. parse normalized version with full-parser & custom parser.Context +// custom parser.Context will resolve custom ref-links and collect them into mentions & references +// 2. with resolved links, start render phase. we should know all at-uris at this phase. + +// Normalize normalizes markdown body +// +// @alice.com -> [did:plc:alice] +// #123 -> [at://...] +// [link](https://tangled.org/...) -> [link](at://...) +// +// NOTE: last two cases are not implemented yet. +func (m *MarkdownNormalizer) Normalize(ctx context.Context, source []byte) ([]byte, []syntax.DID, []syntax.ATURI) { + var ( + normalizedSource []byte + mentions = sets.New[syntax.DID]() + rawRefLinks = sets.New[models.ReferenceLink]() + ) + + root := markdownParser.Parse(text.NewReader(source)) + cursor := 0 + ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + switch n.Kind() { + case textension.KindAt: + node := n.(*textension.AtNode) + + start := node.Segment.Start + stop := node.Segment.Stop + + if cursor > start { + break + } + + ident, err := m.dir.LookupHandle(ctx, syntax.Handle(node.Handle)) + if err != nil { + return ast.WalkContinue, nil + } + + mentions.Insert(ident.DID) + + normalizedSource = append(normalizedSource, source[cursor:start]...) + normalizedSource = append(normalizedSource, byte('[')) + normalizedSource = append(normalizedSource, []byte(ident.DID.String())...) + normalizedSource = append(normalizedSource, byte(']')) + + cursor = stop + return ast.WalkSkipChildren, nil + + // case ast.HashRef: + // node := n.(*textension.HashRefNode) + // // TODO: normalize to at-uri + // // example: [at://...] + // return ast.WalkSkipChildren, nil + + case ast.KindLink: + node := n.(*ast.Link) + dest := string(node.Destination) + if ref := parseTangledLink(m.hostname, dest); ref != nil { + rawRefLinks.Insert(*ref) + } + + // TODO: convert dest to at-uri + // example: [link](at://...) + + return ast.WalkSkipChildren, nil + + case ast.KindAutoLink: + node := n.(*ast.AutoLink) + if node.AutoLinkType == ast.AutoLinkURL { + dest := string(node.URL(source)) + if ref := parseTangledLink(m.hostname, dest); ref != nil { + rawRefLinks.Insert(*ref) + } + } + + // TODO: convert dest to at-uri + // example: + + return ast.WalkSkipChildren, nil + } + + return ast.WalkContinue, nil + }) + if cursor < len(source) { + normalizedSource = append(normalizedSource, source[cursor:]...) + } + + var resolvedRefs []models.ReferenceLink + for rawRef := range rawRefLinks.All() { + ident, err := m.dir.Lookup(ctx, syntax.AtIdentifier(rawRef.Handle)) + if err != nil || ident == nil { + continue + } + rawRef.Handle = string(ident.DID) + resolvedRefs = append(resolvedRefs, rawRef) + } + + // TODO: ideally tangled-uri should be universally resolvable and should not require DB query. + // We are querying DB here because issue/pull-ids are not decentralized yet. + references, err := db.ValidateReferenceLinks(m.db, resolvedRefs) + if err != nil { + } + + return normalizedSource, slices.Collect(mentions.All()), references +} + +func parseTangledLink(baseHost string, urlStr string) *models.ReferenceLink { + u, err := url.Parse(urlStr) + if err != nil { + return nil + } + if u.Host != "" && !strings.EqualFold(u.Host, baseHost) { + return nil + } + + p := path.Clean(u.Path) + parts := strings.FieldsFunc(p, func(r rune) bool { return r == '/' }) + if len(parts) < 4 { + // need at least: atIdentifier / repoName / kind / id + return nil + } + + var ( + atIdentifier = parts[0] + repoName = parts[1] + kindSeg = parts[2] + subjectSeg = parts[3] + ) + + atIdentifier = strings.TrimPrefix(atIdentifier, "@") + + var kind models.RefKind + switch kindSeg { + case "issues": + kind = models.RefKindIssue + case "pulls": + kind = models.RefKindPull + default: + return nil + } + + subjectId, err := strconv.Atoi(subjectSeg) + if err != nil { + return nil + } + var commentId *int + if u.Fragment != "" { + if strings.HasPrefix(u.Fragment, "comment-") { + commentIdStr := u.Fragment[len("comment-"):] + if id, err := strconv.Atoi(commentIdStr); err == nil { + commentId = &id + } + } + } + + return &models.ReferenceLink{ + Handle: atIdentifier, + Repo: repoName, + Kind: kind, + SubjectId: subjectId, + CommentId: commentId, + } +} diff --git a/appview/pages/markup/normalizer/normalizer_test.go b/appview/pages/markup/normalizer/normalizer_test.go new file mode 100644 index 00000000..ba7ab4da --- /dev/null +++ b/appview/pages/markup/normalizer/normalizer_test.go @@ -0,0 +1,33 @@ +package normalizer + +import ( + "context" + "testing" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/stretchr/testify/assert" +) + +func TestNormalize(t *testing.T) { + mockDir := identity.NewMockDirectory() + mockDir.Insert(identity.Identity{DID: "did:plc:alice", Handle: "alice.tngl.sh"}) + + norm := NewMarkdownNormalizer("hostname", mockDir) + + ctx := context.Background() + + t.Run("normalize at-mention", func(t *testing.T) { + source := "hello @alice.tngl.sh!" + result, mentions, _ := norm.Normalize(ctx, []byte(source)) + assert.Equal(t, "hello [did:plc:alice]!", string(result)) + assert.Equal(t, []syntax.DID{"did:plc:alice"}, mentions) + }) + + t.Run("don't normalize invalid at-mention", func(t *testing.T) { + source := "hello @alice!" + result, mentions, _ := norm.Normalize(ctx, []byte(source)) + assert.Equal(t, "hello @alice!", string(result)) + assert.Equal(t, 0, len(mentions)) + }) +} diff --git a/appview/pages/markup/parser/parser.go b/appview/pages/markup/parser/parser.go new file mode 100644 index 00000000..24f84703 --- /dev/null +++ b/appview/pages/markup/parser/parser.go @@ -0,0 +1,45 @@ +package parser + +import ( + "github.com/bluesky-social/indigo/atproto/syntax" + gparser "github.com/yuin/goldmark/parser" +) + +// parseContext is custom gomark [gparser.Context] that resolves DID or AT-URI references +type parseContext struct { + gparser.Context +} + +// Reference implements [gparser.Context] +func (p *parseContext) Reference(label string) (gparser.Reference, bool) { + if ref, ok := p.Context.Reference(label); ok { + return ref, ok + } + if _, err := syntax.ParseDID(label); err != nil { + ref := gparser.NewReference([]byte(label), []byte(label), nil) + p.Context.AddReference(ref) + return ref, true + } + if _, err := syntax.ParseATURI(label); err != nil { + ref := gparser.NewReference([]byte(label), []byte(label), nil) + p.Context.AddReference(ref) + return ref, true + } + return nil, false +} + +func (p *parseContext) Mentions() []syntax.DID { + return p.Get(mentionListKey).([]syntax.DID) +} + +var _ gparser.Context = (*parseContext)(nil) + +var mentionListKey = gparser.NewContextKey() +var referenceListKey = gparser.NewContextKey() + +func NewParseContext(inner gparser.Context) gparser.Context { + pc := &parseContext{Context: inner} + pc.Set(mentionListKey, []syntax.DID{}) + pc.Set(referenceListKey, []syntax.ATURI{}) + return pc +} diff --git a/appview/pages/markup/parser/reflink.go b/appview/pages/markup/parser/reflink.go new file mode 100644 index 00000000..90b13314 --- /dev/null +++ b/appview/pages/markup/parser/reflink.go @@ -0,0 +1,84 @@ +package parser + +import ( + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "tangled.org/core/appview/models" +) + +// KindCustomLink is a NodeKind of the [CustomLinkNode]. +var KindCustomLink = ast.NewNodeKind("CustomLink") + +type CustomLinkNode struct { + ast.BaseInline + + // union type + Mention *TangledMentionLink + Commit *TangledCommitLink + Issue *TangledIssueLink + Pull *TangledPullLink +} + +type TangledMentionLink struct { + Handle syntax.Handle +} + +type TangledCommitLink struct { + // TODO: +} + +type TangledIssueLink struct { + Id int + Title string + IsOpen bool +} + +type TangledPullLink struct { + Id int + Title string + State models.PullState +} + +var _ ast.Node = (*CustomLinkNode)(nil) + +// Dump implements [ast.Node]. +func (n *CustomLinkNode) Dump(source []byte, level int) { + ast.DumpHelper(n, source, level, nil, nil) +} + +// Kind implements [ast.Node]. +func (n *CustomLinkNode) Kind() ast.NodeKind { + return KindCustomLink +} + +type refLinkTransformer struct{} + +var _ parser.ASTTransformer = (*refLinkTransformer)(nil) + +// Transform implements [parser.ASTTransformer]. +func (t *refLinkTransformer) Transform(node *ast.Document, reader text.Reader, pc parser.Context) { + ast.Walk(node, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + + switch n.Kind() { + case ast.KindLink: + node := n.(*ast.Link) + dest := node.Destination + if did, err := syntax.ParseDID(string(dest)); err != nil { + list := pc.Get(mentionListKey).([]syntax.DID) + list = append(list, did) + } + return ast.WalkSkipChildren, nil + } + + // 1. find links that has DID + // 2. check if pc has the resolved object + // 3. replace to renderable AtUriLink nodes (PR id, title, state etc) + + return ast.WalkContinue, nil + }) +} diff --git a/appview/pages/pages.go b/appview/pages/pages.go index cd739a43..2b90dac9 100644 --- a/appview/pages/pages.go +++ b/appview/pages/pages.go @@ -62,7 +62,6 @@ func NewPages(config *config.Config, res *idresolver.Resolver, database *db.DB, CamoUrl: config.Camo.Host, CamoSecret: config.Camo.SharedSecret, Sanitizer: markup.NewSanitizer(), - Files: Files, } p := &Pages{ -- 2.51.2