diff --git a/appview/db/reference.go b/appview/db/reference.go index 4d3c20fb..82deaacf 100644 --- a/appview/db/reference.go +++ b/appview/db/reference.go @@ -11,6 +11,11 @@ import ( "tangled.org/core/orm" ) +func ValidateAppviewLinks(e Execer, appviewLinks []models.AppviewLink) (map[string]syntax.ATURI, error) { + // TODO: return url->ATURI map + panic("unimplemented") +} + // ValidateReferenceLinks resolves refLinks to Issue/PR/Comment ATURIs. // It will ignore missing refLinks. func ValidateReferenceLinks(e Execer, refLinks []models.ReferenceLink) ([]syntax.ATURI, error) { diff --git a/appview/models/reference.go b/appview/models/reference.go index 27513803..d6cf536d 100644 --- a/appview/models/reference.go +++ b/appview/models/reference.go @@ -27,6 +27,16 @@ type ReferenceLink struct { CommentRkey *syntax.RecordKey } +type AppviewLink struct { + Url string + Owner syntax.AtIdentifier + OwnerDid syntax.DID // optional resolved DID + RepoSlug string + Kind RefKind + SubjectId int + CommentRkey *syntax.RecordKey +} + type RichReferenceLink struct { ReferenceLink Title string diff --git a/appview/pages/markup/markdown_normalizer.go b/appview/pages/markup/markdown_normalizer.go new file mode 100644 index 00000000..cb924824 --- /dev/null +++ b/appview/pages/markup/markdown_normalizer.go @@ -0,0 +1,216 @@ +package markup + +import ( + "context" + "fmt" + "maps" + "slices" + "strings" + + "github.com/bluesky-social/indigo/atproto/identity" + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/yuin/goldmark" + "github.com/yuin/goldmark/ast" + "github.com/yuin/goldmark/extension" + "github.com/yuin/goldmark/parser" + "github.com/yuin/goldmark/text" + "github.com/yuin/goldmark/util" + "tangled.org/core/appview/db" + "tangled.org/core/appview/models" + textension "tangled.org/core/appview/pages/markup/extension" + "tangled.org/core/sets" +) + +// normalize(body string) -> (string, []syntax.DID, []syntax.ATURI, []syntax.CID) +// @boltless.me -> [did:...] +// https://tangled.org/tangled.org/core/issues/1 -> [at://...] +// (additionally return list of blobs referenced in markdown) +// +// parse_markdown(string) -> AST -> transform_markdown(body AST) -> (AST) +// 1. parse markdown (string->AST). reflinks are parsed at this phase +// 2. transform markdown (AST->AST). we convert blob links here. (pds/porxie + camo) +// blob links are handled from HTML sanitizer. (to handle both img tag and markdown) +// +// NOTE: can we do blob link transformation on render? don't want to touch AST. Markdown should not have an AST. +// [did:...] -> "@boltless.me" +// [at://...] -> "#123 my issue" +// ![image](blob://...) -> +// +// we check if number of referenced blobs match the number of uploaded blobs + +// custom goldmark parse context to accept any at-uri DID references +type parseContext struct { + parser.Context // inner +} + +func (p *parseContext) Reference(label string) (parser.Reference, bool) { + if _, err := syntax.ParseDID(label); err == nil { + return parser.NewReference([]byte(label), []byte(label), nil), true + } + if _, err := syntax.ParseATURI(label); err == nil { + return parser.NewReference([]byte(label), []byte(label), nil), true + } + return p.Context.Reference(label) +} + +var _ parser.Context = new(parseContext) + +var smallParser parser.Parser + +func init() { + smallParser = goldmark.DefaultParser() + smallParser.AddOptions( + // GFM - TaskList + parser.WithInlineParsers(util.Prioritized(extension.NewTaskCheckBoxParser(), 0)), + // GFM - Linkify + parser.WithInlineParsers(util.Prioritized(extension.NewLinkifyParser(), 999)), + // GFM - Strikethrough + parser.WithInlineParsers(util.Prioritized(extension.NewStrikethroughParser(), 500)), + // GFM - Table + parser.WithParagraphTransformers(util.Prioritized(extension.NewTableParagraphTransformer(), 200)), + parser.WithASTTransformers(util.Prioritized(extension.NewTableASTTransformer(), 0)), + // AtExt + parser.WithInlineParsers(util.Prioritized(textension.NewAtParser(), 500)), + ) +} + + +// @boltless.me -> [did:...] +// https://tangled.org/tangled.org/core/issues/1 -> [at://...] +// ![](blob://...) -> no-op (additionally return list of blobs referenced in markdown) +func Normalize(body string) (string, []syntax.DID, []syntax.ATURI, []syntax.CID, error) { + // TODO: fill these + var dir identity.Directory + var ctx context.Context + var ddb db.Execer + + sourceBytes := []byte(body) + + // 1. parse string -> AST (with handles and appviewLinks) + // 2. walk AST (collect handles, appviewLinks, blob cids) + // 3. resolve handles & appviewLinks + // 4. re-walk AST, normalize it. + + root := smallParser.Parse( + text.NewReader(sourceBytes), + parser.WithContext(&parseContext{parser.NewContext()}), + ) + + var ( + rawMentions = sets.New[syntax.Handle]() + rawAppviewLinks = sets.New[models.AppviewLink]() + blobCIDs = sets.New[syntax.CID]() + ) + if err := ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + switch n.Kind() { + case textension.KindAt: + rawHandle := n.(*textension.AtNode).Handle + if handle, err := syntax.ParseHandle(rawHandle); err == nil { + rawMentions.Insert(handle) + } + return ast.WalkSkipChildren, nil + case ast.KindLink: + panic("unimplemented") + case ast.KindAutoLink: + panic("unimplemented") + case ast.KindImage: + n := n.(*ast.Image) + if strings.HasPrefix(string(n.Destination), "blob://") { + if cid, err := syntax.ParseCID(string(n.Destination[7:])); err != nil { + blobCIDs.Insert(cid) + } + } + return ast.WalkSkipChildren, nil + default: + return ast.WalkContinue, nil + } + }); err != nil { + return body, nil, nil, nil, fmt.Errorf("walking ast: %w", err) + } + + // resolve handles & appviewlinks to DID & AT-URI + var ( + normalized = "" + mentions = make(map[syntax.Handle]syntax.DID) + appviewLinks = sets.New[models.AppviewLink]() + ) + + // resolve all at-identifiers + for handle := range rawMentions.All() { + ident, err := dir.LookupHandle(ctx, handle) + if err != nil || ident == nil || ident.Handle.IsInvalidHandle() { + continue + } + mentions[handle] = ident.DID + } + for link := range rawAppviewLinks.All() { + ident, err := dir.Lookup(ctx, link.Owner) + if err != nil || ident == nil || ident.Handle.IsInvalidHandle() { + continue + } + link.OwnerDid = ident.DID + appviewLinks.Insert(link) + } + references, err := db.ValidateAppviewLinks(ddb, slices.Collect(appviewLinks.All())) + if err != nil { + // no-op + } + + // normalize markdown body + // collect mentions, references and blobs while walking + // this time, we collect [did:...] and [at://...] as valid mentions and references + if err := ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) { + if !entering { + return ast.WalkContinue, nil + } + // TODO: AtExt -> [did:...] + // TODO: Link -> [at://...] + // TODO: AutoLink -> [at://...] + // TODO: RefLink -> collect <- shit, AST doesn't hold reflink??? + // welp, doesn't matter. we collect ANY link which destination is AT-URI + // for mentions... we only treat links without title. + switch n.Kind() { + case textension.KindAt: + panic("unimplemented") + case ast.KindLink: + // if destination is in at-uri map, switch destination to at-uri + + // but shit, we don't have a way to distinguish link and reflink + // as we cannot distinguish them, we cannot normalize them. + + // Link -> "[foo](bar)"? "[foo][bar]"? We don't know + + + // - [](https://tangled.org/...) + // - [whatever](https://tangled.org/...) + // if destination is in appviewLink + panic("unimplemented") + case ast.KindAutoLink: + panic("unimplemented") + default: + return ast.WalkContinue, nil + } + }); err != nil { + return body, nil, nil, nil, fmt.Errorf("walking ast: %w", err) + } + + + + + // NOTE: + // it's bad idea to return mentions & references from normalizer + // because body might be already normalized + + // But we also walk quite a lot here... + // so we can just collect mentions & references while re-walking + + + + + + + return normalized, slices.Collect(maps.Values(mentions)), slices.Collect(maps.Values(references)), slices.Collect(blobCIDs.All()), nil +} diff --git a/appview/pages/markup/markdown_renderer.go b/appview/pages/markup/markdown_renderer.go new file mode 100644 index 00000000..e829338c --- /dev/null +++ b/appview/pages/markup/markdown_renderer.go @@ -0,0 +1,2 @@ +package markup +