Something went wrong. Try again.
Monorepo for Tangled
Something went wrong. Try again.
Go
at sl/md-editor
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217package markup
import ( "context" "fmt" "maps" "slices" "strings"
"github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/yuin/goldmark" "github.com/yuin/goldmark/ast" "github.com/yuin/goldmark/extension" "github.com/yuin/goldmark/parser" "github.com/yuin/goldmark/text" "github.com/yuin/goldmark/util" "tangled.org/core/appview/db" "tangled.org/core/appview/models" textension "tangled.org/core/appview/pages/markup/extension" "tangled.org/core/sets")
// normalize(body string) -> (string, []syntax.DID, []syntax.ATURI, []syntax.CID)// @boltless.me -> [did:...]// https://tangled.org/tangled.org/core/issues/1 -> [at://...]// (additionally return list of blobs referenced in markdown)//// parse_markdown(string) -> AST -> transform_markdown(body AST) -> (AST)// 1. parse markdown (string->AST). reflinks are parsed at this phase// 2. transform markdown (AST->AST). we convert blob links here. (pds/porxie + camo)// blob links are handled from HTML sanitizer. (to handle both img tag and markdown)//// NOTE: can we do blob link transformation on render? don't want to touch AST. Markdown should not have an AST.// [did:...] -> "@boltless.me"// [at://...] -> "#123 my issue"//  -> <img>//// we check if number of referenced blobs match the number of uploaded blobs
// custom goldmark parse context to accept any at-uri DID referencestype parseContext struct { parser.Context // inner}
func (p *parseContext) Reference(label string) (parser.Reference, bool) { if _, err := syntax.ParseDID(label); err == nil { return parser.NewReference([]byte(label), []byte(label), nil), true } if _, err := syntax.ParseATURI(label); err == nil { return parser.NewReference([]byte(label), []byte(label), nil), true } return p.Context.Reference(label)}
var _ parser.Context = new(parseContext)
var smallParser parser.Parser
func init() { smallParser = goldmark.DefaultParser() smallParser.AddOptions( // GFM - TaskList parser.WithInlineParsers(util.Prioritized(extension.NewTaskCheckBoxParser(), 0)), // GFM - Linkify parser.WithInlineParsers(util.Prioritized(extension.NewLinkifyParser(), 999)), // GFM - Strikethrough parser.WithInlineParsers(util.Prioritized(extension.NewStrikethroughParser(), 500)), // GFM - Table parser.WithParagraphTransformers(util.Prioritized(extension.NewTableParagraphTransformer(), 200)), parser.WithASTTransformers(util.Prioritized(extension.NewTableASTTransformer(), 0)), // AtExt parser.WithInlineParsers(util.Prioritized(textension.NewAtParser(), 500)), )}
// @boltless.me -> [did:...]// https://tangled.org/tangled.org/core/issues/1 -> [at://...]//  -> no-op (additionally return list of blobs referenced in markdown)func Normalize(body string) (string, []syntax.DID, []syntax.ATURI, []syntax.CID, error) { // TODO: fill these var dir identity.Directory var ctx context.Context var ddb db.Execer
sourceBytes := []byte(body)
// 1. parse string -> AST (with handles and appviewLinks) // 2. walk AST (collect handles, appviewLinks, blob cids) // 3. resolve handles & appviewLinks // 4. re-walk AST, normalize it.
root := smallParser.Parse( text.NewReader(sourceBytes), parser.WithContext(&parseContext{parser.NewContext()}), )
var ( rawMentions = sets.New[syntax.Handle]() rawAppviewLinks = sets.New[models.AppviewLink]() blobCIDs = sets.New[syntax.CID]() ) if err := ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } switch n.Kind() { case textension.KindAt: rawHandle := n.(*textension.AtNode).Handle if handle, err := syntax.ParseHandle(rawHandle); err == nil { rawMentions.Insert(handle) } return ast.WalkSkipChildren, nil case ast.KindLink: panic("unimplemented") case ast.KindAutoLink: panic("unimplemented") case ast.KindImage: n := n.(*ast.Image) if strings.HasPrefix(string(n.Destination), "blob://") { if cid, err := syntax.ParseCID(string(n.Destination[7:])); err != nil { blobCIDs.Insert(cid) } } return ast.WalkSkipChildren, nil default: return ast.WalkContinue, nil } }); err != nil { return body, nil, nil, nil, fmt.Errorf("walking ast: %w", err) }
// resolve handles & appviewlinks to DID & AT-URI var ( normalized = "" mentions = make(map[syntax.Handle]syntax.DID) appviewLinks = sets.New[models.AppviewLink]() )
// resolve all at-identifiers for handle := range rawMentions.All() { ident, err := dir.LookupHandle(ctx, handle) if err != nil || ident == nil || ident.Handle.IsInvalidHandle() { continue } mentions[handle] = ident.DID } for link := range rawAppviewLinks.All() { ident, err := dir.Lookup(ctx, link.Owner) if err != nil || ident == nil || ident.Handle.IsInvalidHandle() { continue } link.OwnerDid = ident.DID appviewLinks.Insert(link) } references, err := db.ValidateAppviewLinks(ddb, slices.Collect(appviewLinks.All())) if err != nil { // no-op }
// normalize markdown body // collect mentions, references and blobs while walking // this time, we collect [did:...] and [at://...] as valid mentions and references if err := ast.Walk(root, func(n ast.Node, entering bool) (ast.WalkStatus, error) { if !entering { return ast.WalkContinue, nil } // TODO: AtExt -> [did:...] // TODO: Link -> [at://...] // TODO: AutoLink -> [at://...] // TODO: RefLink -> collect <- shit, AST doesn't hold reflink??? // welp, doesn't matter. we collect ANY link which destination is AT-URI // for mentions... we only treat links without title. switch n.Kind() { case textension.KindAt: panic("unimplemented") case ast.KindLink: // if destination is in at-uri map, switch destination to at-uri
// but shit, we don't have a way to distinguish link and reflink // as we cannot distinguish them, we cannot normalize them.
// Link -> "[foo](bar)"? "[foo][bar]"? We don't know
// - [](https://tangled.org/...) // - [whatever](https://tangled.org/...) // if destination is in appviewLink panic("unimplemented") case ast.KindAutoLink: panic("unimplemented") default: return ast.WalkContinue, nil } }); err != nil { return body, nil, nil, nil, fmt.Errorf("walking ast: %w", err) }
// NOTE: // it's bad idea to return mentions & references from normalizer // because body might be already normalized
// But we also walk quite a lot here... // so we can just collect mentions & references while re-walking
return normalized, slices.Collect(maps.Values(mentions)), slices.Collect(maps.Values(references)), slices.Collect(blobCIDs.All()), nil}