From c0a454cc0a8a0bbb59251691a111874f0fc744cc Mon Sep 17 00:00:00 2001 From: Will Andrews Date: Tue, 12 May 2026 05:01:10 +0000 Subject: [PATCH] Merge remote-tracking branch 'upstream/master' --- api/tangled/cbor_gen.go | 970 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- api/tangled/feedstar.go | 97 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---- api/tangled/gitrefUpdate.go | 6 ++---- api/tangled/repocollaborator.go | 7 +++---- api/tangled/repodescribeRepo.go | 39 +++++++++++++++++++++++++++++++++++++++ api/tangled/repoissue.go | 3 +-- api/tangled/repopull.go | 10 ++++------ api/tangled/tangledrepo.go | 4 ++-- appview/compat113/compat.go | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/compat113/compat_test.go | 113 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/compat113/version.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/compat113/version_test.go | 35 +++++++++++++++++++++++++++++++++++ appview/config/config.go | 1 + appview/db/artifact.go | 8 ++++---- appview/db/collaborators.go | 22 +++++++++++----------- appview/db/db.go | 465 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------- appview/db/issues.go | 36 ++++++++++++++++++------------------ appview/db/jetstream.go | 8 ++------ appview/db/language.go | 12 ++++++------ appview/db/notifications.go | 9 ++++++--- appview/db/profile.go | 2 +- appview/db/pulls.go | 113 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------------------------------- appview/db/reference.go | 16 ++++++++-------- appview/db/repos.go | 227 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------------- appview/db/repos_rename_test.go | 279 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/db/site_deploys.go | 14 +++++++------- appview/db/sites.go | 54 +++++++++++++++++++++++++++--------------------------- appview/db/star.go | 138 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------------------------------------------------------- appview/db/timeline.go | 9 ++++----- appview/db/vouch.go | 10 +++++----- appview/db/webhooks.go | 17 ++++++++--------- appview/indexer/issues/indexer.go | 10 +++++----- appview/indexer/issues/indexer_test.go | 60 ++++++++++++++++++++++++++++++------------------------------ appview/indexer/notifier.go | 30 +++++++++++++++++++----------- appview/indexer/pulls/indexer.go | 10 +++++----- appview/indexer/repos/indexer.go | 8 ++++---- appview/ingester.go | 246 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------------------------------------------------------------ appview/ingester_repo.go | 395 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/ingester_repo_test.go | 754 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/issues/issues.go | 6 +++--- appview/labels/labels.go | 2 +- appview/metrics/metrics.go | 19 +++++++++++++++++++ appview/metrics/middleware.go | 40 ++++++++++++++++++++++++++++++++++++++++ appview/middleware/middleware.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++----- appview/migration/migrate_add_repo_did.go | 131 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------------- appview/models/artifact.go | 2 +- appview/models/collaborator.go | 2 +- appview/models/issue.go | 15 +++------------ appview/models/language.go | 6 ++---- appview/models/pull.go | 71 +++++++++++++++++++++++------------------------------------------------ appview/models/repo.go | 63 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++--- appview/models/repo_test.go | 86 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/models/search.go | 4 ++-- appview/models/search_test.go | 8 ++++---- appview/models/site_deploy.go | 8 ++++++-- appview/models/sites.go | 10 +++++++--- appview/models/star.go | 16 +++++++++++----- appview/models/webhook.go | 13 +++++++++++-- appview/notify/db/db.go | 29 +++++++++++++++-------------- appview/notify/logging/notifier.go | 5 +++++ appview/notify/merged_notifier.go | 4 ++++ appview/notify/notifier.go | 3 +++ appview/notify/posthog/notifier.go | 46 ++++++++++++++++++++++++++++++++++------------ appview/notify/webhook/notifier.go | 102 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-------------------------------------- appview/pages/pages.go | 14 ++++++++++++++ appview/pages/repoinfo/repoinfo.go | 4 ++-- appview/pages/templates/fragments/starBtn.html | 2 +- appview/pages/templates/goodfirstissues/index.html | 4 ++-- appview/pages/templates/knots/dashboard.html | 2 +- appview/pages/templates/layouts/fragments/topbar.html | 17 ++++++++++++++++- appview/pages/templates/layouts/repobase.html | 29 +++++++++++++++++++---------- appview/pages/templates/notifications/fragments/item.html | 6 +++--- appview/pages/templates/repo/commit.html | 4 ++-- appview/pages/templates/repo/empty.html | 2 +- appview/pages/templates/repo/fork.html | 186 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------- appview/pages/templates/repo/forks.html | 36 ++++++++++++++++++++++++++++++++++++ appview/pages/templates/repo/fragments/cloneDropdown.html | 8 ++++---- appview/pages/templates/repo/index.html | 1 + appview/pages/templates/repo/needsUpgrade.html | 5 +++++ appview/pages/templates/repo/pulls/fragments/pullActions.html | 2 +- appview/pages/templates/repo/pulls/fragments/pullHeader.html | 2 +- appview/pages/templates/repo/pulls/pull.html | 2 +- appview/pages/templates/repo/settings/general.html | 36 ++++++++++++++++++++++++++++++++++++ appview/pages/templates/repo/settings/hooks.html | 10 ++++++++++ appview/pages/templates/repo/settings/sites.html | 6 +++--- appview/pages/templates/spindles/dashboard.html | 2 +- appview/pages/templates/timeline/fragments/preview.html | 8 ++++---- appview/pages/templates/timeline/fragments/timeline.html | 8 ++++---- appview/pages/templates/user/fragments/issueEvent.html | 5 +++-- appview/pages/templates/user/fragments/pullEvent.html | 5 +++-- appview/pages/templates/user/fragments/repoCard.html | 10 ++++++++-- appview/pages/templates/user/overview.html | 2 +- appview/pipelines/pipelines.go | 8 ++++---- appview/pulls/comment.go | 2 +- appview/pulls/compose_helpers_test.go | 2 +- appview/pulls/create.go | 32 +++++++++++--------------------- appview/pulls/lifecycle.go | 4 ++-- appview/pulls/list.go | 12 ++++++------ appview/pulls/merge.go | 2 +- appview/pulls/resubmit.go | 19 +++++++------------ appview/pulls/single.go | 12 ++++++------ appview/repo/artifact.go | 9 +++++---- appview/repo/feed.go | 6 +++--- appview/repo/index.go | 7 ++++--- appview/repo/opengraph.go | 2 +- appview/repo/rename_test.go | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/repo/repo.go | 285 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------- appview/repo/repo_util.go | 2 +- appview/repo/router.go | 2 ++ appview/repo/settings.go | 23 ++++++++++++----------- appview/repo/tags.go | 4 ++-- appview/repo/webhooks.go | 38 ++++++++++++++++++++++---------------- appview/reporesolver/resolver.go | 16 ++++++++-------- appview/repoverify/verify.go | 159 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/repoverify/verify_test.go | 66 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ appview/settings/settings.go | 4 ++-- appview/sites/sites.go | 2 +- appview/state/gfi.go | 11 +++++------ appview/state/knotstream.go | 43 ++++++++++++++----------------------------- appview/state/profile.go | 6 +++--- appview/state/router.go | 5 ++++- appview/state/star.go | 63 +++++++++++++++++++++++++++++++++++++++++++++++---------------- appview/state/state.go | 127 ++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------------------------------------------------------------- appview/strings/strings.go | 5 +++-- cmd/appview/main.go | 9 +++++++++ cmd/cborgen/cborgen.go | 2 ++ cmd/knotmirror/main.go | 25 +++++++++++++++++++++++++ docs/DOCS.md | 16 ++++++++++++++++ hook/setup.go | 9 +++++++++ input.css | 50 ++++++++++++++++++++++++++++++++++++++++++++++++-- jobs/src/index.ts | 4 ++-- jobs/src/postings/archive/software-engineer.md | 0 knotmirror/db/db.go | 5 +++++ knotmirror/db/migrations.go | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/db/migrations_list.go | 56 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/db/repos.go | 147 ++++++++++++++++++++++++++++++++++++++++++++++++++++++--------------------------------------------------------------------------------------------- knotmirror/git.go | 54 +++++++++++++++++------------------------------------- knotmirror/knotstream/slurper.go | 61 ++++++++++++++++++++++++++++++++++--------------------------- knotmirror/migrate/migrate.go | 135 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotmirror/models/models.go | 5 +++-- knotmirror/resyncer.go | 2 +- knotmirror/tapclient.go | 22 +++++++--------------- knotmirror/xrpc/git_list_branches.go | 18 ++++++++++-------- knotmirror/xrpc/proxy.go | 34 ++++++++++++++++++++++++++++------ knotmirror/xrpc/sync_request_crawl.go | 9 ++++++++- knotserver/db/db.go | 118 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------------------- knotserver/db/didassign.go | 7 +++---- knotserver/db/events.go | 9 ++++----- knotserver/db/repo_aliases.go | 60 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/git.go | 20 +++++++++++++------- knotserver/git/language.go | 25 +++++++++++++++++++++++-- knotserver/git/tree.go | 2 +- knotserver/git_test.go | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/ingester.go | 115 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------------------- knotserver/ingester_repo_test.go | 223 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/internal.go | 15 ++++++++------- knotserver/migrate.go | 4 ++-- knotserver/server.go | 1 + knotserver/xrpc/create_repo.go | 25 +++++++++++++++++++------ knotserver/xrpc/delete_branch.go | 7 +++++-- knotserver/xrpc/hidden_ref.go | 7 +++++-- knotserver/xrpc/merge.go | 3 +-- knotserver/xrpc/repo_describe_repo.go | 39 +++++++++++++++++++++++++++++++++++++++ knotserver/xrpc/repo_describe_repo_test.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ knotserver/xrpc/set_default_branch.go | 10 ++++++---- knotserver/xrpc/version.go | 18 +++++------------- knotserver/xrpc/xrpc.go | 1 + lexicons/feed/star.json | 30 ++++++++++++++++++++++++------ lexicons/git/refUpdate.json | 8 ++------ lexicons/issue/issue.json | 6 +----- lexicons/pulls/pull.json | 9 +-------- lexicons/repo/collaborator.json | 7 ++----- lexicons/repo/describeRepo.json | 53 +++++++++++++++++++++++++++++++++++++++++++++++++++++ lexicons/repo/repo.json | 5 ++--- nix/modules/appview.nix | 7 +++++++ nix/pkgs/knot-unwrapped.nix | 2 +- ogre/src/components/shared/logo.tsx | 67 ++++++++++--------------------------------------------------------- orm/orm.go | 12 ++++++++++++ spindle/engines/nixery/setup_steps.go | 6 +++++- spindle/ingester.go | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++---------- spindle/xrpc/add_secret.go | 7 +++++-- spindle/xrpc/list_secrets.go | 7 +++++-- spindle/xrpc/pipeline_cancel_pipeline.go | 7 +++++-- spindle/xrpc/remove_secret.go | 7 +++++-- 184 file(s) changed, 6469 insertion(s)(+), 2026 deletion(s)(-) diff --git a/api/tangled/cbor_gen.go b/api/tangled/cbor_gen.go --- a/api/tangled/cbor_gen.go +++ b/api/tangled/cbor_gen.go @@ -866,17 +866,8 @@ return err } cw := cbg.NewCborWriter(w) - fieldCount := 4 - if t.Subject == nil { - fieldCount-- - } - - if t.SubjectDid == nil { - fieldCount-- - } - - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + if _, err := cw.Write([]byte{163}); err != nil { return err } @@ -899,36 +890,20 @@ if _, err := cw.WriteString(string("sh.tangled.feed.star")); err != nil { return err } - // t.Subject (string) (string) - if t.Subject != nil { + // t.Subject (tangled.FeedStar_Subject) (struct) + if len("subject") > 1000000 { + return xerrors.Errorf("Value in field \"subject\" was too long") + } - if len("subject") > 1000000 { - return xerrors.Errorf("Value in field \"subject\" was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { + return err + } + if _, err := cw.WriteString(string("subject")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subject"))); err != nil { - return err - } - if _, err := cw.WriteString(string("subject")); err != nil { - return err - } - - if t.Subject == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.Subject) > 1000000 { - return xerrors.Errorf("Value in field t.Subject was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Subject))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.Subject)); err != nil { - return err - } - } + if err := t.Subject.MarshalCBOR(cw); err != nil { + return err } // t.CreatedAt (string) (string) @@ -953,38 +928,6 @@ } if _, err := cw.WriteString(string(t.CreatedAt)); err != nil { return err } - - // t.SubjectDid (string) (string) - if t.SubjectDid != nil { - - if len("subjectDid") > 1000000 { - return xerrors.Errorf("Value in field \"subjectDid\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("subjectDid"))); err != nil { - return err - } - if _, err := cw.WriteString(string("subjectDid")); err != nil { - return err - } - - if t.SubjectDid == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.SubjectDid) > 1000000 { - return xerrors.Errorf("Value in field t.SubjectDid was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.SubjectDid))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.SubjectDid)); err != nil { - return err - } - } - } return nil } @@ -1013,7 +956,7 @@ } n := extra - nameBuf := make([]byte, 10) + nameBuf := make([]byte, 9) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -1040,10 +983,11 @@ } t.LexiconTypeID = string(sval) } - // t.Subject (string) (string) + // t.Subject (tangled.FeedStar_Subject) (struct) case "subject": { + b, err := cr.ReadByte() if err != nil { return err @@ -1052,14 +996,12 @@ if b != cbg.CborNull[0] { if err := cr.UnreadByte(); err != nil { return err } - - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err + t.Subject = new(FeedStar_Subject) + if err := t.Subject.UnmarshalCBOR(cr); err != nil { + return xerrors.Errorf("unmarshaling t.Subject pointer: %w", err) } + } - t.Subject = (*string)(&sval) - } } // t.CreatedAt (string) (string) case "createdAt": @@ -1072,26 +1014,135 @@ } t.CreatedAt = string(sval) } - // t.SubjectDid (string) (string) - case "subjectDid": + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *FeedStar_Repo) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } + + // t.Did (string) (string) + if len("did") > 1000000 { + return xerrors.Errorf("Value in field \"did\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("did"))); err != nil { + return err + } + if _, err := cw.WriteString(string("did")); err != nil { + return err + } + + if len(t.Did) > 1000000 { + return xerrors.Errorf("Value in field t.Did was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Did))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Did)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sh.tangled.feed.star#repo"))); err != nil { + return err + } + if _, err := cw.WriteString(string("sh.tangled.feed.star#repo")); err != nil { + return err + } + return nil +} + +func (t *FeedStar_Repo) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedStar_Repo{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedStar_Repo: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Did (string) (string) + case "did": { - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } + t.Did = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": - t.SubjectDid = (*string)(&sval) + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err } + + t.LexiconTypeID = string(sval) } default: @@ -1104,20 +1155,146 @@ } return nil } -func (t *GitRefUpdate) MarshalCBOR(w io.Writer) error { +func (t *FeedStar_String) MarshalCBOR(w io.Writer) error { if t == nil { _, err := w.Write(cbg.CborNull) return err } cw := cbg.NewCborWriter(w) - fieldCount := 9 + + if _, err := cw.Write([]byte{162}); err != nil { + return err + } - if t.OwnerDid == nil { - fieldCount-- + // t.Uri (string) (string) + if len("uri") > 1000000 { + return xerrors.Errorf("Value in field \"uri\" was too long") } - if t.RepoDid == nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("uri"))); err != nil { + return err + } + if _, err := cw.WriteString(string("uri")); err != nil { + return err + } + + if len(t.Uri) > 1000000 { + return xerrors.Errorf("Value in field t.Uri was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Uri))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Uri)); err != nil { + return err + } + + // t.LexiconTypeID (string) (string) + if len("$type") > 1000000 { + return xerrors.Errorf("Value in field \"$type\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("$type"))); err != nil { + return err + } + if _, err := cw.WriteString(string("$type")); err != nil { + return err + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("sh.tangled.feed.star#string"))); err != nil { + return err + } + if _, err := cw.WriteString(string("sh.tangled.feed.star#string")); err != nil { + return err + } + return nil +} + +func (t *FeedStar_String) UnmarshalCBOR(r io.Reader) (err error) { + *t = FeedStar_String{} + + cr := cbg.NewCborReader(r) + + maj, extra, err := cr.ReadHeader() + if err != nil { + return err + } + defer func() { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + }() + + if maj != cbg.MajMap { + return fmt.Errorf("cbor input should be of type map") + } + + if extra > cbg.MaxLength { + return fmt.Errorf("FeedStar_String: map struct too large (%d)", extra) + } + + n := extra + + nameBuf := make([]byte, 5) + for i := uint64(0); i < n; i++ { + nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) + if err != nil { + return err + } + + if !ok { + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(cr, func(cid.Cid) {}); err != nil { + return err + } + continue + } + + switch string(nameBuf[:nameLen]) { + // t.Uri (string) (string) + case "uri": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Uri = string(sval) + } + // t.LexiconTypeID (string) (string) + case "$type": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.LexiconTypeID = string(sval) + } + + default: + // Field doesn't exist on this type, so ignore it + if err := cbg.ScanForLinks(r, func(cid.Cid) {}); err != nil { + return err + } + } + } + + return nil +} +func (t *GitRefUpdate) MarshalCBOR(w io.Writer) error { + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + + cw := cbg.NewCborWriter(w) + fieldCount := 8 + + if t.OwnerDid == nil { fieldCount-- } @@ -1161,6 +1338,29 @@ return err } if err := t.Meta.MarshalCBOR(cw); err != nil { + return err + } + + // t.Repo (string) (string) + if len("repo") > 1000000 { + return xerrors.Errorf("Value in field \"repo\" was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { + return err + } + if _, err := cw.WriteString(string("repo")); err != nil { + return err + } + + if len(t.Repo) > 1000000 { + return xerrors.Errorf("Value in field t.Repo was too long") + } + + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Repo))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Repo)); err != nil { return err } @@ -1229,38 +1429,6 @@ if _, err := cw.WriteString(string(t.OldSha)); err != nil { return err } - // t.RepoDid (string) (string) - if t.RepoDid != nil { - - if len("repoDid") > 1000000 { - return xerrors.Errorf("Value in field \"repoDid\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repoDid"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repoDid")); err != nil { - return err - } - - if t.RepoDid == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.RepoDid) > 1000000 { - return xerrors.Errorf("Value in field t.RepoDid was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.RepoDid))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.RepoDid)); err != nil { - return err - } - } - } - // t.OwnerDid (string) (string) if t.OwnerDid != nil { @@ -1293,29 +1461,6 @@ } } } - // t.RepoName (string) (string) - if len("repoName") > 1000000 { - return xerrors.Errorf("Value in field \"repoName\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repoName"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repoName")); err != nil { - return err - } - - if len(t.RepoName) > 1000000 { - return xerrors.Errorf("Value in field t.RepoName was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.RepoName))); err != nil { - return err - } - if _, err := cw.WriteString(string(t.RepoName)); err != nil { - return err - } - // t.CommitterDid (string) (string) if len("committerDid") > 1000000 { return xerrors.Errorf("Value in field \"committerDid\" was too long") @@ -1413,6 +1558,17 @@ } } } + // t.Repo (string) (string) + case "repo": + + { + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Repo = string(sval) + } // t.LexiconTypeID (string) (string) case "$type": @@ -1446,27 +1602,6 @@ } t.OldSha = string(sval) } - // t.RepoDid (string) (string) - case "repoDid": - - { - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.RepoDid = (*string)(&sval) - } - } // t.OwnerDid (string) (string) case "ownerDid": @@ -1487,17 +1622,6 @@ } t.OwnerDid = (*string)(&sval) } - } - // t.RepoName (string) (string) - case "repoName": - - { - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.RepoName = string(sval) } // t.CommitterDid (string) (string) case "committerDid": @@ -6518,6 +6642,10 @@ if t.Labels == nil { fieldCount-- } + if t.Name == nil { + fieldCount-- + } + if t.RepoDid == nil { fieldCount-- } @@ -6566,26 +6694,35 @@ return err } // t.Name (string) (string) - if len("name") > 1000000 { - return xerrors.Errorf("Value in field \"name\" was too long") - } + if t.Name != nil { - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { - return err - } - if _, err := cw.WriteString(string("name")); err != nil { - return err - } + if len("name") > 1000000 { + return xerrors.Errorf("Value in field \"name\" was too long") + } - if len(t.Name) > 1000000 { - return xerrors.Errorf("Value in field t.Name was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("name"))); err != nil { + return err + } + if _, err := cw.WriteString(string("name")); err != nil { + return err + } + + if t.Name == nil { + if _, err := cw.Write(cbg.CborNull); err != nil { + return err + } + } else { + if len(*t.Name) > 1000000 { + return xerrors.Errorf("Value in field t.Name was too long") + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Name))); err != nil { - return err - } - if _, err := cw.WriteString(string(t.Name)); err != nil { - return err + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Name))); err != nil { + return err + } + if _, err := cw.WriteString(string(*t.Name)); err != nil { + return err + } + } } // t.LexiconTypeID (string) (string) @@ -6920,12 +7057,22 @@ // t.Name (string) (string) case "name": { - sval, err := cbg.ReadStringWithMax(cr, 1000000) + b, err := cr.ReadByte() if err != nil { return err } + if b != cbg.CborNull[0] { + if err := cr.UnreadByte(); err != nil { + return err + } - t.Name = string(sval) + sval, err := cbg.ReadStringWithMax(cr, 1000000) + if err != nil { + return err + } + + t.Name = (*string)(&sval) + } } // t.LexiconTypeID (string) (string) case "$type": @@ -7522,50 +7669,32 @@ return err } cw := cbg.NewCborWriter(w) - fieldCount := 5 - if t.Repo == nil { - fieldCount-- + if _, err := cw.Write([]byte{164}); err != nil { + return err } - if t.RepoDid == nil { - fieldCount-- + // t.Repo (string) (string) + if len("repo") > 1000000 { + return xerrors.Errorf("Value in field \"repo\" was too long") } - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { return err } - - // t.Repo (string) (string) - if t.Repo != nil { - - if len("repo") > 1000000 { - return xerrors.Errorf("Value in field \"repo\" was too long") - } + if _, err := cw.WriteString(string("repo")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repo")); err != nil { - return err - } - - if t.Repo == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.Repo) > 1000000 { - return xerrors.Errorf("Value in field t.Repo was too long") - } + if len(t.Repo) > 1000000 { + return xerrors.Errorf("Value in field t.Repo was too long") + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Repo))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.Repo)); err != nil { - return err - } - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Repo))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Repo)); err != nil { + return err } // t.LexiconTypeID (string) (string) @@ -7587,38 +7716,6 @@ if _, err := cw.WriteString(string("sh.tangled.repo.collaborator")); err != nil { return err } - // t.RepoDid (string) (string) - if t.RepoDid != nil { - - if len("repoDid") > 1000000 { - return xerrors.Errorf("Value in field \"repoDid\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repoDid"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repoDid")); err != nil { - return err - } - - if t.RepoDid == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.RepoDid) > 1000000 { - return xerrors.Errorf("Value in field t.RepoDid was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.RepoDid))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.RepoDid)); err != nil { - return err - } - } - } - // t.Subject (string) (string) if len("subject") > 1000000 { return xerrors.Errorf("Value in field \"subject\" was too long") @@ -7712,22 +7809,12 @@ // t.Repo (string) (string) case "repo": { - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.Repo = (*string)(&sval) - } + t.Repo = string(sval) } // t.LexiconTypeID (string) (string) case "$type": @@ -7739,27 +7826,6 @@ return err } t.LexiconTypeID = string(sval) - } - // t.RepoDid (string) (string) - case "repoDid": - - { - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.RepoDid = (*string)(&sval) - } } // t.Subject (string) (string) case "subject": @@ -7801,7 +7867,7 @@ return err } cw := cbg.NewCborWriter(w) - fieldCount := 8 + fieldCount := 7 if t.Body == nil { fieldCount-- @@ -7812,14 +7878,6 @@ fieldCount-- } if t.References == nil { - fieldCount-- - } - - if t.Repo == nil { - fieldCount-- - } - - if t.RepoDid == nil { fieldCount-- } @@ -7860,35 +7918,26 @@ } } // t.Repo (string) (string) - if t.Repo != nil { + if len("repo") > 1000000 { + return xerrors.Errorf("Value in field \"repo\" was too long") + } - if len("repo") > 1000000 { - return xerrors.Errorf("Value in field \"repo\" was too long") - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { + return err + } + if _, err := cw.WriteString(string("repo")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repo")); err != nil { - return err - } - - if t.Repo == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.Repo) > 1000000 { - return xerrors.Errorf("Value in field t.Repo was too long") - } + if len(t.Repo) > 1000000 { + return xerrors.Errorf("Value in field t.Repo was too long") + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Repo))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.Repo)); err != nil { - return err - } - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Repo))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Repo)); err != nil { + return err } // t.LexiconTypeID (string) (string) @@ -7933,38 +7982,6 @@ if _, err := cw.WriteString(string(t.Title)); err != nil { return err } - // t.RepoDid (string) (string) - if t.RepoDid != nil { - - if len("repoDid") > 1000000 { - return xerrors.Errorf("Value in field \"repoDid\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repoDid"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repoDid")); err != nil { - return err - } - - if t.RepoDid == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.RepoDid) > 1000000 { - return xerrors.Errorf("Value in field t.RepoDid was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.RepoDid))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.RepoDid)); err != nil { - return err - } - } - } - // t.Mentions ([]string) (slice) if t.Mentions != nil { @@ -8128,22 +8145,12 @@ // t.Repo (string) (string) case "repo": { - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.Repo = (*string)(&sval) - } + t.Repo = string(sval) } // t.LexiconTypeID (string) (string) case "$type": @@ -8166,27 +8173,6 @@ return err } t.Title = string(sval) - } - // t.RepoDid (string) (string) - case "repoDid": - - { - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.RepoDid = (*string)(&sval) - } } // t.Mentions ([]string) (slice) case "mentions": @@ -9965,13 +9951,9 @@ return err } cw := cbg.NewCborWriter(w) - fieldCount := 3 + fieldCount := 2 if t.Repo == nil { - fieldCount-- - } - - if t.RepoDid == nil { fieldCount-- } @@ -10033,38 +10015,6 @@ } if _, err := cw.WriteString(string(t.Branch)); err != nil { return err } - - // t.RepoDid (string) (string) - if t.RepoDid != nil { - - if len("repoDid") > 1000000 { - return xerrors.Errorf("Value in field \"repoDid\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repoDid"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repoDid")); err != nil { - return err - } - - if t.RepoDid == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.RepoDid) > 1000000 { - return xerrors.Errorf("Value in field t.RepoDid was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.RepoDid))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.RepoDid)); err != nil { - return err - } - } - } return nil } @@ -10093,7 +10043,7 @@ } n := extra - nameBuf := make([]byte, 7) + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -10141,27 +10091,6 @@ } t.Branch = string(sval) } - // t.RepoDid (string) (string) - case "repoDid": - - { - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.RepoDid = (*string)(&sval) - } - } default: // Field doesn't exist on this type, so ignore it @@ -10344,50 +10273,32 @@ return err } cw := cbg.NewCborWriter(w) - fieldCount := 3 - if t.Repo == nil { - fieldCount-- + if _, err := cw.Write([]byte{162}); err != nil { + return err } - if t.RepoDid == nil { - fieldCount-- + // t.Repo (string) (string) + if len("repo") > 1000000 { + return xerrors.Errorf("Value in field \"repo\" was too long") } - if _, err := cw.Write(cbg.CborEncodeMajorType(cbg.MajMap, uint64(fieldCount))); err != nil { + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { return err } - - // t.Repo (string) (string) - if t.Repo != nil { - - if len("repo") > 1000000 { - return xerrors.Errorf("Value in field \"repo\" was too long") - } + if _, err := cw.WriteString(string("repo")); err != nil { + return err + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repo"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repo")); err != nil { - return err - } - - if t.Repo == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.Repo) > 1000000 { - return xerrors.Errorf("Value in field t.Repo was too long") - } + if len(t.Repo) > 1000000 { + return xerrors.Errorf("Value in field t.Repo was too long") + } - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.Repo))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.Repo)); err != nil { - return err - } - } + if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(t.Repo))); err != nil { + return err + } + if _, err := cw.WriteString(string(t.Repo)); err != nil { + return err } // t.Branch (string) (string) @@ -10412,38 +10323,6 @@ } if _, err := cw.WriteString(string(t.Branch)); err != nil { return err } - - // t.RepoDid (string) (string) - if t.RepoDid != nil { - - if len("repoDid") > 1000000 { - return xerrors.Errorf("Value in field \"repoDid\" was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("repoDid"))); err != nil { - return err - } - if _, err := cw.WriteString(string("repoDid")); err != nil { - return err - } - - if t.RepoDid == nil { - if _, err := cw.Write(cbg.CborNull); err != nil { - return err - } - } else { - if len(*t.RepoDid) > 1000000 { - return xerrors.Errorf("Value in field t.RepoDid was too long") - } - - if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len(*t.RepoDid))); err != nil { - return err - } - if _, err := cw.WriteString(string(*t.RepoDid)); err != nil { - return err - } - } - } return nil } @@ -10472,7 +10351,7 @@ } n := extra - nameBuf := make([]byte, 7) + nameBuf := make([]byte, 6) for i := uint64(0); i < n; i++ { nameLen, ok, err := cbg.ReadFullStringIntoBuf(cr, nameBuf, 1000000) if err != nil { @@ -10492,22 +10371,12 @@ // t.Repo (string) (string) case "repo": { - b, err := cr.ReadByte() + sval, err := cbg.ReadStringWithMax(cr, 1000000) if err != nil { return err } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.Repo = (*string)(&sval) - } + t.Repo = string(sval) } // t.Branch (string) (string) case "branch": @@ -10519,27 +10388,6 @@ return err } t.Branch = string(sval) - } - // t.RepoDid (string) (string) - case "repoDid": - - { - b, err := cr.ReadByte() - if err != nil { - return err - } - if b != cbg.CborNull[0] { - if err := cr.UnreadByte(); err != nil { - return err - } - - sval, err := cbg.ReadStringWithMax(cr, 1000000) - if err != nil { - return err - } - - t.RepoDid = (*string)(&sval) - } } default: diff --git a/api/tangled/feedstar.go b/api/tangled/feedstar.go --- a/api/tangled/feedstar.go +++ b/api/tangled/feedstar.go @@ -5,7 +5,13 @@ // schema: sh.tangled.feed.star import ( + "bytes" + "encoding/json" + "fmt" + "io" + "github.com/bluesky-social/indigo/lex/util" + cbg "github.com/whyrusleeping/cbor-gen" ) const ( @@ -17,8 +23,91 @@ util.RegisterType("sh.tangled.feed.star", &FeedStar{}) } // // RECORDTYPE: FeedStar type FeedStar struct { - LexiconTypeID string `json:"$type,const=sh.tangled.feed.star" cborgen:"$type,const=sh.tangled.feed.star"` - CreatedAt string `json:"createdAt" cborgen:"createdAt"` - Subject *string `json:"subject,omitempty" cborgen:"subject,omitempty"` - SubjectDid *string `json:"subjectDid,omitempty" cborgen:"subjectDid,omitempty"` + LexiconTypeID string `json:"$type,const=sh.tangled.feed.star" cborgen:"$type,const=sh.tangled.feed.star"` + CreatedAt string `json:"createdAt" cborgen:"createdAt"` + Subject *FeedStar_Subject `json:"subject" cborgen:"subject"` +} + +// FeedStar_Repo is a "repo" in the sh.tangled.feed.star schema. +// +// RECORDTYPE: FeedStar_Repo +type FeedStar_Repo struct { + LexiconTypeID string `json:"$type,const=sh.tangled.feed.star#repo" cborgen:"$type,const=sh.tangled.feed.star#repo"` + Did string `json:"did" cborgen:"did"` +} + +// FeedStar_String is a "string" in the sh.tangled.feed.star schema. +// +// RECORDTYPE: FeedStar_String +type FeedStar_String struct { + LexiconTypeID string `json:"$type,const=sh.tangled.feed.star#string" cborgen:"$type,const=sh.tangled.feed.star#string"` + Uri string `json:"uri" cborgen:"uri"` +} + +type FeedStar_Subject struct { + FeedStar_Repo *FeedStar_Repo + FeedStar_String *FeedStar_String +} + +func (t *FeedStar_Subject) MarshalJSON() ([]byte, error) { + if t.FeedStar_Repo != nil { + t.FeedStar_Repo.LexiconTypeID = "sh.tangled.feed.star#repo" + return json.Marshal(t.FeedStar_Repo) + } + if t.FeedStar_String != nil { + t.FeedStar_String.LexiconTypeID = "sh.tangled.feed.star#string" + return json.Marshal(t.FeedStar_String) + } + return nil, fmt.Errorf("cannot marshal empty enum") +} +func (t *FeedStar_Subject) UnmarshalJSON(b []byte) error { + typ, err := util.TypeExtract(b) + if err != nil { + return err + } + + switch typ { + case "sh.tangled.feed.star#repo": + t.FeedStar_Repo = new(FeedStar_Repo) + return json.Unmarshal(b, t.FeedStar_Repo) + case "sh.tangled.feed.star#string": + t.FeedStar_String = new(FeedStar_String) + return json.Unmarshal(b, t.FeedStar_String) + + default: + return fmt.Errorf("closed enums must have a matching value") + } +} + +func (t *FeedStar_Subject) MarshalCBOR(w io.Writer) error { + + if t == nil { + _, err := w.Write(cbg.CborNull) + return err + } + if t.FeedStar_Repo != nil { + return t.FeedStar_Repo.MarshalCBOR(w) + } + if t.FeedStar_String != nil { + return t.FeedStar_String.MarshalCBOR(w) + } + return fmt.Errorf("cannot cbor marshal empty enum") +} +func (t *FeedStar_Subject) UnmarshalCBOR(r io.Reader) error { + typ, b, err := util.CborTypeExtractReader(r) + if err != nil { + return err + } + + switch typ { + case "sh.tangled.feed.star#repo": + t.FeedStar_Repo = new(FeedStar_Repo) + return t.FeedStar_Repo.UnmarshalCBOR(bytes.NewReader(b)) + case "sh.tangled.feed.star#string": + t.FeedStar_String = new(FeedStar_String) + return t.FeedStar_String.UnmarshalCBOR(bytes.NewReader(b)) + + default: + return fmt.Errorf("closed enums must have a matching value") + } } diff --git a/api/tangled/gitrefUpdate.go b/api/tangled/gitrefUpdate.go --- a/api/tangled/gitrefUpdate.go +++ b/api/tangled/gitrefUpdate.go @@ -29,10 +29,8 @@ // ownerDid: did of the owner of the repo OwnerDid *string `json:"ownerDid,omitempty" cborgen:"ownerDid,omitempty"` // ref: Ref being updated Ref string `json:"ref" cborgen:"ref"` - // repoDid: DID of the repo itself - RepoDid *string `json:"repoDid,omitempty" cborgen:"repoDid,omitempty"` - // repoName: name of the repo - RepoName string `json:"repoName" cborgen:"repoName"` + // repo: DID of the repo itself + Repo string `json:"repo" cborgen:"repo"` } // GitRefUpdate_CommitCountBreakdown is a "commitCountBreakdown" in the sh.tangled.git.refUpdate schema. diff --git a/api/tangled/repocollaborator.go b/api/tangled/repocollaborator.go --- a/api/tangled/repocollaborator.go +++ b/api/tangled/repocollaborator.go @@ -19,8 +19,7 @@ // RECORDTYPE: RepoCollaborator type RepoCollaborator struct { LexiconTypeID string `json:"$type,const=sh.tangled.repo.collaborator" cborgen:"$type,const=sh.tangled.repo.collaborator"` CreatedAt string `json:"createdAt" cborgen:"createdAt"` - // repo: repo to add this user to - Repo *string `json:"repo,omitempty" cborgen:"repo,omitempty"` - RepoDid *string `json:"repoDid,omitempty" cborgen:"repoDid,omitempty"` - Subject string `json:"subject" cborgen:"subject"` + // repo: repo DID to add this user to + Repo string `json:"repo" cborgen:"repo"` + Subject string `json:"subject" cborgen:"subject"` } diff --git a/api/tangled/repodescribeRepo.go b/api/tangled/repodescribeRepo.go new file mode 100644 --- /dev/null +++ b/api/tangled/repodescribeRepo.go @@ -0,0 +1,39 @@ +// Code generated by cmd/lexgen (see Makefile's lexgen); DO NOT EDIT. + +package tangled + +// schema: sh.tangled.repo.describeRepo + +import ( + "context" + + "github.com/bluesky-social/indigo/lex/util" +) + +const ( + RepoDescribeRepoNSID = "sh.tangled.repo.describeRepo" +) + +// RepoDescribeRepo_Output is the output of a sh.tangled.repo.describeRepo call. +type RepoDescribeRepo_Output struct { + // ownerDid: DID of the current owner according to the knot. + OwnerDid string `json:"ownerDid" cborgen:"ownerDid"` + RepoDid string `json:"repoDid" cborgen:"repoDid"` + // rkey: Current rkey of the sh.tangled.repo record tracked by this knot + Rkey string `json:"rkey" cborgen:"rkey"` +} + +// RepoDescribeRepo calls the XRPC method "sh.tangled.repo.describeRepo". +// +// repoDid: DID of the git repo as minted by the knot +func RepoDescribeRepo(ctx context.Context, c util.LexClient, repoDid string) (*RepoDescribeRepo_Output, error) { + var out RepoDescribeRepo_Output + + params := map[string]interface{}{} + params["repoDid"] = repoDid + if err := c.LexDo(ctx, util.Query, "", "sh.tangled.repo.describeRepo", params, nil, &out); err != nil { + return nil, err + } + + return &out, nil +} diff --git a/api/tangled/repoissue.go b/api/tangled/repoissue.go --- a/api/tangled/repoissue.go +++ b/api/tangled/repoissue.go @@ -22,7 +22,6 @@ Body *string `json:"body,omitempty" cborgen:"body,omitempty"` CreatedAt string `json:"createdAt" cborgen:"createdAt"` Mentions []string `json:"mentions,omitempty" cborgen:"mentions,omitempty"` References []string `json:"references,omitempty" cborgen:"references,omitempty"` - Repo *string `json:"repo,omitempty" cborgen:"repo,omitempty"` - RepoDid *string `json:"repoDid,omitempty" cborgen:"repoDid,omitempty"` + Repo string `json:"repo" cborgen:"repo"` Title string `json:"title" cborgen:"title"` } diff --git a/api/tangled/repopull.go b/api/tangled/repopull.go --- a/api/tangled/repopull.go +++ b/api/tangled/repopull.go @@ -39,14 +39,12 @@ } // RepoPull_Source is a "source" in the sh.tangled.repo.pull schema. type RepoPull_Source struct { - Branch string `json:"branch" cborgen:"branch"` - Repo *string `json:"repo,omitempty" cborgen:"repo,omitempty"` - RepoDid *string `json:"repoDid,omitempty" cborgen:"repoDid,omitempty"` + Branch string `json:"branch" cborgen:"branch"` + Repo *string `json:"repo,omitempty" cborgen:"repo,omitempty"` } // RepoPull_Target is a "target" in the sh.tangled.repo.pull schema. type RepoPull_Target struct { - Branch string `json:"branch" cborgen:"branch"` - Repo *string `json:"repo,omitempty" cborgen:"repo,omitempty"` - RepoDid *string `json:"repoDid,omitempty" cborgen:"repoDid,omitempty"` + Branch string `json:"branch" cborgen:"branch"` + Repo string `json:"repo" cborgen:"repo"` } diff --git a/api/tangled/tangledrepo.go b/api/tangled/tangledrepo.go --- a/api/tangled/tangledrepo.go +++ b/api/tangled/tangledrepo.go @@ -24,8 +24,8 @@ // knot: knot where the repo was created Knot string `json:"knot" cborgen:"knot"` // labels: List of labels that this repo subscribes to Labels []string `json:"labels,omitempty" cborgen:"labels,omitempty"` - // name: name of the repo - Name string `json:"name" cborgen:"name"` + // name: Cosmetic name of the repo. + Name *string `json:"name,omitempty" cborgen:"name,omitempty"` // repoDid: DID of the repo itself, if assigned RepoDid *string `json:"repoDid,omitempty" cborgen:"repoDid,omitempty"` // source: source of the repo diff --git a/appview/compat113/compat.go b/appview/compat113/compat.go new file mode 100644 --- /dev/null +++ b/appview/compat113/compat.go @@ -0,0 +1,87 @@ +package compat113 + +import ( + "encoding/json" + "io" + + lexutil "github.com/bluesky-social/indigo/lex/util" + "tangled.org/core/api/tangled" +) + +func Collaborator(r *tangled.RepoCollaborator) *lexutil.LexiconTypeDecoder { + return &lexutil.LexiconTypeDecoder{Val: &collaboratorWrapper{inner: r}} +} + +func Pull(r *tangled.RepoPull) *lexutil.LexiconTypeDecoder { + return &lexutil.LexiconTypeDecoder{Val: &pullWrapper{inner: r}} +} + +type collaboratorWrapper struct { + LexiconTypeID string `cborgen:"$type,const=sh.tangled.repo.collaborator"` + inner *tangled.RepoCollaborator +} + +func (c *collaboratorWrapper) MarshalJSON() ([]byte, error) { + c.inner.LexiconTypeID = "sh.tangled.repo.collaborator" + return marshalWithRepoDidShadow(c.inner, false) +} + +func (c *collaboratorWrapper) MarshalCBOR(w io.Writer) error { + return c.inner.MarshalCBOR(w) +} + +type pullWrapper struct { + LexiconTypeID string `cborgen:"$type,const=sh.tangled.repo.pull"` + inner *tangled.RepoPull +} + +func (c *pullWrapper) MarshalJSON() ([]byte, error) { + c.inner.LexiconTypeID = "sh.tangled.repo.pull" + return marshalWithRepoDidShadow(c.inner, true) +} + +func (c *pullWrapper) MarshalCBOR(w io.Writer) error { + return c.inner.MarshalCBOR(w) +} + +func marshalWithRepoDidShadow(inner any, nestedTarget bool) ([]byte, error) { + raw, err := json.Marshal(inner) + if err != nil { + return nil, err + } + var top map[string]json.RawMessage + if err := json.Unmarshal(raw, &top); err != nil { + return raw, nil + } + if nestedTarget { + injectIntoNested(top, "target") + injectIntoNested(top, "source") + } else { + addRepoDidShadow(top) + } + return json.Marshal(top) +} + +func injectIntoNested(parent map[string]json.RawMessage, key string) { + raw, ok := parent[key] + if !ok { + return + } + var nested map[string]json.RawMessage + if err := json.Unmarshal(raw, &nested); err != nil { + return + } + addRepoDidShadow(nested) + if reb, err := json.Marshal(nested); err == nil { + parent[key] = reb + } +} + +func addRepoDidShadow(m map[string]json.RawMessage) { + if _, has := m["repoDid"]; has { + return + } + if v, ok := m["repo"]; ok { + m["repoDid"] = v + } +} diff --git a/appview/compat113/compat_test.go b/appview/compat113/compat_test.go new file mode 100644 --- /dev/null +++ b/appview/compat113/compat_test.go @@ -0,0 +1,113 @@ +package compat113 + +import ( + "encoding/json" + "testing" + + "tangled.org/core/api/tangled" +) + +func ptr[T any](v T) *T { return &v } + +func TestCollaboratorShadowsRepoDid(t *testing.T) { + rec := &tangled.RepoCollaborator{ + CreatedAt: "2026-05-08T00:00:00Z", + Repo: "did:plc:abalone", + Subject: "did:plc:limpet", + } + + out, err := json.Marshal(Collaborator(rec)) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var got map[string]any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + if got["$type"] != "sh.tangled.repo.collaborator" { + t.Errorf("$type = %v, want sh.tangled.repo.collaborator", got["$type"]) + } + if got["repo"] != "did:plc:abalone" { + t.Errorf("repo = %v, want did:plc:abalone", got["repo"]) + } + if got["repoDid"] != "did:plc:abalone" { + t.Errorf("repoDid shadow missing or wrong: got %v", got["repoDid"]) + } +} + +func TestPullShadowsTargetRepoDid(t *testing.T) { + rec := &tangled.RepoPull{ + CreatedAt: "2026-05-08T00:00:00Z", + Title: "rename whelk handler", + Target: &tangled.RepoPull_Target{ + Branch: "main", + Repo: "did:plc:scallop", + }, + Source: &tangled.RepoPull_Source{ + Branch: "feature-1", + }, + } + + out, err := json.Marshal(Pull(rec)) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var got map[string]any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + target, ok := got["target"].(map[string]any) + if !ok { + t.Fatalf("target missing or wrong type: %v", got["target"]) + } + if target["repo"] != "did:plc:scallop" { + t.Errorf("target.repo = %v", target["repo"]) + } + if target["repoDid"] != "did:plc:scallop" { + t.Errorf("target.repoDid shadow missing: %v", target["repoDid"]) + } + + if _, has := got["repoDid"]; has { + t.Errorf("top-level repoDid should not be set on pull: %v", got["repoDid"]) + } +} + +func TestPullShadowsForkSourceRepoDid(t *testing.T) { + rec := &tangled.RepoPull{ + CreatedAt: "2026-05-08T00:00:00Z", + Title: "fork-based PR", + Target: &tangled.RepoPull_Target{ + Branch: "main", + Repo: "did:plc:scallop", + }, + Source: &tangled.RepoPull_Source{ + Branch: "feature-2", + Repo: ptr("did:plc:periwinkle"), + }, + } + + out, err := json.Marshal(Pull(rec)) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + var got map[string]any + if err := json.Unmarshal(out, &got); err != nil { + t.Fatalf("unmarshal: %v", err) + } + + source, ok := got["source"].(map[string]any) + if !ok { + t.Fatalf("source missing: %v", got["source"]) + } + if source["repo"] != "did:plc:periwinkle" { + t.Errorf("source.repo = %v", source["repo"]) + } + if source["repoDid"] != "did:plc:periwinkle" { + t.Errorf("source.repoDid shadow missing: %v", source["repoDid"]) + } +} diff --git a/appview/compat113/version.go b/appview/compat113/version.go new file mode 100644 --- /dev/null +++ b/appview/compat113/version.go @@ -0,0 +1,60 @@ +package compat113 + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "tangled.org/core/api/tangled" +) + +const versionProbeTimeout = 5 * time.Second + +func KnotSupports114(ctx context.Context, host string, dev bool) bool { + scheme := "https" + if dev { + scheme = "http" + } + client := &indigoxrpc.Client{ + Host: fmt.Sprintf("%s://%s", scheme, host), + Client: &http.Client{Timeout: versionProbeTimeout}, + } + + ctx, cancel := context.WithTimeout(ctx, versionProbeTimeout) + defer cancel() + + resp, err := tangled.KnotVersion(ctx, client) + if err != nil || resp == nil { + return true + } + return atLeast114(resp.Version) +} + +func atLeast114(v string) bool { + v = strings.TrimSpace(v) + v = strings.TrimPrefix(v, "v") + if strings.HasPrefix(v, "(devel)") { + return true + } + if v == "" { + return false + } + parts := strings.SplitN(v, ".", 3) + if len(parts) < 2 { + return false + } + major, err := strconv.Atoi(parts[0]) + if err != nil { + return false + } + minorRaw := strings.SplitN(parts[1], "-", 2)[0] + minor, err := strconv.Atoi(minorRaw) + if err != nil { + return false + } + return major > 1 || (major == 1 && minor >= 14) +} diff --git a/appview/compat113/version_test.go b/appview/compat113/version_test.go new file mode 100644 --- /dev/null +++ b/appview/compat113/version_test.go @@ -0,0 +1,35 @@ +package compat113 + +import "testing" + +func TestAtLeast114(t *testing.T) { + cases := []struct { + in string + want bool + }{ + {"v1.14.0", true}, + {"v1.14.0-alpha", true}, + {"v1.14.5", true}, + {"v1.13.0", false}, + {"v1.13.0-alpha", false}, + {"v1.0.0", false}, + {"v2.0.0", true}, + {"1.14.0", true}, + {"1.13.99", false}, + {"(devel)", true}, + {"", false}, + {"garbagio-furioso", false}, + {"v1", false}, + {"vX.Y.Z", false}, + {"unknown", false}, + {"unknown-abc1234", false}, + {"unknown-abc1234-modified", false}, + } + for _, c := range cases { + t.Run(c.in, func(t *testing.T) { + if got := atLeast114(c.in); got != c.want { + t.Errorf("atLeast114(%q) = %v, want %v", c.in, got, c.want) + } + }) + } +} diff --git a/appview/config/config.go b/appview/config/config.go --- a/appview/config/config.go +++ b/appview/config/config.go @@ -14,6 +14,7 @@ type CoreConfig struct { CookieSecret string `env:"COOKIE_SECRET, default=00000000000000000000000000000000"` DbPath string `env:"DB_PATH, default=appview.db"` ListenAddr string `env:"LISTEN_ADDR, default=0.0.0.0:3000"` + MetricsListenAddr string `env:"METRICS_LISTEN_ADDR, default=0.0.0.0:9090"` AppviewHost string `env:"APPVIEW_HOST, default=tangled.org"` AppviewName string `env:"APPVIEW_NAME, default=Tangled"` Dev bool `env:"DEV, default=false"` diff --git a/appview/db/artifact.go b/appview/db/artifact.go --- a/appview/db/artifact.go +++ b/appview/db/artifact.go @@ -16,7 +16,7 @@ _, err := e.Exec( `insert or ignore into artifacts ( did, rkey, - repo_at, + repo_did, tag, created, blob_cid, @@ -27,7 +27,7 @@ ) values (?, ?, ?, ?, ?, ?, ?, ?, ?)`, artifact.Did, artifact.Rkey, - artifact.RepoAt, + artifact.RepoDid, artifact.Tag[:], artifact.CreatedAt.Format(time.RFC3339), artifact.BlobCid.String(), @@ -56,7 +56,7 @@ query := fmt.Sprintf(`select did, rkey, - repo_at, + repo_did, tag, created, blob_cid, @@ -82,7 +82,7 @@ if err := rows.Scan( &artifact.Did, &artifact.Rkey, - &artifact.RepoAt, + &artifact.RepoDid, &tag, &createdAt, &blobCid, diff --git a/appview/db/collaborators.go b/appview/db/collaborators.go --- a/appview/db/collaborators.go +++ b/appview/db/collaborators.go @@ -11,8 +11,8 @@ ) func AddCollaborator(e Execer, c models.Collaborator) error { _, err := e.Exec( - `insert into collaborators (did, rkey, subject_did, repo_at) values (?, ?, ?, ?);`, - c.Did, c.Rkey, c.SubjectDid, c.RepoAt, + `insert into collaborators (did, rkey, subject_did, repo_did) values (?, ?, ?, ?);`, + c.Did, c.Rkey, c.SubjectDid, string(c.RepoDid), ) return err } @@ -37,29 +37,29 @@ return err } func CollaboratingIn(e Execer, collaborator string) ([]models.Repo, error) { - rows, err := e.Query(`select repo_at from collaborators where subject_did = ?`, collaborator) + rows, err := e.Query(`select repo_did from collaborators where subject_did = ?`, collaborator) if err != nil { return nil, err } defer rows.Close() - var repoAts []string + var repoDids []string for rows.Next() { - var aturi string - err := rows.Scan(&aturi) + var repoDid string + err := rows.Scan(&repoDid) if err != nil { return nil, err } - repoAts = append(repoAts, aturi) + repoDids = append(repoDids, repoDid) } if err := rows.Err(); err != nil { return nil, err } - if repoAts == nil { + if repoDids == nil { return nil, nil } - return GetRepos(e, orm.FilterIn("at_uri", repoAts)) + return GetRepos(e, orm.FilterIn("repo_did", repoDids)) } func GetCollaborators(e Execer, filters ...orm.Filter) ([]models.Collaborator, error) { @@ -79,7 +79,7 @@ id, did, rkey, subject_did, - repo_at, + repo_did, created from collaborators %s`, whereClause, @@ -97,7 +97,7 @@ &collaborator.Id, &collaborator.Did, &collaborator.Rkey, &collaborator.SubjectDid, - &collaborator.RepoAt, + &collaborator.RepoDid, &createdAt, ); err != nil { return nil, err diff --git a/appview/db/db.go b/appview/db/db.go --- a/appview/db/db.go +++ b/appview/db/db.go @@ -3,6 +3,7 @@ import ( "context" "database/sql" + "fmt" "log/slog" "strings" @@ -115,18 +116,6 @@ created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), issue_at text, unique(repo_at, issue_id), foreign key (repo_at) references repos(at_uri) on delete cascade - ); - create table if not exists comments ( - id integer primary key autoincrement, - owner_did text not null, - issue_id integer not null, - repo_at text not null, - comment_id integer not null, - comment_at text not null, - body text not null, - created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), - unique(issue_id, comment_id), - foreign key (repo_at, issue_id) references issues(repo_at, issue_id) on delete cascade ); create table if not exists pulls ( -- identifiers @@ -693,9 +682,7 @@ create index if not exists idx_notifications_recipient_created on notifications(recipient_did, created desc); create index if not exists idx_notifications_recipient_read on notifications(recipient_did, read); create index if not exists idx_references_from_at on reference_links(from_at); create index if not exists idx_references_to_at on reference_links(to_at); - create index if not exists idx_webhooks_repo_at on webhooks(repo_at); create index if not exists idx_webhook_deliveries_webhook_id on webhook_deliveries(webhook_id); - create index if not exists idx_site_deploys_repo_at on site_deploys(repo_at); create index if not exists idx_newsletter_prefs_user_did on newsletter_preferences(user_did); `) if err != nil { @@ -1518,6 +1505,456 @@ `) return err }) conn.ExecContext(ctx, "pragma foreign_keys = on;") + + conn.ExecContext(ctx, "pragma foreign_keys = off;") + orm.RunMigration(conn, logger, "drop-pipeline-statuses-pipeline-fk", func(tx *sql.Tx) error { + _, err := tx.Exec(` + create table if not exists pipeline_statuses_new ( + id integer primary key autoincrement, + spindle text not null, + rkey text not null, + + pipeline_knot text not null, + pipeline_rkey text not null, + + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + workflow text not null, + status text not null, + error text, + exit_code integer not null default 0, + + unique (spindle, rkey) + ); + + insert into pipeline_statuses_new + select * from pipeline_statuses; + + drop table pipeline_statuses; + alter table pipeline_statuses_new rename to pipeline_statuses; + `) + return err + }) + conn.ExecContext(ctx, "pragma foreign_keys = on;") + + orm.RunMigration(conn, logger, "add-repo-renames", func(tx *sql.Tx) error { + res, err := tx.Exec(` + update repos + set name = name || '-renamed-' || id || '-' || lower(hex(randomblob(4))) + where id in ( + select id from ( + select id, row_number() over ( + partition by did, knot, name + order by created desc, id desc + ) as rn + from repos + ) where rn > 1 + ); + `) + if err != nil { + return err + } + if n, _ := res.RowsAffected(); n > 0 { + logger.Warn("suffixed legacy duplicate repo names before adding unique index", "rows", n) + } + + var remaining int + if err := tx.QueryRow(` + select count(*) from ( + select 1 from repos group by did, knot, name having count(*) > 1 + ) + `).Scan(&remaining); err != nil { + return fmt.Errorf("checking for residual duplicate (did, knot, name) groups: %w", err) + } + if remaining > 0 { + return fmt.Errorf("add-repo-renames: %d duplicate (did, knot, name) groups remain after suffix pass; manual cleanup required before unique index can be created", remaining) + } + + _, err = tx.Exec(` + create table if not exists repo_renames ( + owner_did text not null, + old_rkey text not null, + repo_did text not null, + renamed_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + primary key (owner_did, old_rkey) + ); + create unique index if not exists idx_repos_owner_knot_name + on repos(did, knot, name); + `) + return err + }) + + orm.RunMigration(conn, logger, "repos-canonical-rkey-uniqueness", func(tx *sql.Tx) error { + _, err := tx.Exec(` + drop index if exists idx_repos_owner_knot_name; + create unique index if not exists idx_repos_did_rkey + on repos(did, rkey); + `) + return err + }) + + orm.RunMigration(conn, logger, "repo-did-references", func(tx *sql.Tx) error { + tables := []struct{ table, oldCol, newCol string }{ + {"issues", "repo_at", "repo_did"}, + {"pulls", "repo_at", "repo_did"}, + {"pull_comments", "repo_at", "repo_did"}, + {"stars", "subject_at", "subject_did"}, + {"artifacts", "repo_at", "repo_did"}, + {"webhooks", "repo_at", "repo_did"}, + {"repo_sites", "repo_at", "repo_did"}, + {"site_deploys", "repo_at", "repo_did"}, + {"collaborators", "repo_at", "repo_did"}, + {"repo_issue_seqs", "repo_at", "repo_did"}, + {"repo_pull_seqs", "repo_at", "repo_did"}, + {"repo_languages", "repo_at", "repo_did"}, + {"repo_labels", "repo_at", "repo_did"}, + } + + stmts := "" + for _, t := range tables { + stmts += fmt.Sprintf( + `ALTER TABLE %s ADD COLUMN %s TEXT; + UPDATE %s SET %s = (SELECT repos.repo_did FROM repos WHERE repos.at_uri = %s.%s); + CREATE INDEX IF NOT EXISTS idx_%s_%s ON %s(%s); + `, t.table, t.newCol, t.table, t.newCol, t.table, t.oldCol, t.table, t.newCol, t.table, t.newCol) + } + + stmts += `ALTER TABLE pulls ADD COLUMN source_repo_did TEXT; + UPDATE pulls SET source_repo_did = (SELECT repos.repo_did FROM repos WHERE repos.at_uri = pulls.source_repo_at); + + UPDATE profile_pinned_repositories SET pin = ( + SELECT repos.repo_did FROM repos WHERE repos.at_uri = profile_pinned_repositories.pin + ) WHERE pin LIKE 'at://%' + AND EXISTS (SELECT 1 FROM repos WHERE repos.at_uri = profile_pinned_repositories.pin AND repos.repo_did IS NOT NULL AND repos.repo_did != ''); + ` + + _, err := tx.Exec(stmts) + return err + }) + + orm.RunMigration(conn, logger, "backfill-pds-rewrites-star-issue-pull-collab", func(tx *sql.Tx) error { + type source struct { + userDidCol string + table string + nsid string + fkCol string + } + sources := []source{ + {"did", "stars", "sh.tangled.feed.star", "subject_at"}, + {"did", "issues", "sh.tangled.repo.issue", "repo_at"}, + {"owner_did", "pulls", "sh.tangled.repo.pull", "repo_at"}, + {"did", "collaborators", "sh.tangled.repo.collaborator", "repo_at"}, + } + + for _, src := range sources { + _, err := tx.Exec(fmt.Sprintf(` + INSERT INTO pds_migration (name, did, collection, rkey, status) + SELECT 'add-repo-did', t.%s, '%s', t.rkey, 'pending' + FROM %s t + JOIN repos r ON r.at_uri = t.%s + WHERE r.repo_did IS NOT NULL AND r.repo_did != '' + ON CONFLICT(name, did, collection, rkey) DO NOTHING + `, src.userDidCol, src.nsid, src.table, src.fkCol)) + if err != nil { + return fmt.Errorf("backfill pds rewrites for %s: %w", src.table, err) + } + } + + return nil + }) + + orm.RunMigration(conn, logger, "backfill-pds-rewrites-profiles", func(tx *sql.Tx) error { + _, err := tx.Exec(` + INSERT INTO pds_migration (name, did, collection, rkey, status) + SELECT DISTINCT 'add-repo-did', pp.did, 'sh.tangled.actor.profile', 'self', 'pending' + FROM profile_pinned_repositories pp + JOIN repos r ON r.at_uri = pp.pin + WHERE pp.pin LIKE 'at://%' + AND r.repo_did IS NOT NULL AND r.repo_did != '' + ON CONFLICT(name, did, collection, rkey) DO NOTHING + `) + if err != nil { + return fmt.Errorf("backfill pds rewrites for profiles: %w", err) + } + return nil + }) + + conn.ExecContext(ctx, "pragma foreign_keys = off;") + orm.RunMigration(conn, logger, "drop-old-at-uri-columns", func(tx *sql.Tx) error { + _, err := tx.Exec(` + CREATE TABLE repos_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + name TEXT NOT NULL, + knot TEXT NOT NULL, + rkey TEXT NOT NULL, + at_uri TEXT NOT NULL UNIQUE, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + description TEXT CHECK (length(description) <= 200), + source TEXT, + spindle TEXT, + website TEXT, + topics TEXT, + repo_did TEXT, + UNIQUE(did, rkey) + ); + INSERT INTO repos_new (id, did, name, knot, rkey, at_uri, created, description, source, spindle, website, topics, repo_did) + SELECT id, did, name, knot, rkey, at_uri, created, description, source, spindle, website, topics, repo_did + FROM repos; + DROP TABLE repos; + ALTER TABLE repos_new RENAME TO repos; + CREATE UNIQUE INDEX idx_repos_repo_did ON repos(repo_did); + CREATE UNIQUE INDEX idx_repos_did_rkey ON repos(did, rkey); + + CREATE TABLE issues_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + rkey TEXT NOT NULL, + at_uri TEXT GENERATED ALWAYS AS ('at://' || did || '/' || 'sh.tangled.repo.issue' || '/' || rkey) STORED, + repo_did TEXT NOT NULL, + issue_id INTEGER NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + open INTEGER NOT NULL DEFAULT 1, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + edited TEXT, + deleted TEXT, + UNIQUE(did, rkey), + UNIQUE(repo_did, issue_id), + UNIQUE(at_uri), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO issues_new (id, did, rkey, repo_did, issue_id, title, body, open, created, edited, deleted) + SELECT id, did, rkey, repo_did, issue_id, title, body, open, created, edited, deleted + FROM issues WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE issues; + ALTER TABLE issues_new RENAME TO issues; + CREATE INDEX idx_issues_repo_did ON issues(repo_did); + + CREATE TABLE pulls_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pull_id INTEGER NOT NULL, + at_uri TEXT GENERATED ALWAYS AS ('at://' || owner_did || '/' || 'sh.tangled.repo.pull' || '/' || rkey) STORED, + repo_did TEXT NOT NULL, + owner_did TEXT NOT NULL, + rkey TEXT NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + target_branch TEXT NOT NULL, + state INTEGER NOT NULL DEFAULT 0 CHECK (state IN (0, 1, 2, 3)), + source_branch TEXT, + source_repo_did TEXT, + change_id TEXT, + dependent_on TEXT, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(repo_did, pull_id), + UNIQUE(at_uri), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO pulls_new (id, pull_id, repo_did, owner_did, rkey, title, body, target_branch, state, source_branch, source_repo_did, change_id, dependent_on, created) + SELECT id, pull_id, repo_did, owner_did, rkey, title, body, target_branch, state, source_branch, source_repo_did, change_id, dependent_on, created + FROM pulls WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE pulls; + ALTER TABLE pulls_new RENAME TO pulls; + CREATE INDEX idx_pulls_repo_did ON pulls(repo_did); + CREATE INDEX idx_pulls_source_repo_did ON pulls(source_repo_did); + + CREATE TABLE pull_comments_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + pull_id INTEGER NOT NULL, + submission_id INTEGER NOT NULL, + repo_did TEXT NOT NULL, + owner_did TEXT NOT NULL, + comment_at TEXT NOT NULL, + body TEXT NOT NULL, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + FOREIGN KEY (repo_did, pull_id) REFERENCES pulls(repo_did, pull_id) ON DELETE CASCADE, + FOREIGN KEY (submission_id) REFERENCES pull_submissions(id) ON DELETE CASCADE + ); + INSERT INTO pull_comments_new (id, pull_id, submission_id, repo_did, owner_did, comment_at, body, created) + SELECT id, pull_id, submission_id, repo_did, owner_did, comment_at, body, created + FROM pull_comments WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE pull_comments; + ALTER TABLE pull_comments_new RENAME TO pull_comments; + CREATE INDEX idx_pull_comments_repo_did ON pull_comments(repo_did); + + CREATE TABLE stars_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + rkey TEXT NOT NULL, + subject_type TEXT NOT NULL CHECK (subject_type IN ('repo', 'string')), + subject TEXT NOT NULL, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(did, rkey), + UNIQUE(did, subject) + ); + INSERT INTO stars_new (id, did, rkey, subject_type, subject, created) + SELECT id, did, rkey, 'repo', subject_did, created + FROM stars + WHERE subject_did IS NOT NULL AND subject_did != ''; + INSERT OR IGNORE INTO stars_new (id, did, rkey, subject_type, subject, created) + SELECT id, did, rkey, 'string', subject_at, created + FROM stars + WHERE (subject_did IS NULL OR subject_did = '') + AND subject_at LIKE 'at://%/sh.tangled.string/%'; + DROP TABLE stars; + ALTER TABLE stars_new RENAME TO stars; + CREATE INDEX idx_stars_subject ON stars(subject); + CREATE INDEX idx_stars_subject_type ON stars(subject_type); + CREATE INDEX idx_stars_created ON stars(created); + + CREATE TABLE collaborators_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + rkey TEXT, + subject_did TEXT NOT NULL, + repo_did TEXT NOT NULL, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + UNIQUE(did, rkey), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO collaborators_new (id, did, rkey, subject_did, repo_did, created) + SELECT id, did, NULLIF(rkey, ''), subject_did, repo_did, created + FROM collaborators WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE collaborators; + ALTER TABLE collaborators_new RENAME TO collaborators; + CREATE INDEX idx_collaborators_repo_did ON collaborators(repo_did); + + CREATE TABLE artifacts_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + did TEXT NOT NULL, + rkey TEXT NOT NULL, + repo_did TEXT NOT NULL, + tag BINARY(20) NOT NULL, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + blob_cid TEXT NOT NULL, + name TEXT NOT NULL, + size INTEGER NOT NULL DEFAULT 0, + mimetype TEXT NOT NULL DEFAULT '*/*', + UNIQUE(did, rkey), + UNIQUE(repo_did, tag, name), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO artifacts_new (id, did, rkey, repo_did, tag, created, blob_cid, name, size, mimetype) + SELECT id, did, rkey, repo_did, tag, created, blob_cid, name, size, mimetype + FROM artifacts WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE artifacts; + ALTER TABLE artifacts_new RENAME TO artifacts; + CREATE INDEX idx_artifacts_repo_did ON artifacts(repo_did); + + CREATE TABLE webhooks_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_did TEXT NOT NULL, + url TEXT NOT NULL, + secret TEXT, + active INTEGER NOT NULL DEFAULT 1, + events TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO webhooks_new (id, repo_did, url, secret, active, events, created_at, updated_at) + SELECT id, repo_did, url, secret, active, events, created_at, updated_at + FROM webhooks WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE webhooks; + ALTER TABLE webhooks_new RENAME TO webhooks; + CREATE INDEX idx_webhooks_repo_did ON webhooks(repo_did); + + CREATE TABLE repo_sites_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_did TEXT NOT NULL UNIQUE, + branch TEXT NOT NULL, + dir TEXT NOT NULL DEFAULT '/', + is_index INTEGER NOT NULL DEFAULT 0, + created TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + updated TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO repo_sites_new (id, repo_did, branch, dir, is_index, created, updated) + SELECT id, repo_did, branch, dir, is_index, created, updated + FROM repo_sites WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE repo_sites; + ALTER TABLE repo_sites_new RENAME TO repo_sites; + + CREATE TABLE site_deploys_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_did TEXT NOT NULL, + branch TEXT NOT NULL, + dir TEXT NOT NULL DEFAULT '/', + commit_sha TEXT NOT NULL DEFAULT '', + status TEXT NOT NULL CHECK (status IN ('success', 'failure')), + trigger TEXT NOT NULL CHECK (trigger IN ('config_change', 'push')), + error TEXT NOT NULL DEFAULT '', + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO site_deploys_new (id, repo_did, branch, dir, commit_sha, status, trigger, error, created_at) + SELECT id, repo_did, branch, dir, commit_sha, status, trigger, error, created_at + FROM site_deploys WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE site_deploys; + ALTER TABLE site_deploys_new RENAME TO site_deploys; + CREATE INDEX idx_site_deploys_repo_did ON site_deploys(repo_did); + + CREATE TABLE repo_issue_seqs_new ( + repo_did TEXT PRIMARY KEY, + next_issue_id INTEGER NOT NULL DEFAULT 1, + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO repo_issue_seqs_new (repo_did, next_issue_id) + SELECT repo_did, next_issue_id + FROM repo_issue_seqs WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE repo_issue_seqs; + ALTER TABLE repo_issue_seqs_new RENAME TO repo_issue_seqs; + + CREATE TABLE repo_pull_seqs_new ( + repo_did TEXT PRIMARY KEY, + next_pull_id INTEGER NOT NULL DEFAULT 1, + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO repo_pull_seqs_new (repo_did, next_pull_id) + SELECT repo_did, next_pull_id + FROM repo_pull_seqs WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE repo_pull_seqs; + ALTER TABLE repo_pull_seqs_new RENAME TO repo_pull_seqs; + + CREATE TABLE repo_languages_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_did TEXT NOT NULL, + ref TEXT NOT NULL, + is_default_ref INTEGER NOT NULL DEFAULT 0, + language TEXT NOT NULL, + bytes INTEGER NOT NULL CHECK (bytes >= 0), + UNIQUE(repo_did, ref, language), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO repo_languages_new (id, repo_did, ref, is_default_ref, language, bytes) + SELECT id, repo_did, ref, is_default_ref, language, bytes + FROM repo_languages WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE repo_languages; + ALTER TABLE repo_languages_new RENAME TO repo_languages; + + CREATE TABLE repo_labels_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repo_did TEXT NOT NULL, + label_at TEXT NOT NULL, + UNIQUE(repo_did, label_at), + FOREIGN KEY (repo_did) REFERENCES repos(repo_did) ON DELETE CASCADE + ); + INSERT INTO repo_labels_new (id, repo_did, label_at) + SELECT id, repo_did, label_at + FROM repo_labels WHERE repo_did IS NOT NULL AND repo_did != ''; + DROP TABLE repo_labels; + ALTER TABLE repo_labels_new RENAME TO repo_labels; + `) + return err + }) + conn.ExecContext(ctx, "pragma foreign_keys = on;") + + orm.RunMigration(conn, logger, "migrate-knots-to-repo-did-rename", func(tx *sql.Tx) error { + _, err := tx.Exec(` + update registrations set needs_upgrade = 1; + `) + return err + }) return &DB{ db, diff --git a/appview/db/issues.go b/appview/db/issues.go --- a/appview/db/issues.go +++ b/appview/db/issues.go @@ -19,9 +19,9 @@ func PutIssue(tx *sql.Tx, issue *models.Issue) error { // ensure sequence exists _, err := tx.Exec(` - insert or ignore into repo_issue_seqs (repo_at, next_issue_id) + insert or ignore into repo_issue_seqs (repo_did, next_issue_id) values (?, 1) - `, issue.RepoAt) + `, issue.RepoDid) if err != nil { return err } @@ -57,19 +57,19 @@ var newIssueId int err := tx.QueryRow(` update repo_issue_seqs set next_issue_id = next_issue_id + 1 - where repo_at = ? + where repo_did = ? returning next_issue_id - 1 - `, issue.RepoAt).Scan(&newIssueId) + `, issue.RepoDid).Scan(&newIssueId) if err != nil { return err } // insert new issue row := tx.QueryRow(` - insert into issues (repo_at, did, rkey, issue_id, title, body) + insert into issues (repo_did, did, rkey, issue_id, title, body) values (?, ?, ?, ?, ?, ?) returning rowid, issue_id - `, issue.RepoAt, issue.Did, issue.Rkey, newIssueId, issue.Title, issue.Body) + `, issue.RepoDid, issue.Did, issue.Rkey, newIssueId, issue.Title, issue.Body) err = row.Scan(&issue.Id, &issue.IssueId) if err != nil { @@ -132,7 +132,7 @@ select id, did, rkey, - repo_at, + repo_did, issue_id, title, body, @@ -166,7 +166,7 @@ err := rows.Scan( &issue.Id, &issue.Did, &issue.Rkey, - &issue.RepoAt, + &issue.RepoDid, &issue.IssueId, &issue.Title, &issue.Body, @@ -201,23 +201,23 @@ issueMap[atUri] = &issue } // collect reverse repos - repoAts := make([]string, 0, len(issueMap)) // or just []string{} + repoDids := make([]string, 0, len(issueMap)) for _, issue := range issueMap { - repoAts = append(repoAts, string(issue.RepoAt)) + repoDids = append(repoDids, string(issue.RepoDid)) } - repos, err := GetRepos(e, orm.FilterIn("at_uri", repoAts)) + repos, err := GetRepos(e, orm.FilterIn("repo_did", repoDids)) if err != nil { return nil, fmt.Errorf("failed to build repo mappings: %w", err) } repoMap := make(map[string]*models.Repo) for i := range repos { - repoMap[string(repos[i].RepoAt())] = &repos[i] + repoMap[repos[i].RepoDid] = &repos[i] } for issueAt, i := range issueMap { - if r, ok := repoMap[string(i.RepoAt)]; ok { + if r, ok := repoMap[string(i.RepoDid)]; ok { i.Repo = r } else { // do not show up the issue if the repo is deleted @@ -274,11 +274,11 @@ return issues, nil } -func GetIssue(e Execer, repoAt syntax.ATURI, issueId int) (*models.Issue, error) { +func GetIssue(e Execer, repoDid string, issueId int) (*models.Issue, error) { issues, err := GetIssuesPaginated( e, pagination.Page{}, - orm.FilterEq("repo_at", repoAt), + orm.FilterEq("repo_did", repoDid), orm.FilterEq("issue_id", issueId), ) if err != nil { @@ -530,14 +530,14 @@ _, err := e.Exec(query, args...) return err } -func GetIssueCount(e Execer, repoAt syntax.ATURI) (models.IssueCount, error) { +func GetIssueCount(e Execer, repoDid string) (models.IssueCount, error) { row := e.QueryRow(` select count(case when open = 1 then 1 end) as open_count, count(case when open = 0 then 1 end) as closed_count from issues - where repo_at = ?`, - repoAt, + where repo_did = ?`, + repoDid, ) var count models.IssueCount diff --git a/appview/db/jetstream.go b/appview/db/jetstream.go --- a/appview/db/jetstream.go +++ b/appview/db/jetstream.go @@ -1,10 +1,6 @@ package db -type DbWrapper struct { - Execer -} - -func (db DbWrapper) SaveLastTimeUs(lastTimeUs int64) error { +func (db *DB) SaveLastTimeUs(lastTimeUs int64) error { _, err := db.Exec(` insert into _jetstream (id, last_time_us) values (1, ?) @@ -13,7 +9,7 @@ `, lastTimeUs) return err } -func (db DbWrapper) GetLastTimeUs() (int64, error) { +func (db *DB) GetLastTimeUs() (int64, error) { var lastTimeUs int64 row := db.QueryRow(`select last_time_us from _jetstream where id = 1;`) err := row.Scan(&lastTimeUs) diff --git a/appview/db/language.go b/appview/db/language.go --- a/appview/db/language.go +++ b/appview/db/language.go @@ -24,7 +24,7 @@ whereClause = " where " + strings.Join(conditions, " and ") } query := fmt.Sprintf( - `select id, repo_at, ref, is_default_ref, language, bytes from repo_languages %s`, + `select id, repo_did, ref, is_default_ref, language, bytes from repo_languages %s`, whereClause, ) rows, err := e.Query(query, args...) @@ -40,7 +40,7 @@ var isDefaultRef int err := rows.Scan( &rl.Id, - &rl.RepoAt, + &rl.RepoDid, &rl.Ref, &isDefaultRef, &rl.Language, @@ -65,7 +65,7 @@ } func InsertRepoLanguages(e Execer, langs []models.RepoLanguage) error { stmt, err := e.Prepare( - "insert or replace into repo_languages (repo_at, ref, is_default_ref, language, bytes) values (?, ?, ?, ?, ?)", + "insert or replace into repo_languages (repo_did, ref, is_default_ref, language, bytes) values (?, ?, ?, ?, ?)", ) if err != nil { return err @@ -77,7 +77,7 @@ if l.IsDefaultRef { isDefaultRef = 1 } - _, err := stmt.Exec(l.RepoAt, l.Ref, isDefaultRef, l.Language, l.Bytes) + _, err := stmt.Exec(l.RepoDid, l.Ref, isDefaultRef, l.Language, l.Bytes) if err != nil { return err } @@ -105,10 +105,10 @@ _, err := e.Exec(query, args...) return err } -func UpdateRepoLanguages(tx *sql.Tx, repoAt syntax.ATURI, ref string, langs []models.RepoLanguage) error { +func UpdateRepoLanguages(tx *sql.Tx, repoDid syntax.DID, ref string, langs []models.RepoLanguage) error { err := DeleteRepoLanguages( tx, - orm.FilterEq("repo_at", repoAt), + orm.FilterEq("repo_did", repoDid), orm.FilterEq("ref", ref), ) if err != nil { diff --git a/appview/db/notifications.go b/appview/db/notifications.go --- a/appview/db/notifications.go +++ b/appview/db/notifications.go @@ -135,7 +135,7 @@ query := fmt.Sprintf(` select n.id, n.recipient_did, n.actor_did, n.type, n.entity_type, n.entity_id, n.read, n.created, n.repo_id, n.issue_id, n.pull_id, - r.id as r_id, r.did as r_did, r.name as r_name, r.description as r_description, r.website as r_website, r.topics as r_topics, + r.id as r_id, r.did as r_did, r.rkey as r_rkey, r.name as r_name, r.description as r_description, r.website as r_website, r.topics as r_topics, i.id as i_id, i.did as i_did, i.issue_id as i_issue_id, i.title as i_title, i.open as i_open, p.id as p_id, p.owner_did as p_owner_did, p.pull_id as p_pull_id, p.title as p_title, p.state as p_state from notifications n @@ -164,7 +164,7 @@ var repo models.Repo var issue models.Issue var pull models.Pull var rId, iId, pId sql.NullInt64 - var rDid, rName, rDescription, rWebsite, rTopicStr sql.NullString + var rDid, rRkey, rName, rDescription, rWebsite, rTopicStr sql.NullString var iDid sql.NullString var iIssueId sql.NullInt64 var iTitle sql.NullString @@ -177,7 +177,7 @@ err := rows.Scan( &n.ID, &n.RecipientDid, &n.ActorDid, &typeStr, &n.EntityType, &n.EntityId, &n.Read, &createdStr, &n.RepoId, &n.IssueId, &n.PullId, - &rId, &rDid, &rName, &rDescription, &rWebsite, &rTopicStr, + &rId, &rDid, &rRkey, &rName, &rDescription, &rWebsite, &rTopicStr, &iId, &iDid, &iIssueId, &iTitle, &iOpen, &pId, &pOwnerDid, &pPullId, &pTitle, &pState, ) @@ -198,6 +198,9 @@ if rId.Valid { repo.Id = rId.Int64 if rDid.Valid { repo.Did = rDid.String + } + if rRkey.Valid { + repo.Rkey = rRkey.String } if rName.Valid { repo.Name = rName.String diff --git a/appview/db/profile.go b/appview/db/profile.go --- a/appview/db/profile.go +++ b/appview/db/profile.go @@ -489,7 +489,7 @@ case models.VanityStatRepositoryCount: query = `select count(id) from repos where did = ?` args = append(args, did) case models.VanityStatStarCount: - query = `select count(id) from stars where subject_at like 'at://' || ? || '%'` + query = `select count(s.id) from stars s join repos r on s.subject = r.repo_did where s.subject_type = 'repo' and r.did = ?` args = append(args, did) case models.VanityStatNone: return 0, nil diff --git a/appview/db/pulls.go b/appview/db/pulls.go --- a/appview/db/pulls.go +++ b/appview/db/pulls.go @@ -30,13 +30,13 @@ } if existing.Branch != new.Branch { return false } - if existing.RepoAt == nil && new.RepoAt == nil { + if existing.RepoDid == nil && new.RepoDid == nil { return true } - if existing.RepoAt == nil || new.RepoAt == nil { + if existing.RepoDid == nil || new.RepoDid == nil { return false } - return *existing.RepoAt == *new.RepoAt + return *existing.RepoDid == *new.RepoDid } func compareSubmissions(existing, new []*models.PullSubmission) bool { @@ -60,9 +60,9 @@ func PutPull(tx *sql.Tx, pull *models.Pull) error { // ensure sequence exists _, err := tx.Exec(` - insert or ignore into repo_pull_seqs (repo_at, next_pull_id) + insert or ignore into repo_pull_seqs (repo_did, next_pull_id) values (?, 1) - `, pull.RepoAt) + `, pull.RepoDid) if err != nil { return err } @@ -94,7 +94,7 @@ if existingPull.Title == pull.Title && existingPull.Body == pull.Body && existingPull.TargetBranch == pull.TargetBranch && - existingPull.RepoAt == pull.RepoAt && + existingPull.RepoDid == pull.RepoDid && dependentOnEqual && pullSourceEqual && submissionsEqual { @@ -119,9 +119,9 @@ } func createNewPull(tx *sql.Tx, pull *models.Pull) error { _, err := tx.Exec(` - insert or ignore into repo_pull_seqs (repo_at, next_pull_id) + insert or ignore into repo_pull_seqs (repo_did, next_pull_id) values (?, 1) - `, pull.RepoAt) + `, pull.RepoDid) if err != nil { return err } @@ -130,9 +130,9 @@ var nextId int err = tx.QueryRow(` update repo_pull_seqs set next_pull_id = next_pull_id + 1 - where repo_at = ? + where repo_did = ? returning next_pull_id - 1 - `, pull.RepoAt).Scan(&nextId) + `, pull.RepoDid).Scan(&nextId) if err != nil { return err } @@ -140,19 +140,19 @@ pull.PullId = nextId pull.State = models.PullOpen - var sourceBranch, sourceRepoAt *string + var sourceBranch, sourceRepoDid *string if pull.PullSource != nil { sourceBranch = &pull.PullSource.Branch - if pull.PullSource.RepoAt != nil { - x := pull.PullSource.RepoAt.String() - sourceRepoAt = &x + if pull.PullSource.RepoDid != nil { + x := string(*pull.PullSource.RepoDid) + sourceRepoDid = &x } } result, err := tx.Exec( ` insert into pulls ( - repo_at, + repo_did, owner_did, pull_id, title, @@ -162,10 +162,10 @@ rkey, state, dependent_on, source_branch, - source_repo_at + source_repo_did ) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - pull.RepoAt, + pull.RepoDid, pull.OwnerDid, pull.PullId, pull.Title, @@ -175,7 +175,7 @@ pull.Rkey, pull.State, pull.DependentOn, sourceBranch, - sourceRepoAt, + sourceRepoDid, ) if err != nil { return err @@ -224,12 +224,12 @@ return nil } func updatePull(tx *sql.Tx, pull *models.Pull, existingPull *models.Pull) error { - var sourceBranch, sourceRepoAt *string + var sourceBranch, sourceRepoDid *string if pull.PullSource != nil { sourceBranch = &pull.PullSource.Branch - if pull.PullSource.RepoAt != nil { - x := pull.PullSource.RepoAt.String() - sourceRepoAt = &x + if pull.PullSource.RepoDid != nil { + x := string(*pull.PullSource.RepoDid) + sourceRepoDid = &x } } @@ -240,9 +240,9 @@ body = ?, target_branch = ?, dependent_on = ?, source_branch = ?, - source_repo_at = ? + source_repo_did = ? where owner_did = ? and rkey = ? - `, pull.Title, pull.Body, pull.TargetBranch, pull.DependentOn, sourceBranch, sourceRepoAt, pull.OwnerDid, pull.Rkey) + `, pull.Title, pull.Body, pull.TargetBranch, pull.DependentOn, sourceBranch, sourceRepoDid, pull.OwnerDid, pull.Rkey) if err != nil { return err } @@ -283,9 +283,9 @@ } return nil } -func NextPullId(e Execer, repoAt syntax.ATURI) (int, error) { +func NextPullId(e Execer, repoDid string) (int, error) { var pullId int - err := e.QueryRow(`select next_pull_id from repo_pull_seqs where repo_at = ?`, repoAt).Scan(&pullId) + err := e.QueryRow(`select next_pull_id from repo_pull_seqs where repo_did = ?`, repoDid).Scan(&pullId) return pullId - 1, err } @@ -316,7 +316,7 @@ query := fmt.Sprintf(` select id, owner_did, - repo_at, + repo_did, pull_id, created, title, @@ -325,7 +325,7 @@ target_branch, body, rkey, source_branch, - source_repo_at, + source_repo_did, dependent_on from pulls @@ -344,11 +344,11 @@ for rows.Next() { var pull models.Pull var createdAt string - var sourceBranch, sourceRepoAt, dependentOn sql.NullString + var sourceBranch, sourceRepoDid, dependentOn sql.NullString err := rows.Scan( &pull.ID, &pull.OwnerDid, - &pull.RepoAt, + &pull.RepoDid, &pull.PullId, &createdAt, &pull.Title, @@ -357,7 +357,7 @@ &pull.TargetBranch, &pull.Body, &pull.Rkey, &sourceBranch, - &sourceRepoAt, + &sourceRepoDid, &dependentOn, ) if err != nil { @@ -374,12 +374,12 @@ if sourceBranch.Valid { pull.PullSource = &models.PullSource{ Branch: sourceBranch.String, } - if sourceRepoAt.Valid { - sourceRepoAtParsed, err := syntax.ParseATURI(sourceRepoAt.String) + if sourceRepoDid.Valid { + sourceRepoDidParsed, err := syntax.ParseDID(sourceRepoDid.String) if err != nil { return nil, err } - pull.PullSource.RepoAt = &sourceRepoAtParsed + pull.PullSource.RepoDid = &sourceRepoDidParsed } } @@ -417,32 +417,31 @@ p.Labels = labels } } - // build up reverse mappings: p.Repo and p.PullSource - var repoAts []syntax.ATURI + // build up reverse mappings: p.Repo and p.PullSource.Repo + var repoDids []syntax.DID for _, p := range pulls { - repoAts = append(repoAts, p.RepoAt) - if p.PullSource != nil && p.PullSource.RepoAt != nil { - repoAts = append(repoAts, *p.PullSource.RepoAt) + repoDids = append(repoDids, p.RepoDid) + if p.PullSource != nil && p.PullSource.RepoDid != nil { + repoDids = append(repoDids, *p.PullSource.RepoDid) } } - repos, err := GetRepos(e, orm.FilterIn("at_uri", repoAts)) + repos, err := GetRepos(e, orm.FilterIn("repo_did", repoDids)) if err != nil && !errors.Is(err, sql.ErrNoRows) { - return nil, fmt.Errorf("failed to get source repos: %w", err) + return nil, fmt.Errorf("failed to get repos: %w", err) } - repoMap := make(map[syntax.ATURI]*models.Repo) + repoMap := make(map[syntax.DID]*models.Repo) for _, r := range repos { - repoMap[r.RepoAt()] = &r + repoMap[syntax.DID(r.RepoDid)] = &r } for _, p := range pulls { - if repo, ok := repoMap[p.RepoAt]; ok { + if repo, ok := repoMap[p.RepoDid]; ok { p.Repo = repo } - - if p.PullSource != nil && p.PullSource.RepoAt != nil { - if sourceRepo, ok := repoMap[*p.PullSource.RepoAt]; ok { + if p.PullSource != nil && p.PullSource.RepoDid != nil { + if sourceRepo, ok := repoMap[*p.PullSource.RepoDid]; ok { p.PullSource.Repo = sourceRepo } } @@ -625,7 +624,7 @@ select id, pull_id, submission_id, - repo_at, + repo_did, owner_did, comment_at, body, @@ -651,7 +650,7 @@ err := rows.Scan( &comment.ID, &comment.PullId, &comment.SubmissionId, - &comment.RepoAt, + &comment.RepoDid, &comment.OwnerDid, &comment.CommentAt, &comment.Body, @@ -705,7 +704,7 @@ rows, err := e.Query(` select p.owner_did, - p.repo_at, + p.repo_did, p.pull_id, p.created, p.title, @@ -718,7 +717,7 @@ r.created from pulls p join - repos r on p.repo_at = r.at_uri + repos r on p.repo_did = r.repo_did where p.owner_did = ? and p.created >= date ('now', ?) order by @@ -734,7 +733,7 @@ var repo models.Repo var pullCreatedAt, repoCreatedAt string err := rows.Scan( &pull.OwnerDid, - &pull.RepoAt, + &pull.RepoDid, &pull.PullId, &pullCreatedAt, &pull.Title, @@ -774,11 +773,11 @@ return pulls, nil } func NewPullComment(tx *sql.Tx, comment *models.PullComment) (int64, error) { - query := `insert into pull_comments (owner_did, repo_at, submission_id, comment_at, pull_id, body) values (?, ?, ?, ?, ?, ?)` + query := `insert into pull_comments (owner_did, repo_did, submission_id, comment_at, pull_id, body) values (?, ?, ?, ?, ?, ?)` res, err := tx.Exec( query, comment.OwnerDid, - comment.RepoAt, + comment.RepoDid, comment.SubmissionId, comment.CommentAt, comment.PullId, @@ -888,7 +887,7 @@ return err } -func GetPullCount(e Execer, repoAt syntax.ATURI) (models.PullCount, error) { +func GetPullCount(e Execer, repoDid string) (models.PullCount, error) { row := e.QueryRow(` select count(case when state = ? then 1 end) as open_count, @@ -896,12 +895,12 @@ count(case when state = ? then 1 end) as merged_count, count(case when state = ? then 1 end) as closed_count, count(case when state = ? then 1 end) as deleted_count from pulls - where repo_at = ?`, + where repo_did = ?`, models.PullOpen, models.PullMerged, models.PullClosed, models.PullAbandoned, - repoAt, + repoDid, ) var count models.PullCount diff --git a/appview/db/reference.go b/appview/db/reference.go --- a/appview/db/reference.go +++ b/appview/db/reference.go @@ -60,7 +60,7 @@ join repos r on r.did = inp.owner_did and r.name = inp.name join issues i - on i.repo_at = r.at_uri + on i.repo_did = r.repo_did and i.issue_id = inp.issue_id left join issue_comments c on inp.comment_id is not null @@ -131,11 +131,11 @@ join repos r on r.did = inp.owner_did and r.name = inp.name join pulls p - on p.repo_at = r.at_uri + on p.repo_did = r.repo_did and p.pull_id = inp.pull_id left join pull_comments c on inp.comment_id is not null - and c.repo_at = r.at_uri and c.pull_id = p.pull_id + and c.repo_did = p.repo_did and c.pull_id = p.pull_id and c.id = inp.comment_id `, strings.Join(vals, ","), @@ -319,7 +319,7 @@ fmt.Sprintf( `select r.did, r.name, i.issue_id, i.title, i.open from issues i join repos r - on r.at_uri = i.repo_at + on r.repo_did = i.repo_did where (i.did, i.rkey) in (%s)`, strings.Join(vals, ","), ), @@ -357,7 +357,7 @@ from issue_comments c join issues i on i.at_uri = c.issue_at join repos r - on r.at_uri = i.repo_at + on r.repo_did = i.repo_did where %s and %s`, filter.Condition(), exclude.Condition(), @@ -401,7 +401,7 @@ fmt.Sprintf( `select r.did, r.name, p.pull_id, p.title, p.state from pulls p join repos r - on r.at_uri = p.repo_at + on r.repo_did = p.repo_did where (p.owner_did, p.rkey) in (%s)`, strings.Join(vals, ","), ), @@ -437,9 +437,9 @@ fmt.Sprintf( `select r.did, r.name, p.pull_id, c.id, p.title, p.state from repos r join pulls p - on r.at_uri = p.repo_at + on r.repo_did = p.repo_did join pull_comments c - on r.at_uri = c.repo_at and p.pull_id = c.pull_id + on p.repo_did = c.repo_did and p.pull_id = c.pull_id where %s and %s`, filter.Condition(), exclude.Condition(), diff --git a/appview/db/repos.go b/appview/db/repos.go --- a/appview/db/repos.go +++ b/appview/db/repos.go @@ -17,6 +17,75 @@ "tangled.org/core/appview/pagination" "tangled.org/core/orm" ) +func RenameRepo(tx *sql.Tx, did, oldRkey, newRkey, newName string) error { + newAtURI := fmt.Sprintf("at://%s/sh.tangled.repo/%s", did, newRkey) + + res, err := tx.Exec( + `update repos set rkey = ?, name = ?, at_uri = ? where did = ? and rkey = ?`, + newRkey, newName, newAtURI, did, oldRkey, + ) + if err != nil { + return fmt.Errorf("update repos row: %w", err) + } + if n, _ := res.RowsAffected(); n == 0 { + return fmt.Errorf("no repo row found for did=%s rkey=%s", did, oldRkey) + } + + if _, err := tx.Exec( + `update pipelines set repo_name = ? where repo_owner = ? and repo_name = ?`, + newRkey, did, oldRkey, + ); err != nil { + return fmt.Errorf("rename pipelines.repo_name: %w", err) + } + + return nil +} + +func UpdateRepoDisplayName(e Execer, did, rkey, newName string) error { + _, err := e.Exec( + `update repos set name = ? where did = ? and rkey = ?`, + newName, did, rkey, + ) + return err +} + +func RecordRepoRename(e Execer, ownerDid, oldRkey, repoDid string) error { + _, err := e.Exec( + `insert into repo_renames (owner_did, old_rkey, repo_did) + values (?, ?, ?) + on conflict(owner_did, old_rkey) do update set + repo_did = excluded.repo_did, + renamed_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now')`, + ownerDid, oldRkey, repoDid, + ) + return err +} + +func DeleteRepoRename(e Execer, ownerDid, oldRkey string) error { + _, err := e.Exec( + `delete from repo_renames where owner_did = ? and old_rkey = ?`, + ownerDid, oldRkey, + ) + return err +} + +func LookupRepoRename(e Execer, ownerDid, oldRkey string) (*models.Repo, error) { + var repoDid string + err := e.QueryRow( + `select repo_did from repo_renames where owner_did = ? and old_rkey = ?`, + ownerDid, oldRkey, + ).Scan(&repoDid) + if err != nil { + return nil, err + } + + repo, err := GetRepoByDid(e, repoDid) + if err != nil { + return nil, err + } + return repo, nil +} + func GetRepos(e Execer, filters ...orm.Filter) ([]models.Repo, error) { return GetReposPaginated(e, pagination.Page{}, filters...) } @@ -66,7 +135,7 @@ return nil, err } defer rows.Close() - repoMap := make(map[syntax.ATURI]*models.Repo) + repoMap := make(map[string]*models.Repo) for rows.Next() { var repo models.Repo var createdAt string @@ -116,7 +185,7 @@ repo.RepoDid = repoDid.String } repo.RepoStats = &models.RepoStats{} - repoMap[repo.RepoAt()] = &repo + repoMap[repo.RepoDid] = &repo } if err = rows.Err(); err != nil { @@ -133,13 +202,13 @@ inClause := strings.TrimSuffix(strings.Repeat("?, ", len(repoMap)), ", ") args = make([]any, len(repoMap)) i := 0 for _, r := range repoMap { - args[i] = r.RepoAt() + args[i] = r.RepoDid i++ } // get labels for all repos labelsQuery := fmt.Sprintf( - `select repo_at, label_at from repo_labels where repo_at in (%s)`, + `select repo_did, label_at from repo_labels where repo_did in (%s)`, inClause, ) @@ -150,27 +219,27 @@ } defer rows.Close() for rows.Next() { - var repoat, labelat string - if err := rows.Scan(&repoat, &labelat); err != nil { + var repoDid, labelat string + if err := rows.Scan(&repoDid, &labelat); err != nil { continue } - if r, ok := repoMap[syntax.ATURI(repoat)]; ok { + if r, ok := repoMap[repoDid]; ok { r.Labels = append(r.Labels, labelat) } } // get primary language for all repos languageQuery := fmt.Sprintf(` - select repo_at, language + select repo_did, language from ( select - repo_at, language, + repo_did, language, row_number() over ( - partition by repo_at + partition by repo_did order by bytes desc ) as rn from repo_languages - where repo_at in (%s) + where repo_did in (%s) and is_default_ref = 1 and language <> '' ) @@ -184,12 +253,12 @@ } defer rows.Close() for rows.Next() { - var repoat, lang string - if err := rows.Scan(&repoat, &lang); err != nil { + var repoDid, lang string + if err := rows.Scan(&repoDid, &lang); err != nil { log.Println("err", "err", err) continue } - if r, ok := repoMap[syntax.ATURI(repoat)]; ok { + if r, ok := repoMap[repoDid]; ok { r.RepoStats.Language = lang } } @@ -199,7 +268,7 @@ } // get star counts starCountQuery := fmt.Sprintf( - `select subject_at, count(1) from stars where subject_at in (%s) group by subject_at`, + `select subject, count(1) from stars where subject_type = 'repo' and subject in (%s) group by subject`, inClause, ) @@ -210,13 +279,13 @@ } defer rows.Close() for rows.Next() { - var repoat string + var repoDid string var count int - if err := rows.Scan(&repoat, &count); err != nil { + if err := rows.Scan(&repoDid, &count); err != nil { log.Println("err", "err", err) continue } - if r, ok := repoMap[syntax.ATURI(repoat)]; ok { + if r, ok := repoMap[repoDid]; ok { r.RepoStats.StarCount = count } } @@ -227,12 +296,12 @@ // get issue counts issueCountQuery := fmt.Sprintf(` select - repo_at, + repo_did, count(case when open = 1 then 1 end) as open_count, count(case when open = 0 then 1 end) as closed_count from issues - where repo_at in (%s) - group by repo_at + where repo_did in (%s) + group by repo_did `, inClause) rows, err = e.Query(issueCountQuery, args...) @@ -242,13 +311,13 @@ } defer rows.Close() for rows.Next() { - var repoat string + var repoDid string var open, closed int - if err := rows.Scan(&repoat, &open, &closed); err != nil { + if err := rows.Scan(&repoDid, &open, &closed); err != nil { log.Println("err", "err", err) continue } - if r, ok := repoMap[syntax.ATURI(repoat)]; ok { + if r, ok := repoMap[repoDid]; ok { r.RepoStats.IssueCount.Open = open r.RepoStats.IssueCount.Closed = closed } @@ -260,14 +329,14 @@ // get pull counts pullCountQuery := fmt.Sprintf(` select - repo_at, + repo_did, count(case when state = ? then 1 end) as open_count, count(case when state = ? then 1 end) as merged_count, count(case when state = ? then 1 end) as closed_count, count(case when state = ? then 1 end) as deleted_count from pulls - where repo_at in (%s) - group by repo_at + where repo_did in (%s) + group by repo_did `, inClause) pullArgs := append([]any{ @@ -284,13 +353,13 @@ } defer rows.Close() for rows.Next() { - var repoat string + var repoDid string var open, merged, closed, deleted int - if err := rows.Scan(&repoat, &open, &merged, &closed, &deleted); err != nil { + if err := rows.Scan(&repoDid, &open, &merged, &closed, &deleted); err != nil { log.Println("err", "err", err) continue } - if r, ok := repoMap[syntax.ATURI(repoat)]; ok { + if r, ok := repoMap[repoDid]; ok { r.RepoStats.PullCount.Open = open r.RepoStats.PullCount.Merged = merged r.RepoStats.PullCount.Closed = closed @@ -301,6 +370,37 @@ if err = rows.Err(); err != nil { return nil, fmt.Errorf("failed to execute pulls-count query: %w", err) } + // get forks + forksInClause := strings.TrimSuffix(strings.Repeat("?, ", len(repoMap)), ", ") + forkArgs := make([]any, len(repoMap)) + + forksCountQuery := fmt.Sprintf( + `select source, count(1) from repos where source in (%s) group by source`, + forksInClause, + ) + + rows, err = e.Query(forksCountQuery, forkArgs...) + if err != nil { + return nil, fmt.Errorf("failed to execute fork-count query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var repodid string + var count int + if err := rows.Scan(&repodid, &count); err != nil { + log.Println("err", "err", err) + continue + } + + if r, ok := repoMap[repodid]; ok { + r.RepoStats.ForkCount = count + } + } + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("failed to execute fork-count query: %w", err) + } + var repos []models.Repo for _, r := range repoMap { repos = append(repos, *r) @@ -406,10 +506,10 @@ repoDid = &repo.RepoDid } _, err := tx.Exec( `update repos - set knot = ?, description = ?, website = ?, topics = ?, repo_did = coalesce(?, repo_did) + set name = ?, knot = ?, description = ?, website = ?, topics = ?, repo_did = coalesce(?, repo_did) where did = ? and rkey = ? `, - repo.Knot, repo.Description, repo.Website, repo.TopicStr(), repoDid, repo.Did, repo.Rkey, + repo.Name, repo.Knot, repo.Description, repo.Website, repo.TopicStr(), repoDid, repo.Did, repo.Rkey, ) return err } @@ -437,7 +537,7 @@ repo.Id = id for _, dl := range repo.Labels { if err := SubscribeLabel(tx, &models.RepoLabel{ - RepoAt: repo.RepoAt(), + RepoDid: syntax.DID(repo.RepoDid), LabelAt: syntax.ATURI(dl), }); err != nil { return fmt.Errorf("failed to subscribe to label: %w", err) @@ -447,22 +547,22 @@ return nil } -func RemoveRepo(e Execer, did, name string) error { - _, err := e.Exec(`delete from repos where did = ? and name = ?`, did, name) +func RemoveRepo(e Execer, did, rkey string) error { + _, err := e.Exec(`delete from repos where did = ? and rkey = ?`, did, rkey) return err } -func GetRepoSource(e Execer, repoAt syntax.ATURI) (string, error) { +func GetRepoSource(e Execer, repoDid string) (string, error) { var nullableSource sql.NullString - err := e.QueryRow(`select source from repos where at_uri = ?`, repoAt).Scan(&nullableSource) + err := e.QueryRow(`select source from repos where repo_did = ?`, repoDid).Scan(&nullableSource) if err != nil { return "", err } return nullableSource.String, nil } -func GetRepoSourceRepo(e Execer, repoAt syntax.ATURI) (*models.Repo, error) { - source, err := GetRepoSource(e, repoAt) +func GetRepoSourceRepo(e Execer, repoDid string) (*models.Repo, error) { + source, err := GetRepoSource(e, repoDid) if source == "" || errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -481,7 +581,7 @@ rows, err := e.Query( `select distinct r.id, r.did, r.name, r.knot, r.rkey, r.description, r.website, r.created, r.source, r.repo_did from repos r - left join collaborators c on r.at_uri = c.repo_at + left join collaborators c on r.repo_did = c.repo_did where (r.did = ? or c.subject_did = ?) and r.source is not null and r.source != '' @@ -537,7 +637,7 @@ return repos, nil } -func GetForkByDid(e Execer, did string, name string) (*models.Repo, error) { +func GetForkByDid(e Execer, did string, rkey string) (*models.Repo, error) { var repo models.Repo var createdAt string var nullableDescription sql.NullString @@ -549,8 +649,8 @@ row := e.QueryRow( `select id, did, name, knot, rkey, description, website, topics, created, source, repo_did from repos - where did = ? and name = ? and source is not null and source != ''`, - did, name, + where did = ? and rkey = ? and source is not null and source != ''`, + did, rkey, ) err := row.Scan(&repo.Id, &repo.Did, &repo.Name, &repo.Knot, &repo.Rkey, &nullableDescription, &nullableWebsite, &nullableTopicStr, &createdAt, &nullableSource, &nullableRepoDid) @@ -599,20 +699,21 @@ userDidCol string table string nsid syntax.NSID fkCol string + fkVal string } sources := []record{ - {"did", "repos", tangled.RepoNSID, "at_uri"}, - {"did", "issues", tangled.RepoIssueNSID, "repo_at"}, - {"owner_did", "pulls", tangled.RepoPullNSID, "repo_at"}, - {"did", "collaborators", tangled.RepoCollaboratorNSID, "repo_at"}, - {"did", "artifacts", tangled.RepoArchiveNSID, "repo_at"}, - {"did", "stars", tangled.FeedStarNSID, "subject_at"}, + {"did", "repos", tangled.RepoNSID, "at_uri", repoAtUri}, + {"did", "issues", tangled.RepoIssueNSID, "repo_did", repoDid}, + {"owner_did", "pulls", tangled.RepoPullNSID, "repo_did", repoDid}, + {"did", "collaborators", tangled.RepoCollaboratorNSID, "repo_did", repoDid}, + {"did", "artifacts", tangled.RepoArchiveNSID, "repo_did", repoDid}, + {"did", "stars", tangled.FeedStarNSID, "subject", repoDid}, } for _, src := range sources { rows, err := tx.Query( fmt.Sprintf(`SELECT %s, rkey FROM %s WHERE %s = ?`, src.userDidCol, src.table, src.fkCol), - repoAtUri, + src.fkVal, ) if err != nil { return fmt.Errorf("query %s for pds rewrites: %w", src.table, err) @@ -689,22 +790,22 @@ return nil } -func UpdateDescription(e Execer, repoAt, newDescription string) error { +func UpdateDescription(e Execer, repoDid, newDescription string) error { _, err := e.Exec( - `update repos set description = ? where at_uri = ?`, newDescription, repoAt) + `update repos set description = ? where repo_did = ?`, newDescription, repoDid) return err } -func UpdateSpindle(e Execer, repoAt string, spindle *string) error { +func UpdateSpindle(e Execer, repoDid string, spindle *string) error { _, err := e.Exec( - `update repos set spindle = ? where at_uri = ?`, spindle, repoAt) + `update repos set spindle = ? where repo_did = ?`, spindle, repoDid) return err } func SubscribeLabel(e Execer, rl *models.RepoLabel) error { - query := `insert or ignore into repo_labels (repo_at, label_at) values (?, ?)` + query := `insert or ignore into repo_labels (repo_did, label_at) values (?, ?)` - _, err := e.Exec(query, rl.RepoAt.String(), rl.LabelAt.String()) + _, err := e.Exec(query, string(rl.RepoDid), rl.LabelAt.String()) return err } @@ -739,7 +840,7 @@ if conditions != nil { whereClause = " where " + strings.Join(conditions, " and ") } - query := fmt.Sprintf(`select id, repo_at, label_at from repo_labels %s`, whereClause) + query := fmt.Sprintf(`select id, repo_did, label_at from repo_labels %s`, whereClause) rows, err := e.Query(query, args...) if err != nil { @@ -751,7 +852,7 @@ var labels []models.RepoLabel for rows.Next() { var label models.RepoLabel - err := rows.Scan(&label.Id, &label.RepoAt, &label.LabelAt) + err := rows.Scan(&label.Id, &label.RepoDid, &label.LabelAt) if err != nil { return nil, err } @@ -765,3 +866,13 @@ } return labels, nil } + +func GetForkCount(e Execer, sourceDID string) (int, error) { + forks := 0 + err := e.QueryRow( + `select count(source) from repos where source = ?`, sourceDID).Scan(&forks) + if err != nil { + return 0, err + } + return forks, nil +} diff --git a/appview/db/repos_rename_test.go b/appview/db/repos_rename_test.go new file mode 100644 --- /dev/null +++ b/appview/db/repos_rename_test.go @@ -0,0 +1,279 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "path/filepath" + "testing" + + "tangled.org/core/appview/models" + "tangled.org/core/orm" +) + +func newTestDB(t *testing.T) *DB { + t.Helper() + path := filepath.Join(t.TempDir(), "test.db") + d, err := Make(context.Background(), path) + if err != nil { + t.Fatalf("Make: %v", err) + } + t.Cleanup(func() { d.Close() }) + return d +} + +func seedRepo(t *testing.T, d *DB, did, knot, name, rkey, repoDid string) *models.Repo { + t.Helper() + tx, err := d.Begin() + if err != nil { + t.Fatalf("Begin: %v", err) + } + repo := &models.Repo{ + Did: did, + Name: name, + Knot: knot, + Rkey: rkey, + RepoDid: repoDid, + } + if err := AddRepo(tx, repo); err != nil { + t.Fatalf("AddRepo: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + return repo +} + +func TestRenameRepo_HappyPath(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "foo", "foo", "did:plc:repo1") + + tx, err := d.Begin() + if err != nil { + t.Fatalf("Begin: %v", err) + } + defer tx.Rollback() + + if err := RenameRepo(tx, "did:plc:akshay", "foo", "bar", "Bar"); err != nil { + t.Fatalf("RenameRepo: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + + got, err := GetRepoByDid(d, "did:plc:repo1") + if err != nil { + t.Fatalf("GetRepoByDid: %v", err) + } + if got.Rkey != "bar" { + t.Errorf("rkey = %q, want %q", got.Rkey, "bar") + } + if got.Name != "Bar" { + t.Errorf("name = %q, want %q", got.Name, "Bar") + } +} + +func TestUpdateRepoDisplayName_HappyPath(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "foo", "foo", "did:plc:repo1") + + if err := UpdateRepoDisplayName(d, "did:plc:akshay", "foo", "Foo"); err != nil { + t.Fatalf("UpdateRepoDisplayName: %v", err) + } + + got, err := GetRepoByDid(d, "did:plc:repo1") + if err != nil { + t.Fatalf("GetRepoByDid: %v", err) + } + if got.Name != "Foo" { + t.Errorf("name = %q, want %q", got.Name, "Foo") + } + if got.Rkey != "foo" { + t.Errorf("rkey should be unchanged but got %q, want %q", got.Rkey, "foo") + } +} + +func TestRecordAndLookupRepoRename(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "bar", "rkey1", "did:plc:repo1") + + if err := RecordRepoRename(d, "did:plc:akshay", "foo", "did:plc:repo1"); err != nil { + t.Fatalf("RecordRepoRename: %v", err) + } + + repo, err := LookupRepoRename(d, "did:plc:akshay", "foo") + if err != nil { + t.Fatalf("LookupRepoRename: %v", err) + } + if repo.RepoDid != "did:plc:repo1" { + t.Errorf("repoDid = %q, want %q", repo.RepoDid, "did:plc:repo1") + } + if repo.Name != "bar" { + t.Errorf("name = %q, want %q", repo.Name, "bar") + } +} + +func TestLookupRepoRename_MultipleOldNamesResolveToCurrent(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "baz", "baz", "did:plc:repo1") + + if err := RecordRepoRename(d, "did:plc:akshay", "foo", "did:plc:repo1"); err != nil { + t.Fatalf("record foo: %v", err) + } + if err := RecordRepoRename(d, "did:plc:akshay", "bar", "did:plc:repo1"); err != nil { + t.Fatalf("record bar: %v", err) + } + + for _, oldName := range []string{"foo", "bar"} { + repo, err := LookupRepoRename(d, "did:plc:akshay", oldName) + if err != nil { + t.Fatalf("lookup %q: %v", oldName, err) + } + if repo.Name != "baz" { + t.Errorf("lookup %q: name = %q, want %q", oldName, repo.Name, "baz") + } + } +} + +func TestRecordRepoRename_UpsertRefreshesTarget(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "current", "rkey1", "did:plc:repo1") + seedRepo(t, d, "did:plc:akshay", "knot.example", "other", "rkey2", "did:plc:repo2") + + if err := RecordRepoRename(d, "did:plc:akshay", "shared", "did:plc:repo1"); err != nil { + t.Fatalf("first record: %v", err) + } + if err := RecordRepoRename(d, "did:plc:akshay", "shared", "did:plc:repo2"); err != nil { + t.Fatalf("second record: %v", err) + } + + repo, err := LookupRepoRename(d, "did:plc:akshay", "shared") + if err != nil { + t.Fatalf("LookupRepoRename: %v", err) + } + if repo.RepoDid != "did:plc:repo2" { + t.Errorf("latest record should win: repoDid = %q, want %q", repo.RepoDid, "did:plc:repo2") + } +} + +func TestLookupRepoRename_StaleSelfHeal(t *testing.T) { + d := newTestDB(t) + + if err := RecordRepoRename(d, "did:plc:akshay", "foo", "did:plc:ghost"); err != nil { + t.Fatalf("RecordRepoRename: %v", err) + } + + _, err := LookupRepoRename(d, "did:plc:akshay", "foo") + if !errors.Is(err, sql.ErrNoRows) { + t.Errorf("target should be gone and fall through to 404: err = %v, want sql.ErrNoRows", err) + } +} + +func TestLookupRepoRename_NoRow(t *testing.T) { + d := newTestDB(t) + + _, err := LookupRepoRename(d, "did:plc:akshay", "nothing") + if !errors.Is(err, sql.ErrNoRows) { + t.Errorf("err = %v, want sql.ErrNoRows", err) + } +} + +func TestDuplicateRkeyUnderSameDID_Rejected(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "myrepo", "myrepo", "did:plc:repo1") + + tx, err := d.Begin() + if err != nil { + t.Fatalf("Begin: %v", err) + } + defer tx.Rollback() + + err = AddRepo(tx, &models.Repo{ + Did: "did:plc:akshay", + Name: "myrepo", + Knot: "knot.example", + Rkey: "myrepo", + RepoDid: "did:plc:repo2", + }) + if err == nil { + t.Fatal("expected unique violation for duplicate (did, rkey), got nil") + } + if !orm.IsUniqueViolation(err) { + t.Errorf("err = %v, want unique violation", err) + } +} + +func TestRenameRepo_OldRkeyRowGone(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "old", "old", "did:plc:repo1") + + tx, err := d.Begin() + if err != nil { + t.Fatalf("Begin: %v", err) + } + defer tx.Rollback() + + if err := RenameRepo(tx, "did:plc:akshay", "old", "new", "New"); err != nil { + t.Fatalf("RenameRepo: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + + got, err := GetRepoByDid(d, "did:plc:repo1") + if err != nil { + t.Fatalf("GetRepoByDid: %v", err) + } + if got.Rkey != "new" { + t.Errorf("rkey = %q, want %q", got.Rkey, "new") + } + + var dummy int + err = d.QueryRow(`select 1 from repos where did = ? and rkey = ?`, "did:plc:akshay", "old").Scan(&dummy) + if !errors.Is(err, sql.ErrNoRows) { + t.Errorf("old rkey row should be gone, got err = %v", err) + } +} + +func TestRenameRepo_PipelineRenamed(t *testing.T) { + d := newTestDB(t) + seedRepo(t, d, "did:plc:akshay", "knot.example", "old", "old", "did:plc:repo1") + + if _, err := d.Exec( + `insert into triggers (kind) values (?)`, "push", + ); err != nil { + t.Fatalf("seed trigger: %v", err) + } + if _, err := d.Exec( + `insert into pipelines (rkey, knot, repo_owner, repo_name, sha, trigger_id, repo_did) + values (?, ?, ?, ?, ?, ?, ?)`, + "pipe1", "knot.example", "did:plc:akshay", "old", + "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", 1, "did:plc:repo1", + ); err != nil { + t.Fatalf("seed pipeline: %v", err) + } + + tx, err := d.Begin() + if err != nil { + t.Fatalf("Begin: %v", err) + } + defer tx.Rollback() + + if err := RenameRepo(tx, "did:plc:akshay", "old", "new", "New"); err != nil { + t.Fatalf("RenameRepo: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + + var repoName string + if err := d.QueryRow( + `select repo_name from pipelines where repo_owner = ? and rkey = ?`, + "did:plc:akshay", "pipe1", + ).Scan(&repoName); err != nil { + t.Fatalf("query pipeline: %v", err) + } + if repoName != "new" { + t.Errorf("pipeline repo_name = %q, want %q", repoName, "new") + } +} diff --git a/appview/db/site_deploys.go b/appview/db/site_deploys.go --- a/appview/db/site_deploys.go +++ b/appview/db/site_deploys.go @@ -11,7 +11,7 @@ // AddSiteDeploy records a site deploy attempt. func AddSiteDeploy(e Execer, deploy *models.SiteDeploy) error { result, err := e.Exec(` insert into site_deploys ( - repo_at, + repo_did, branch, dir, commit_sha, @@ -20,7 +20,7 @@ trigger, error ) values (?, ?, ?, ?, ?, ?, ?) `, - deploy.RepoAt, + deploy.RepoDid, deploy.Branch, deploy.Dir, deploy.CommitSHA, @@ -42,7 +42,7 @@ return nil } // GetSiteDeploys returns recent deploy records for a repository, newest first. -func GetSiteDeploys(e Execer, repoAt string, limit int) ([]models.SiteDeploy, error) { +func GetSiteDeploys(e Execer, repoDid string, limit int) ([]models.SiteDeploy, error) { if limit <= 0 { limit = 20 } @@ -50,7 +50,7 @@ rows, err := e.Query(` select id, - repo_at, + repo_did, branch, dir, commit_sha, @@ -59,10 +59,10 @@ trigger, error, created_at from site_deploys - where repo_at = ? + where repo_did = ? order by created_at desc limit ? - `, repoAt, limit) + `, repoDid, limit) if err != nil { return nil, fmt.Errorf("failed to query site deploys: %w", err) } @@ -75,7 +75,7 @@ var createdAt string if err := rows.Scan( &d.Id, - &d.RepoAt, + &d.RepoDid, &d.Branch, &d.Dir, &d.CommitSHA, diff --git a/appview/db/sites.go b/appview/db/sites.go --- a/appview/db/sites.go +++ b/appview/db/sites.go @@ -138,18 +138,18 @@ return nil } // GetRepoSiteConfig returns the site configuration for a repo, or nil if not configured. -func GetRepoSiteConfig(e Execer, repoAt string) (*models.RepoSite, error) { +func GetRepoSiteConfig(e Execer, repoDid string) (*models.RepoSite, error) { row := e.QueryRow(` - select id, repo_at, branch, dir, is_index, created, updated + select id, repo_did, branch, dir, is_index, created, updated from repo_sites - where repo_at = ? - `, repoAt) + where repo_did = ? + `, repoDid) var s models.RepoSite var isIndex int var createdStr, updatedStr string - err := row.Scan(&s.ID, &s.RepoAt, &s.Branch, &s.Dir, &isIndex, &createdStr, &updatedStr) + err := row.Scan(&s.ID, &s.RepoDid, &s.Branch, &s.Dir, &isIndex, &createdStr, &updatedStr) if errors.Is(err, sql.ErrNoRows) { return nil, nil } @@ -173,37 +173,37 @@ return &s, nil } // SetRepoSiteConfig inserts or replaces the site configuration for a repo. -func SetRepoSiteConfig(e Execer, repoAt, branch, dir string, isIndex bool) error { +func SetRepoSiteConfig(e Execer, repoDid, branch, dir string, isIndex bool) error { isIndexInt := 0 if isIndex { isIndexInt = 1 } _, err := e.Exec(` - insert into repo_sites (repo_at, branch, dir, is_index, updated) + insert into repo_sites (repo_did, branch, dir, is_index, updated) values (?, ?, ?, ?, strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) - on conflict(repo_at) do update set + on conflict(repo_did) do update set branch = excluded.branch, dir = excluded.dir, is_index = excluded.is_index, updated = excluded.updated - `, repoAt, branch, dir, isIndexInt) + `, repoDid, branch, dir, isIndexInt) return err } // DeleteRepoSiteConfig removes the site configuration for a repo. -func DeleteRepoSiteConfig(e Execer, repoAt string) error { - _, err := e.Exec(`delete from repo_sites where repo_at = ?`, repoAt) +func DeleteRepoSiteConfig(e Execer, repoDid string) error { + _, err := e.Exec(`delete from repo_sites where repo_did = ?`, repoDid) return err } // GetRepoSiteConfigsForDid returns all site configurations for repos owned by a DID. -// RepoName is populated on each returned RepoSite. +// RepoRkey is populated on each returned RepoSite. func GetRepoSiteConfigsForDid(e Execer, did string) ([]*models.RepoSite, error) { rows, err := e.Query(` - select rs.id, rs.repo_at, r.name, rs.branch, rs.dir, rs.is_index, rs.created, rs.updated + select rs.id, rs.repo_did, r.rkey, rs.branch, rs.dir, rs.is_index, rs.created, rs.updated from repo_sites rs - join repos r on r.at_uri = rs.repo_at + join repos r on r.repo_did = rs.repo_did where r.did = ? `, did) if err != nil { @@ -216,7 +216,7 @@ for rows.Next() { var s models.RepoSite var isIndex int var createdStr, updatedStr string - if err := rows.Scan(&s.ID, &s.RepoAt, &s.RepoName, &s.Branch, &s.Dir, &isIndex, &createdStr, &updatedStr); err != nil { + if err := rows.Scan(&s.ID, &s.RepoDid, &s.RepoRkey, &s.Branch, &s.Dir, &isIndex, &createdStr, &updatedStr); err != nil { return nil, err } s.IsIndex = isIndex != 0 @@ -237,34 +237,34 @@ // DeleteRepoSiteConfigsForDid removes all site configurations for repos owned by a DID. func DeleteRepoSiteConfigsForDid(e Execer, did string) error { _, err := e.Exec(` delete from repo_sites - where repo_at in ( - select at_uri from repos where did = ? + where repo_did in ( + select repo_did from repos where did = ? ) `, did) return err } -// GetIndexRepoAtForDid returns the repo_at of the repo that currently holds -// is_index=1 for the given DID, excluding excludeRepoAt (the current repo). +// GetIndexRepoDidForDid returns the repo_did of the repo that currently holds +// is_index=1 for the given DID, excluding excludeRepoDid (the current repo). // Returns "", nil if no other repo is the index site. -func GetIndexRepoAtForDid(e Execer, did, excludeRepoAt string) (string, error) { +func GetIndexRepoDidForDid(e Execer, did, excludeRepoDid string) (string, error) { row := e.QueryRow(` - select rs.repo_at + select rs.repo_did from repo_sites rs - join repos r on r.at_uri = rs.repo_at + join repos r on r.repo_did = rs.repo_did where r.did = ? and rs.is_index = 1 - and rs.repo_at != ? + and rs.repo_did != ? limit 1 - `, did, excludeRepoAt) + `, did, excludeRepoDid) - var repoAt string - err := row.Scan(&repoAt) + var repoDid string + err := row.Scan(&repoDid) if errors.Is(err, sql.ErrNoRows) { return "", nil } if err != nil { return "", err } - return repoAt, nil + return repoDid, nil } diff --git a/appview/db/star.go b/appview/db/star.go --- a/appview/db/star.go +++ b/appview/db/star.go @@ -1,42 +1,40 @@ package db import ( - "database/sql" - "errors" "fmt" "log" "slices" "strings" "time" - "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/appview/models" "tangled.org/core/appview/pagination" "tangled.org/core/orm" ) func AddStar(e Execer, star *models.Star) error { - query := `insert or ignore into stars (did, subject_at, rkey) values (?, ?, ?)` + query := `insert or ignore into stars (did, subject_type, subject, rkey) values (?, ?, ?, ?)` _, err := e.Exec( query, star.Did, - star.RepoAt.String(), + string(star.SubjectType), + star.Subject, star.Rkey, ) return err } // Get a star record -func GetStar(e Execer, did string, subjectAt syntax.ATURI) (*models.Star, error) { +func GetStar(e Execer, did string, subject string) (*models.Star, error) { query := ` - select did, subject_at, created, rkey + select did, subject_type, subject, created, rkey from stars - where did = ? and subject_at = ?` - row := e.QueryRow(query, did, subjectAt) + where did = ? and subject = ?` + row := e.QueryRow(query, did, subject) var star models.Star var created string - err := row.Scan(&star.Did, &star.RepoAt, &created, &star.Rkey) + err := row.Scan(&star.Did, &star.SubjectType, &star.Subject, &created, &star.Rkey) if err != nil { return nil, err } @@ -52,15 +50,15 @@ return &star, nil } -func GetStars(e Execer, subjectAt syntax.ATURI, page pagination.Page) ([]models.Star, error) { +func GetStars(e Execer, subject string, page pagination.Page) ([]models.Star, error) { query := ` - select did, subject_at, created, rkey + select did, subject_type, subject, created, rkey from stars - where subject_at = ? + where subject = ? order by created desc limit ? offset ? ` - rows, err := e.Query(query, subjectAt, page.Limit, page.Offset) + rows, err := e.Query(query, subject, page.Limit, page.Offset) if err != nil { return nil, err } @@ -70,7 +68,7 @@ var stars []models.Star for rows.Next() { var star models.Star var created string - if err := rows.Scan(&star.Did, &star.RepoAt, &created, &star.Rkey); err != nil { + if err := rows.Scan(&star.Did, &star.SubjectType, &star.Subject, &created, &star.Rkey); err != nil { return nil, err } @@ -85,8 +83,8 @@ return stars, rows.Err() } // Remove a star -func DeleteStar(e Execer, did string, subjectAt syntax.ATURI) error { - _, err := e.Exec(`delete from stars where did = ? and subject_at = ?`, did, subjectAt) +func DeleteStar(e Execer, did string, subject string) error { + _, err := e.Exec(`delete from stars where did = ? and subject = ?`, did, subject) return err } @@ -96,36 +94,38 @@ _, err := e.Exec(`delete from stars where did = ? and rkey = ?`, did, rkey) return err } -func GetStarCount(e Execer, subjectAt syntax.ATURI) (int, error) { +func GetStarCount(e Execer, subjectType models.StarSubjectType, subject string) (int, error) { stars := 0 err := e.QueryRow( - `select count(did) from stars where subject_at = ?`, subjectAt).Scan(&stars) + `select count(did) from stars where subject_type = ? and subject = ?`, + string(subjectType), subject, + ).Scan(&stars) if err != nil { return 0, err } return stars, nil } -// getStarStatuses returns a map of repo URIs to star status for a given user +// getStarStatuses returns a map of subjects to star status for a given user // This is an internal helper function to avoid N+1 queries -func getStarStatuses(e Execer, userDid string, repoAts []syntax.ATURI) (map[string]bool, error) { - if len(repoAts) == 0 || userDid == "" { +func getStarStatuses(e Execer, userDid string, subjects []string) (map[string]bool, error) { + if len(subjects) == 0 || userDid == "" { return make(map[string]bool), nil } - placeholders := make([]string, len(repoAts)) - args := make([]any, len(repoAts)+1) + placeholders := make([]string, len(subjects)) + args := make([]any, len(subjects)+1) args[0] = userDid - for i, repoAt := range repoAts { + for i, subj := range subjects { placeholders[i] = "?" - args[i+1] = repoAt.String() + args[i+1] = subj } query := fmt.Sprintf(` - SELECT subject_at + SELECT subject FROM stars - WHERE did = ? AND subject_at IN (%s) + WHERE did = ? AND subject IN (%s) `, strings.Join(placeholders, ",")) rows, err := e.Query(query, args...) @@ -135,34 +135,34 @@ } defer rows.Close() result := make(map[string]bool) - // Initialize all repos as not starred - for _, repoAt := range repoAts { - result[repoAt.String()] = false + // Initialize all subjects as not starred + for _, subj := range subjects { + result[subj] = false } - // Mark starred repos as true + // Mark starred subjects as true for rows.Next() { - var repoAt string - if err := rows.Scan(&repoAt); err != nil { + var subj string + if err := rows.Scan(&subj); err != nil { return nil, err } - result[repoAt] = true + result[subj] = true } return result, nil } -func GetStarStatus(e Execer, userDid string, subjectAt syntax.ATURI) bool { - statuses, err := getStarStatuses(e, userDid, []syntax.ATURI{subjectAt}) +func GetStarStatus(e Execer, userDid string, subject string) bool { + statuses, err := getStarStatuses(e, userDid, []string{subject}) if err != nil { return false } - return statuses[subjectAt.String()] + return statuses[subject] } -// GetStarStatuses returns a map of repo URIs to star status for a given user -func GetStarStatuses(e Execer, userDid string, subjectAts []syntax.ATURI) (map[string]bool, error) { - return getStarStatuses(e, userDid, subjectAts) +// GetStarStatuses returns a map of subjects to star status for a given user +func GetStarStatuses(e Execer, userDid string, subjects []string) (map[string]bool, error) { + return getStarStatuses(e, userDid, subjects) } // GetRepoStars return a list of stars each holding target repository. @@ -175,10 +175,9 @@ conditions = append(conditions, filter.Condition()) args = append(args, filter.Arg()...) } - whereClause := "" - if conditions != nil { - whereClause = " where " + strings.Join(conditions, " and ") - } + conditions = append(conditions, "subject_type = 'repo'") + + whereClause := " where " + strings.Join(conditions, " and ") pageClause := "" if page.Limit != 0 { @@ -186,7 +185,7 @@ pageClause = fmt.Sprintf(" limit %d offset %d", page.Limit, page.Offset) } repoQuery := fmt.Sprintf( - `select did, subject_at, created, rkey + `select did, subject_type, subject, created, rkey from stars %s order by created desc @@ -204,7 +203,7 @@ starMap := make(map[string][]models.Star) for rows.Next() { var star models.Star var created string - err := rows.Scan(&star.Did, &star.RepoAt, &created, &star.Rkey) + err := rows.Scan(&star.Did, &star.SubjectType, &star.Subject, &created, &star.Rkey) if err != nil { return nil, err } @@ -214,8 +213,7 @@ if t, err := time.Parse(time.RFC3339, created); err == nil { star.Created = t } - repoAt := string(star.RepoAt) - starMap[repoAt] = append(starMap[repoAt], star) + starMap[star.Subject] = append(starMap[star.Subject], star) } // populate *Repo in each star @@ -230,14 +228,14 @@ if len(args) == 0 { return nil, nil } - repos, err := GetRepos(e, orm.FilterIn("at_uri", args)) + repos, err := GetRepos(e, orm.FilterIn("repo_did", args)) if err != nil { return nil, err } var repoStars []models.RepoStar for _, r := range repos { - if stars, ok := starMap[string(r.RepoAt())]; ok { + if stars, ok := starMap[r.RepoDid]; ok { for _, star := range stars { repoStars = append(repoStars, models.RepoStar{ Star: star, @@ -275,9 +273,7 @@ } repoQuery := fmt.Sprintf(`select count(1) from stars %s`, whereClause) var count int64 - err := e.QueryRow(repoQuery, args...).Scan(&count) - - if !errors.Is(err, sql.ErrNoRows) && err != nil { + if err := e.QueryRow(repoQuery, args...).Scan(&count); err != nil { return 0, err } @@ -286,23 +282,25 @@ } // GetTopStarredReposLastWeek returns the top 8 most starred repositories from the last week func GetTopStarredReposLastWeek(e Execer) ([]models.Repo, error) { - // first, get the top repo URIs by star count from the last week + // first, get the top repo DIDs by star count from the last week query := ` with recent_starred_repos as ( - select distinct subject_at + select distinct subject from stars where created >= datetime('now', '-7 days') + and subject_type = 'repo' ), repo_star_counts as ( select - s.subject_at, + s.subject, count(*) as stars_gained_last_week from stars s - join recent_starred_repos rsr on s.subject_at = rsr.subject_at + join recent_starred_repos rsr on s.subject = rsr.subject where s.created >= datetime('now', '-7 days') - group by s.subject_at + and s.subject_type = 'repo' + group by s.subject ) - select rsc.subject_at + select rsc.subject from repo_star_counts rsc order by rsc.stars_gained_last_week desc limit 5 @@ -314,26 +312,26 @@ return nil, err } defer rows.Close() - var repoUris []string + var repoDids []string for rows.Next() { - var repoUri string - err := rows.Scan(&repoUri) + var repoDid string + err := rows.Scan(&repoDid) if err != nil { return nil, err } - repoUris = append(repoUris, repoUri) + repoDids = append(repoDids, repoDid) } if err := rows.Err(); err != nil { return nil, err } - if len(repoUris) == 0 { + if len(repoDids) == 0 { return []models.Repo{}, nil } // get full repo data - repos, err := GetRepos(e, orm.FilterIn("at_uri", repoUris)) + repos, err := GetRepos(e, orm.FilterIn("repo_did", repoDids)) if err != nil { return nil, err } @@ -341,12 +339,12 @@ // sort repos by the original trending order repoMap := make(map[string]models.Repo) for _, repo := range repos { - repoMap[repo.RepoAt().String()] = repo + repoMap[repo.RepoDid] = repo } - orderedRepos := make([]models.Repo, 0, len(repoUris)) - for _, uri := range repoUris { - if repo, exists := repoMap[uri]; exists { + orderedRepos := make([]models.Repo, 0, len(repoDids)) + for _, did := range repoDids { + if repo, exists := repoMap[did]; exists { orderedRepos = append(orderedRepos, repo) } } diff --git a/appview/db/timeline.go b/appview/db/timeline.go --- a/appview/db/timeline.go +++ b/appview/db/timeline.go @@ -3,7 +3,6 @@ import ( "sort" - "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/appview/models" "tangled.org/core/appview/pagination" "tangled.org/core/orm" @@ -101,18 +100,18 @@ if loggedInUserDid == "" { return nil, nil } - var repoAts []syntax.ATURI + var repoDids []string for _, r := range repos { - repoAts = append(repoAts, r.RepoAt()) + repoDids = append(repoDids, r.RepoDid) } - return GetStarStatuses(e, loggedInUserDid, repoAts) + return GetStarStatuses(e, loggedInUserDid, repoDids) } func getRepoStarInfo(repo *models.Repo, starStatuses map[string]bool) (bool, int64) { var isStarred bool if starStatuses != nil { - isStarred = starStatuses[repo.RepoAt().String()] + isStarred = starStatuses[repo.RepoDid] } var starCount int64 diff --git a/appview/db/vouch.go b/appview/db/vouch.go --- a/appview/db/vouch.go +++ b/appview/db/vouch.go @@ -373,7 +373,7 @@ select p.owner_did as did, 3 as priority, p.created, 'This user opened a pull request on your repository' as reason from pulls p - join repos r on r.at_uri = p.repo_at + join repos r on r.repo_did = p.repo_did where r.did = ? and p.owner_did != ? @@ -382,7 +382,7 @@ select i.did as did, 4 as priority, i.created, 'This user opened an issue on your repository' as reason from issues i - join repos r on r.at_uri = i.repo_at + join repos r on r.repo_did = i.repo_did where r.did = ? and i.did != ? @@ -391,7 +391,7 @@ select pc.owner_did as did, 5 as priority, pc.created, 'This user commented on a pull request on your repository' as reason from pull_comments pc - join repos r on r.at_uri = pc.repo_at + join repos r on r.repo_did = pc.repo_did where r.did = ? and pc.owner_did != ? @@ -401,7 +401,7 @@ select ic.did as did, 6 as priority, ic.created, 'This user commented on an issue on your repository' as reason from issue_comments ic join issues i on i.at_uri = ic.issue_at - join repos r on r.at_uri = i.repo_at + join repos r on r.repo_did = i.repo_did where r.did = ? and ic.did != ? @@ -418,7 +418,7 @@ select r.did as did, 8 as priority, s.created, 'You recently starred a repository by this user' as reason from stars s - join repos r on r.at_uri = s.subject_at + join repos r on r.at_uri = s.subject where s.did = ? and r.did != ? ) diff --git a/appview/db/webhooks.go b/appview/db/webhooks.go --- a/appview/db/webhooks.go +++ b/appview/db/webhooks.go @@ -6,7 +6,6 @@ "fmt" "strings" "time" - "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/appview/models" "tangled.org/core/orm" ) @@ -28,7 +27,7 @@ query := fmt.Sprintf(` select id, - repo_at, + repo_did, url, secret, active, @@ -55,7 +54,7 @@ var active int err := rows.Scan( &wh.Id, - &wh.RepoAt, + &wh.RepoDid, &wh.Url, &secret, &active, @@ -119,9 +118,9 @@ active = 1 } result, err := e.Exec(` - insert into webhooks (repo_at, url, secret, active, events) + insert into webhooks (repo_did, url, secret, active, events) values (?, ?, ?, ?, ?) - `, webhook.RepoAt.String(), webhook.Url, webhook.Secret, active, eventsStr) + `, string(webhook.RepoDid), webhook.Url, webhook.Secret, active, eventsStr) if err != nil { return fmt.Errorf("failed to insert webhook: %w", err) @@ -285,14 +284,14 @@ return deliveries, nil } // GetWebhooksForRepo is a convenience function to get all webhooks for a repository -func GetWebhooksForRepo(e Execer, repoAt syntax.ATURI) ([]models.Webhook, error) { - return GetWebhooks(e, orm.FilterEq("repo_at", repoAt.String())) +func GetWebhooksForRepo(e Execer, repoDid string) ([]models.Webhook, error) { + return GetWebhooks(e, orm.FilterEq("repo_did", repoDid)) } // GetActiveWebhooksForRepo returns only active webhooks for a repository -func GetActiveWebhooksForRepo(e Execer, repoAt syntax.ATURI) ([]models.Webhook, error) { +func GetActiveWebhooksForRepo(e Execer, repoDid string) ([]models.Webhook, error) { return GetWebhooks(e, - orm.FilterEq("repo_at", repoAt.String()), + orm.FilterEq("repo_did", repoDid), orm.FilterEq("active", 1), ) } diff --git a/appview/indexer/issues/indexer.go b/appview/indexer/issues/indexer.go --- a/appview/indexer/issues/indexer.go +++ b/appview/indexer/issues/indexer.go @@ -31,7 +31,7 @@ unicodeNormalizeName = "uicodeNormalize" // Bump this when the index mapping changes to trigger a rebuild. - issueIndexerVersion = 3 + issueIndexerVersion = 4 ) type Indexer struct { @@ -85,7 +85,7 @@ docMapping.AddFieldMappingsAt("title", textFieldMapping) docMapping.AddFieldMappingsAt("body", textFieldMapping) - docMapping.AddFieldMappingsAt("repo_at", keywordFieldMapping) + docMapping.AddFieldMappingsAt("repo_did", keywordFieldMapping) docMapping.AddFieldMappingsAt("is_open", boolFieldMapping) docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping) docMapping.AddFieldMappingsAt("labels", keywordFieldMapping) @@ -185,7 +185,7 @@ } type issueData struct { ID int64 `json:"id"` - RepoAt string `json:"repo_at"` + RepoDid string `json:"repo_did"` IssueID int `json:"issue_id"` Title string `json:"title"` Body string `json:"body"` @@ -200,7 +200,7 @@ func makeIssueData(issue *models.Issue) *issueData { return &issueData{ ID: issue.Id, - RepoAt: issue.RepoAt.String(), + RepoDid: string(issue.RepoDid), IssueID: issue.IssueId, Title: issue.Title, Body: issue.Body, @@ -274,7 +274,7 @@ bleveutil.MatchPhraseQuery("body", phrase, issueIndexerAnalyzer), )) } - musts = append(musts, bleveutil.KeywordFieldQuery("repo_at", opts.RepoAt)) + musts = append(musts, bleveutil.KeywordFieldQuery("repo_did", opts.RepoDid)) if opts.IsOpen != nil { musts = append(musts, bleveutil.BoolFieldQuery("is_open", *opts.IsOpen)) } diff --git a/appview/indexer/issues/indexer_test.go b/appview/indexer/issues/indexer_test.go --- a/appview/indexer/issues/indexer_test.go +++ b/appview/indexer/issues/indexer_test.go @@ -67,17 +67,17 @@ ctx := context.Background() err := ix.Index(ctx, - models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Fix login bug", Body: "Users cannot login", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug")}, - models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Add dark mode", Body: "Implement dark theme", Open: true, Did: "did:plc:bob", Labels: makeLabelState("feature")}, - models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Fix login timeout", Body: "Login takes too long", Open: false, Did: "did:plc:alice", Labels: makeLabelState("bug")}, + models.Issue{Id: 1, RepoDid: "did:plc:testrepo", Title: "Fix login bug", Body: "Users cannot login", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug")}, + models.Issue{Id: 2, RepoDid: "did:plc:testrepo", Title: "Add dark mode", Body: "Implement dark theme", Open: true, Did: "did:plc:bob", Labels: makeLabelState("feature")}, + models.Issue{Id: 3, RepoDid: "did:plc:testrepo", Title: "Fix login timeout", Body: "Login takes too long", Open: false, Did: "did:plc:alice", Labels: makeLabelState("bug")}, ) require.NoError(t, err) opts := func() models.IssueSearchOptions { return models.IssueSearchOptions{ - RepoAt: "at://did:plc:test/sh.tangled.repo/abc", - IsOpen: boolPtr(true), - Page: pagination.Page{Limit: 10}, + RepoDid: "did:plc:testrepo", + IsOpen: boolPtr(true), + Page: pagination.Page{Limit: 10}, } } @@ -148,16 +148,16 @@ ctx := context.Background() err := ix.Index(ctx, - models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Issue 1", Body: "Body", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug")}, - models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Issue 2", Body: "Body", Open: true, Did: "did:plc:bob", Labels: makeLabelState("bug", "urgent")}, + models.Issue{Id: 1, RepoDid: "did:plc:testrepo", Title: "Issue 1", Body: "Body", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug")}, + models.Issue{Id: 2, RepoDid: "did:plc:testrepo", Title: "Issue 2", Body: "Body", Open: true, Did: "did:plc:bob", Labels: makeLabelState("bug", "urgent")}, ) require.NoError(t, err) result, err := ix.Search(ctx, models.IssueSearchOptions{ - RepoAt: "at://did:plc:test/sh.tangled.repo/abc", - IsOpen: boolPtr(true), - Labels: []string{"bug", "urgent"}, - Page: pagination.Page{Limit: 10}, + RepoDid: "did:plc:testrepo", + IsOpen: boolPtr(true), + Labels: []string{"bug", "urgent"}, + Page: pagination.Page{Limit: 10}, }) require.NoError(t, err) assert.Equal(t, uint64(1), result.Total) @@ -171,17 +171,17 @@ ctx := context.Background() err := ix.Index(ctx, - models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Fix login bug", Body: "Users cannot login", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug")}, - models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Add dark mode", Body: "Implement dark theme", Open: true, Did: "did:plc:bob", Labels: makeLabelState("feature")}, - models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Fix timeout bug", Body: "Timeout on save", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug", "urgent")}, + models.Issue{Id: 1, RepoDid: "did:plc:testrepo", Title: "Fix login bug", Body: "Users cannot login", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug")}, + models.Issue{Id: 2, RepoDid: "did:plc:testrepo", Title: "Add dark mode", Body: "Implement dark theme", Open: true, Did: "did:plc:bob", Labels: makeLabelState("feature")}, + models.Issue{Id: 3, RepoDid: "did:plc:testrepo", Title: "Fix timeout bug", Body: "Timeout on save", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug", "urgent")}, ) require.NoError(t, err) opts := func() models.IssueSearchOptions { return models.IssueSearchOptions{ - RepoAt: "at://did:plc:test/sh.tangled.repo/abc", - IsOpen: boolPtr(true), - Page: pagination.Page{Limit: 10}, + RepoDid: "did:plc:testrepo", + IsOpen: boolPtr(true), + Page: pagination.Page{Limit: 10}, } } @@ -227,9 +227,9 @@ ctx := context.Background() err := ix.Index(ctx, - models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Fix login bug", Body: "Users cannot login", Open: true, Did: "did:plc:alice"}, - models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Add dark mode", Body: "Implement dark theme", Open: true, Did: "did:plc:bob"}, - models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Fix timeout bug", Body: "Timeout on save", Open: true, Did: "did:plc:alice"}, + models.Issue{Id: 1, RepoDid: "did:plc:testrepo", Title: "Fix login bug", Body: "Users cannot login", Open: true, Did: "did:plc:alice"}, + models.Issue{Id: 2, RepoDid: "did:plc:testrepo", Title: "Add dark mode", Body: "Implement dark theme", Open: true, Did: "did:plc:bob"}, + models.Issue{Id: 3, RepoDid: "did:plc:testrepo", Title: "Fix timeout bug", Body: "Timeout on save", Open: true, Did: "did:plc:alice"}, ) require.NoError(t, err) @@ -244,7 +244,7 @@ } require.Equal(t, []string{"dark theme"}, negatedPhrases) result, err := ix.Search(ctx, models.IssueSearchOptions{ - RepoAt: "at://did:plc:test/sh.tangled.repo/abc", + RepoDid: "did:plc:testrepo", IsOpen: boolPtr(true), NegatedPhrases: negatedPhrases, Page: pagination.Page{Limit: 10}, @@ -261,13 +261,13 @@ ctx := context.Background() err := ix.Index(ctx, - models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Issue", Body: "Body", Open: true, Did: "did:plc:alice"}, + models.Issue{Id: 1, RepoDid: "did:plc:testrepo", Title: "Issue", Body: "Body", Open: true, Did: "did:plc:alice"}, ) require.NoError(t, err) result, err := ix.Search(ctx, models.IssueSearchOptions{ Keywords: []string{"nonexistent"}, - RepoAt: "at://did:plc:test/sh.tangled.repo/abc", + RepoDid: "did:plc:testrepo", IsOpen: boolPtr(true), Page: pagination.Page{Limit: 10}, }) @@ -283,20 +283,20 @@ ctx := context.Background() err := ix.Index(ctx, - models.Issue{Id: 1, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority bug", Body: "Urgent", Open: true, Did: "did:plc:alice", + models.Issue{Id: 1, RepoDid: "did:plc:testrepo", Title: "High priority bug", Body: "Urgent", Open: true, Did: "did:plc:alice", Labels: makeLabelState("bug", "priority=high")}, - models.Issue{Id: 2, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "Low priority feature", Body: "Nice to have", Open: true, Did: "did:plc:bob", + models.Issue{Id: 2, RepoDid: "did:plc:testrepo", Title: "Low priority feature", Body: "Nice to have", Open: true, Did: "did:plc:bob", Labels: makeLabelState("feature", "priority=low")}, - models.Issue{Id: 3, RepoAt: "at://did:plc:test/sh.tangled.repo/abc", Title: "High priority feature", Body: "Important", Open: true, Did: "did:plc:alice", + models.Issue{Id: 3, RepoDid: "did:plc:testrepo", Title: "High priority feature", Body: "Important", Open: true, Did: "did:plc:alice", Labels: makeLabelState("feature", "priority=high")}, ) require.NoError(t, err) opts := func() models.IssueSearchOptions { return models.IssueSearchOptions{ - RepoAt: "at://did:plc:test/sh.tangled.repo/abc", - IsOpen: boolPtr(true), - Page: pagination.Page{Limit: 10}, + RepoDid: "did:plc:testrepo", + IsOpen: boolPtr(true), + Page: pagination.Page{Limit: 10}, } } diff --git a/appview/indexer/notifier.go b/appview/indexer/notifier.go --- a/appview/indexer/notifier.go +++ b/appview/indexer/notifier.go @@ -4,7 +4,6 @@ import ( "context" "github.com/bluesky-social/indigo/atproto/syntax" - "tangled.org/core/api/tangled" "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/notify" @@ -14,10 +13,10 @@ ) var _ notify.Notifier = &Indexer{} -func (ix *Indexer) getAndReindexRepo(ctx context.Context, repoAt syntax.ATURI) { - l := log.FromContext(ctx).With("notifier", "indexer", "repo_at", repoAt) +func (ix *Indexer) getAndReindexRepo(ctx context.Context, repoDid syntax.DID) { + l := log.FromContext(ctx).With("notifier", "indexer", "repo_did", repoDid) - repo, err := db.GetRepo(ix.Db, orm.FilterEq("at_uri", repoAt.String())) + repo, err := db.GetRepo(ix.Db, orm.FilterEq("repo_did", string(repoDid))) if err != nil { l.Error("failed to get repo for reindexing", "err", err) return @@ -39,7 +38,7 @@ l.Error("failed to index an issue", "err", err) } l.Debug("reindexing repo after new issue") - ix.getAndReindexRepo(ctx, issue.RepoAt) + ix.getAndReindexRepo(ctx, issue.RepoDid) } func (ix *Indexer) NewIssueState(ctx context.Context, actor syntax.DID, issue *models.Issue) { @@ -61,7 +60,7 @@ l.Error("failed to delete an issue", "err", err) } l.Debug("reindexing repo after issue deletion") - ix.getAndReindexRepo(ctx, issue.RepoAt) + ix.getAndReindexRepo(ctx, issue.RepoDid) } func (ix *Indexer) NewIssueLabelOp(ctx context.Context, issue *models.Issue) { @@ -92,7 +91,7 @@ l.Error("failed to index a pr", "err", err) } l.Debug("reindexing repo after new pull") - ix.getAndReindexRepo(ctx, pull.RepoAt) + ix.getAndReindexRepo(ctx, pull.RepoDid) } func (ix *Indexer) NewPullState(ctx context.Context, actor syntax.DID, pull *models.Pull) { @@ -113,6 +112,15 @@ l.Error("failed to index a repo", "err", err) } } +func (ix *Indexer) RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) { + l := log.FromContext(ctx).With("notifier", "indexer", "repo", newRepo.RepoIdentifier(), "actor", actor, "old_name", oldRepo.Name, "new_name", newRepo.Name) + l.Debug("reindexing repo after rename") + err := ix.Repos.Index(ctx, *newRepo) + if err != nil { + l.Error("failed to reindex repo", "err", err) + } +} + func (ix *Indexer) DeleteRepo(ctx context.Context, repo *models.Repo) { l := log.FromContext(ctx).With("notifier", "indexer", "repo", repo) l.Debug("deleting repo from index") @@ -125,21 +133,21 @@ func (ix *Indexer) NewStar(ctx context.Context, star *models.Star) { l := log.FromContext(ctx).With("notifier", "indexer", "star", star) - if star.RepoAt.Collection().String() != tangled.RepoNSID { + if star.SubjectType != models.StarSubjectRepo { return } l.Debug("reindexing repo after new star") - ix.getAndReindexRepo(ctx, star.RepoAt) + ix.getAndReindexRepo(ctx, syntax.DID(star.Subject)) } func (ix *Indexer) DeleteStar(ctx context.Context, star *models.Star) { l := log.FromContext(ctx).With("notifier", "indexer", "star", star) - if star.RepoAt.Collection().String() != tangled.RepoNSID { + if star.SubjectType != models.StarSubjectRepo { return } l.Debug("reindexing repo after star deletion") - ix.getAndReindexRepo(ctx, star.RepoAt) + ix.getAndReindexRepo(ctx, syntax.DID(star.Subject)) } diff --git a/appview/indexer/pulls/indexer.go b/appview/indexer/pulls/indexer.go --- a/appview/indexer/pulls/indexer.go +++ b/appview/indexer/pulls/indexer.go @@ -30,7 +30,7 @@ unicodeNormalizeName = "uicodeNormalize" // Bump this when the index mapping changes to trigger a rebuild. - pullIndexerVersion = 3 + pullIndexerVersion = 4 ) type Indexer struct { @@ -80,7 +80,7 @@ docMapping.AddFieldMappingsAt("title", textFieldMapping) docMapping.AddFieldMappingsAt("body", textFieldMapping) - docMapping.AddFieldMappingsAt("repo_at", keywordFieldMapping) + docMapping.AddFieldMappingsAt("repo_did", keywordFieldMapping) docMapping.AddFieldMappingsAt("state", keywordFieldMapping) docMapping.AddFieldMappingsAt("author_did", keywordFieldMapping) docMapping.AddFieldMappingsAt("labels", keywordFieldMapping) @@ -180,7 +180,7 @@ } type pullData struct { ID int64 `json:"id"` - RepoAt string `json:"repo_at"` + RepoDid string `json:"repo_did"` PullID int `json:"pull_id"` Title string `json:"title"` Body string `json:"body"` @@ -195,7 +195,7 @@ func makePullData(pull *models.Pull) *pullData { return &pullData{ ID: int64(pull.ID), - RepoAt: pull.RepoAt.String(), + RepoDid: string(pull.RepoDid), PullID: pull.PullId, Title: pull.Title, Body: pull.Body, @@ -275,7 +275,7 @@ bleveutil.MatchPhraseQuery("body", phrase, pullIndexerAnalyzer), )) } - musts = append(musts, bleveutil.KeywordFieldQuery("repo_at", opts.RepoAt)) + musts = append(musts, bleveutil.KeywordFieldQuery("repo_did", opts.RepoDid)) if opts.State != nil { musts = append(musts, bleveutil.KeywordFieldQuery("state", opts.State.String())) } diff --git a/appview/indexer/repos/indexer.go b/appview/indexer/repos/indexer.go --- a/appview/indexer/repos/indexer.go +++ b/appview/indexer/repos/indexer.go @@ -34,7 +34,7 @@ unicodeNormalizeName = "unicodeNormalize" // Bump this when the index mapping changes to trigger a rebuild. - repoIndexerVersion = 6 + repoIndexerVersion = 7 ) type Indexer struct { @@ -120,7 +120,7 @@ docMapping.AddFieldMappingsAt("language", caseInsensitiveKeywordMapping) docMapping.AddFieldMappingsAt("topics_exact", caseInsensitiveKeywordMapping) docMapping.AddFieldMappingsAt("did", keywordFieldMapping) docMapping.AddFieldMappingsAt("knot", keywordFieldMapping) - docMapping.AddFieldMappingsAt("repo_at", keywordFieldMapping) + docMapping.AddFieldMappingsAt("repo_did", keywordFieldMapping) // fork indicator for down-ranking docMapping.AddFieldMappingsAt("is_fork", booleanFieldMapping) @@ -258,7 +258,7 @@ } type repoData struct { ID int64 `json:"id"` - RepoAt string `json:"repo_at"` + RepoDid string `json:"repo_did"` Did string `json:"did"` Name string `json:"name"` NameTrigram string `json:"name_trigram"` @@ -294,7 +294,7 @@ isFork := repo.Source != "" return &repoData{ ID: repo.Id, - RepoAt: repo.RepoAt().String(), + RepoDid: repo.RepoDid, Did: repo.Did, Name: repo.Name, NameTrigram: repo.Name, diff --git a/appview/ingester.go b/appview/ingester.go --- a/appview/ingester.go +++ b/appview/ingester.go @@ -12,6 +12,7 @@ "maps" "net/http" "net/url" "slices" + "strings" "sync" "time" @@ -27,6 +28,8 @@ "tangled.org/core/appview/cache" "tangled.org/core/appview/config" "tangled.org/core/appview/db" "tangled.org/core/appview/models" + "tangled.org/core/appview/notify" + "tangled.org/core/appview/repoverify" "tangled.org/core/appview/serververify" "tangled.org/core/appview/validator" "tangled.org/core/idresolver" @@ -35,13 +38,15 @@ "tangled.org/core/rbac" ) type Ingester struct { - Db db.DbWrapper + Db *db.DB Enforcer *rbac.Enforcer IdResolver *idresolver.Resolver Cache *cache.Cache Config *config.Config Logger *slog.Logger Validator *validator.Validator + Notifier notify.Notifier + Verifier repoverify.Verifier } type processFunc func(ctx context.Context, e *jmodels.Event) error @@ -97,6 +102,8 @@ case tangled.LabelDefinitionNSID: err = i.ingestLabelDefinition(e) case tangled.LabelOpNSID: err = i.ingestLabelOp(e) + case tangled.RepoNSID: + err = i.ingestRepo(ctx, e) } l = i.Logger.With("nsid", e.Commit.Collection) } @@ -111,6 +118,59 @@ l.Error("failed to save cursor", "err", saveErr) } return nil + } +} + +func (i *Ingester) resolveRepoRef(ref string) (*models.Repo, error) { + if strings.HasPrefix(ref, "did:") { + return db.GetRepoByDid(i.Db, ref) + } + return db.GetRepoByAtUri(i.Db, ref) +} + +func (i *Ingester) resolveOldFormatStar(raw json.RawMessage, star *models.Star, l *slog.Logger) (bool, error) { + var legacy struct { + Subject *string `json:"subject"` + SubjectDid *string `json:"subjectDid"` + } + if err := json.Unmarshal(raw, &legacy); err != nil { + return false, err + } + + switch { + case legacy.SubjectDid != nil: + repo, err := i.resolveRepoRef(*legacy.SubjectDid) + if err != nil { + l.Warn("skipping old-format star for unknown repo", "subjectDid", *legacy.SubjectDid) + return false, nil + } + star.SubjectType = models.StarSubjectRepo + star.Subject = repo.RepoDid + return true, nil + + case legacy.Subject != nil: + uri, err := syntax.ParseATURI(*legacy.Subject) + if err != nil { + return false, fmt.Errorf("invalid old-format star subject: %w", err) + } + switch uri.Collection().String() { + case tangled.RepoNSID: + repo, err := db.GetRepoByAtUri(i.Db, uri.String()) + if err != nil { + l.Warn("skipping old-format star for unknown repo", "subject", *legacy.Subject) + return false, nil + } + star.SubjectType = models.StarSubjectRepo + star.Subject = repo.RepoDid + return true, nil + default: + star.SubjectType = models.StarSubjectString + star.Subject = *legacy.Subject + return true, nil + } + + default: + return false, fmt.Errorf("old-format star has neither subject nor subjectDid") } } @@ -123,15 +183,9 @@ l = l.With("nsid", e.Commit.Collection) switch e.Commit.Operation { case jmodels.CommitOperationCreate, jmodels.CommitOperationUpdate: - var subjectUri syntax.ATURI - raw := json.RawMessage(e.Commit.Record) record := tangled.FeedStar{} - err := json.Unmarshal(raw, &record) - if err != nil { - l.Error("invalid record", "err", err) - return err - } + unmarshalErr := json.Unmarshal(raw, &record) star := &models.Star{ Did: did, @@ -139,29 +193,36 @@ Rkey: e.Commit.RKey, } switch { - case record.SubjectDid != nil: - repo, repoErr := db.GetRepo(i.Db, orm.FilterEq("repo_did", *record.SubjectDid)) - if repoErr == nil { - subjectUri = repo.RepoAt() - star.RepoAt = subjectUri + case unmarshalErr != nil: + resolved, resolveErr := i.resolveOldFormatStar(raw, star, l) + if resolveErr != nil { + l.Error("invalid record", "newFmtErr", unmarshalErr, "oldFmtErr", resolveErr) + return unmarshalErr } - case record.Subject != nil: - subjectUri, err = syntax.ParseATURI(*record.Subject) - if err != nil { - l.Error("invalid record", "err", err) - return err + if !resolved { + return nil } - star.RepoAt = subjectUri - repo, repoErr := db.GetRepoByAtUri(i.Db, subjectUri.String()) - if repoErr == nil && repo.RepoDid != "" { - if enqErr := db.EnqueuePdsRecordMigration(ctx, i.Db, "add-repo-did", syntax.DID(did), syntax.NSID(tangled.FeedStarNSID), syntax.RecordKey(e.Commit.RKey)); enqErr != nil { - l.Warn("failed to enqueue PDS rewrite for star", "err", enqErr, "did", did, "repoDid", repo.RepoDid) - } + + case record.Subject == nil: + return fmt.Errorf("star record has nil subject") + + case record.Subject.FeedStar_Repo != nil: + repo, repoErr := i.resolveRepoRef(record.Subject.FeedStar_Repo.Did) + if repoErr != nil { + l.Warn("skipping star for unknown repo", "did", record.Subject.FeedStar_Repo.Did) + return nil } + star.SubjectType = models.StarSubjectRepo + star.Subject = repo.RepoDid + + case record.Subject.FeedStar_String != nil: + star.SubjectType = models.StarSubjectString + star.Subject = record.Subject.FeedStar_String.Uri + default: - l.Error("star record has neither subject nor subjectDid") - return fmt.Errorf("star record has neither subject nor subjectDid") + return fmt.Errorf("star record has empty subject union") } + err = db.AddStar(i.Db, star) case jmodels.CommitOperationDelete: err = db.DeleteStarByRkey(i.Db, did, e.Commit.RKey) @@ -270,12 +331,7 @@ } evidences = append(evidences, uri) } - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to ingest vouch record, invalid db cast") - } - - tx, txErr := ddb.Begin() + tx, txErr := i.Db.Begin() if txErr != nil { return fmt.Errorf("failed to start transaction: %w", txErr) } @@ -413,7 +469,7 @@ artifact := models.Artifact{ Did: did, Rkey: e.Commit.RKey, - RepoAt: repo.RepoAt(), + RepoDid: syntax.DID(repo.RepoDid), Tag: plumbing.Hash(record.Tag), CreatedAt: createdAt, BlobCid: cid.Cid(record.Artifact.Ref), @@ -521,12 +577,7 @@ Pronouns: pronouns, PreferredHandle: preferredHandle, } - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index profile record, invalid db cast") - } - - tx, err := ddb.Begin() + tx, err := i.Db.Begin() if err != nil { return fmt.Errorf("failed to start transaction") } @@ -593,12 +644,7 @@ if memberId.Handle.IsInvalidHandle() { return err } - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("invalid db cast") - } - - err = db.AddSpindleMember(ddb, models.SpindleMember{ + err = db.AddSpindleMember(i.Db, models.SpindleMember{ Did: syntax.DID(did), Rkey: e.Commit.RKey, Instance: record.Instance, @@ -617,14 +663,9 @@ l.Info("added spindle member") case jmodels.CommitOperationDelete: rkey := e.Commit.RKey - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index profile record, invalid db cast") - } - // get record from db first members, err := db.GetSpindleMembers( - ddb, + i.Db, orm.FilterEq("did", did), orm.FilterEq("rkey", rkey), ) @@ -633,7 +674,7 @@ return fmt.Errorf("failed to get member: %w, len(members) = %d", err, len(members)) } member := members[0] - tx, err := ddb.Begin() + tx, err := i.Db.Begin() if err != nil { return fmt.Errorf("failed to start txn: %w", err) } @@ -686,12 +727,7 @@ } instance := e.Commit.RKey - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index profile record, invalid db cast") - } - - err := db.AddSpindle(ddb, models.Spindle{ + err := db.AddSpindle(i.Db, models.Spindle{ Owner: syntax.DID(did), Instance: instance, }) @@ -710,7 +746,7 @@ l.Error("failed to verify spindle after retries", "err", err, "instance", instance) return err } - _, err = serververify.MarkSpindleVerified(ddb, i.Enforcer, instance, did) + _, err = serververify.MarkSpindleVerified(i.Db, i.Enforcer, instance, did) if err != nil { return fmt.Errorf("failed to mark verified: %w", err) } @@ -720,15 +756,10 @@ case jmodels.CommitOperationDelete: instance := e.Commit.RKey - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index profile record, invalid db cast") - } - // get record from db first spindles, err := db.GetSpindles( ctx, - ddb, + i.Db, orm.FilterEq("owner", did), orm.FilterEq("instance", instance), ) @@ -737,7 +768,7 @@ return fmt.Errorf("failed to get spindles: %w, len(spindles) = %d", err, len(spindles)) } spindle := spindles[0] - tx, err := ddb.Begin() + tx, err := i.Db.Begin() if err != nil { return err } @@ -795,11 +826,6 @@ l := i.Logger.With("handler", "ingestString", "nsid", e.Commit.Collection, "did", did, "rkey", rkey) l.Info("ingesting record") - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index string record, invalid db cast") - } - switch e.Commit.Operation { case jmodels.CommitOperationCreate, jmodels.CommitOperationUpdate: raw := json.RawMessage(e.Commit.Record) @@ -817,7 +843,7 @@ l.Error("invalid record", "err", err) return err } - if err = db.AddString(ddb, string); err != nil { + if err = db.AddString(i.Db, string); err != nil { l.Error("failed to add string", "err", err) return err } @@ -826,7 +852,7 @@ return nil case jmodels.CommitOperationDelete: if err := db.DeleteString( - ddb, + i.Db, orm.FilterEq("did", did), orm.FilterEq("rkey", rkey), ); err != nil { @@ -911,12 +937,7 @@ } domain := e.Commit.RKey - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index profile record, invalid db cast") - } - - err := db.AddKnot(ddb, domain, did) + err := db.AddKnot(i.Db, domain, did) if err != nil { l.Error("failed to add knot to db", "err", err, "domain", domain) return err @@ -934,7 +955,7 @@ l.Error("failed to verify knot after retries", "err", err, "domain", domain) return err } - err = serververify.MarkKnotVerified(ddb, i.Enforcer, domain, did) + err = serververify.MarkKnotVerified(i.Db, i.Enforcer, domain, did) if err != nil { return fmt.Errorf("failed to mark verified: %w", err) } @@ -944,14 +965,9 @@ case jmodels.CommitOperationDelete: domain := e.Commit.RKey - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index knot record, invalid db cast") - } - // get record from db first registrations, err := db.GetRegistrations( - ddb, + i.Db, orm.FilterEq("domain", domain), orm.FilterEq("did", did), ) @@ -963,7 +979,7 @@ return fmt.Errorf("got incorrect number of registrations: %d, expected 1", len(registrations)) } registration := registrations[0] - tx, err := ddb.Begin() + tx, err := i.Db.Begin() if err != nil { return err } @@ -1010,11 +1026,6 @@ l := i.Logger.With("handler", "ingestIssue", "nsid", e.Commit.Collection, "did", did, "rkey", rkey) l.Info("ingesting record") - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index issue record, invalid db cast") - } - switch e.Commit.Operation { case jmodels.CommitOperationCreate, jmodels.CommitOperationUpdate: raw := json.RawMessage(e.Commit.Record) @@ -1027,16 +1038,19 @@ } issue := models.IssueFromRecord(did, rkey, record) - if issue.RepoAt == "" { + if issue.RepoDid == "" { return fmt.Errorf("issue record has no repo field") + } + if _, err := syntax.ParseDID(string(issue.RepoDid)); err != nil { + return fmt.Errorf("issue record repo field is not a valid DID: %w", err) } if err := i.Validator.ValidateIssue(&issue); err != nil { return fmt.Errorf("failed to validate issue: %w", err) } - if record.Repo != nil { - repo, repoErr := db.GetRepoByAtUri(i.Db, *record.Repo) + if record.Repo != "" && !strings.HasPrefix(record.Repo, "did:") { + repo, repoErr := db.GetRepoByAtUri(i.Db, record.Repo) if repoErr == nil && repo.RepoDid != "" { if enqErr := db.EnqueuePdsRecordMigration(ctx, i.Db, "add-repo-did", syntax.DID(did), syntax.NSID(tangled.RepoIssueNSID), syntax.RecordKey(e.Commit.RKey)); enqErr != nil { l.Warn("failed to enqueue PDS rewrite for issue", "err", enqErr, "did", did, "repoDid", repo.RepoDid) @@ -1044,7 +1058,7 @@ } } } - tx, err := ddb.BeginTx(ctx, nil) + tx, err := i.Db.BeginTx(ctx, nil) if err != nil { l.Error("failed to begin transaction", "err", err) return err @@ -1066,7 +1080,7 @@ return nil case jmodels.CommitOperationDelete: - tx, err := ddb.BeginTx(ctx, nil) + tx, err := i.Db.BeginTx(ctx, nil) if err != nil { l.Error("failed to begin transaction", "err", err) return err @@ -1101,11 +1115,6 @@ l := i.Logger.With("handler", "ingestPull", "nsid", e.Commit.Collection, "did", did, "rkey", rkey) l.Info("ingesting record") - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index pull record, invalid db cast") - } - switch e.Commit.Operation { case jmodels.CommitOperationCreate, jmodels.CommitOperationUpdate: raw := json.RawMessage(e.Commit.Record) @@ -1188,7 +1197,7 @@ if err := i.Validator.ValidatePull(pull); err != nil { return fmt.Errorf("failed to validate pull: %w", err) } - tx, err := ddb.BeginTx(ctx, nil) + tx, err := i.Db.BeginTx(ctx, nil) if err != nil { l.Error("failed to begin transaction", "err", err) return err @@ -1210,7 +1219,7 @@ return nil case jmodels.CommitOperationDelete: - tx, err := ddb.BeginTx(ctx, nil) + tx, err := i.Db.BeginTx(ctx, nil) if err != nil { l.Error("failed to begin transaction", "err", err) return err @@ -1245,11 +1254,6 @@ l := i.Logger.With("handler", "ingestIssueComment", "nsid", e.Commit.Collection, "did", did, "rkey", rkey) l.Info("ingesting record") - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index issue comment record, invalid db cast") - } - switch e.Commit.Operation { case jmodels.CommitOperationCreate, jmodels.CommitOperationUpdate: raw := json.RawMessage(e.Commit.Record) @@ -1268,7 +1272,7 @@ if err := i.Validator.ValidateIssueComment(comment); err != nil { return fmt.Errorf("failed to validate comment: %w", err) } - tx, err := ddb.Begin() + tx, err := i.Db.Begin() if err != nil { return fmt.Errorf("failed to start transaction: %w", err) } @@ -1283,7 +1287,7 @@ return tx.Commit() case jmodels.CommitOperationDelete: if err := db.DeleteIssueComments( - ddb, + i.Db, orm.FilterEq("did", did), orm.FilterEq("rkey", rkey), ); err != nil { @@ -1304,11 +1308,6 @@ var err error l := i.Logger.With("handler", "ingestLabelDefinition", "nsid", e.Commit.Collection, "did", did, "rkey", rkey) l.Info("ingesting record") - - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index label definition, invalid db cast") - } switch e.Commit.Operation { case jmodels.CommitOperationCreate, jmodels.CommitOperationUpdate: @@ -1328,7 +1327,7 @@ if err := i.Validator.ValidateLabelDefinition(def); err != nil { return fmt.Errorf("failed to validate labeldef: %w", err) } - _, err = db.AddLabelDefinition(ddb, def) + _, err = db.AddLabelDefinition(i.Db, def) if err != nil { return fmt.Errorf("failed to create labeldef: %w", err) } @@ -1337,7 +1336,7 @@ return nil case jmodels.CommitOperationDelete: if err := db.DeleteLabelDefinition( - ddb, + i.Db, orm.FilterEq("did", did), orm.FilterEq("rkey", rkey), ); err != nil { @@ -1359,11 +1358,6 @@ l := i.Logger.With("handler", "ingestLabelOp", "nsid", e.Commit.Collection, "did", did, "rkey", rkey) l.Info("ingesting record") - ddb, ok := i.Db.Execer.(*db.DB) - if !ok { - return fmt.Errorf("failed to index label op, invalid db cast") - } - switch e.Commit.Operation { case jmodels.CommitOperationCreate: raw := json.RawMessage(e.Commit.Record) @@ -1379,7 +1373,7 @@ var repo *models.Repo switch collection { case tangled.RepoIssueNSID: - i, err := db.GetIssues(ddb, orm.FilterEq("at_uri", subject)) + i, err := db.GetIssues(i.Db, orm.FilterEq("at_uri", subject)) if err != nil || len(i) != 1 { return fmt.Errorf("failed to find subject: %w || subject count %d", err, len(i)) } @@ -1388,7 +1382,7 @@ default: return fmt.Errorf("unsupported label subject: %s", collection) } - actx, err := db.NewLabelApplicationCtx(ddb, orm.FilterIn("at_uri", repo.Labels)) + actx, err := db.NewLabelApplicationCtx(i.Db, orm.FilterIn("at_uri", repo.Labels)) if err != nil { return fmt.Errorf("failed to build label application ctx: %w", err) } @@ -1405,7 +1399,7 @@ return fmt.Errorf("failed to validate labelop: %w", err) } } - tx, err := ddb.Begin() + tx, err := i.Db.Begin() if err != nil { return err } diff --git a/appview/ingester_repo.go b/appview/ingester_repo.go new file mode 100644 --- /dev/null +++ b/appview/ingester_repo.go @@ -0,0 +1,395 @@ +package appview + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log/slog" + "slices" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + jmodels "github.com/bluesky-social/jetstream/pkg/models" + "tangled.org/core/api/tangled" + "tangled.org/core/appview/db" + "tangled.org/core/appview/models" + "tangled.org/core/appview/repoverify" + "tangled.org/core/orm" +) + +func (i *Ingester) ingestRepo(ctx context.Context, e *jmodels.Event) error { + l := i.Logger.With("handler", "ingestRepo", "did", e.Did, "rkey", e.Commit.RKey) + + switch e.Commit.Operation { + case jmodels.CommitOperationCreate: + return i.ingestRepoCreate(ctx, e) + case jmodels.CommitOperationUpdate: + return i.ingestRepoUpdate(ctx, e) + case jmodels.CommitOperationDelete: + return i.ingestRepoDelete(ctx, e) + default: + l.Info("unknown repo operation", "op", e.Commit.Operation) + return nil + } +} + +func (i *Ingester) ingestRepoCreate(ctx context.Context, e *jmodels.Event) error { + l := i.Logger.With("handler", "ingestRepoCreate", "did", e.Did, "rkey", e.Commit.RKey) + + record := tangled.Repo{} + if err := json.Unmarshal(json.RawMessage(e.Commit.Record), &record); err != nil { + l.Error("invalid record", "err", err) + return err + } + + if record.RepoDid == nil || *record.RepoDid == "" { + l.Info("skipping repo create from non-DID-migrated knot") + return nil + } + repoDid := *record.RepoDid + + proceed, err := i.verifyOwnership(ctx, l, repoDid, e.Did, record.Knot) + if err != nil { + return err + } + if !proceed { + return nil + } + + _, err = db.GetRepo(i.Db, + orm.FilterEq("did", e.Did), + orm.FilterEq("rkey", e.Commit.RKey), + ) + if err == nil { + l.Info("repo row already exists, skipping create", "did", e.Did, "rkey", e.Commit.RKey) + return nil + } + if !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("failed to check existing repo: %w", err) + } + + prev, err := db.GetRepoByDid(i.Db, repoDid) + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("failed to check existing repoDid: %w", err) + } + + if prev != nil { + l.Info("repoDid exists under different rkey, renaming", + "oldRkey", prev.Rkey, "newRkey", e.Commit.RKey) + + oldRepo := *prev + + tx, txErr := i.Db.Begin() + if txErr != nil { + return fmt.Errorf("failed to begin rename tx: %w", txErr) + } + defer tx.Rollback() + + newName := derefString(record.Name) + if newName == "" { + newName = e.Commit.RKey + } + + if err := db.RenameRepo(tx, e.Did, prev.Rkey, e.Commit.RKey, newName); err != nil { + return fmt.Errorf("failed to rename repo: %w", err) + } + if err := db.RecordRepoRename(tx, e.Did, prev.Rkey, repoDid); err != nil { + return fmt.Errorf("failed to record rename history: %w", err) + } + + renamed := *prev + renamed.Rkey = e.Commit.RKey + renamed.Name = newName + desired := repoFromRecord(&renamed, &record) + if repoMetadataChanged(&renamed, &desired) { + if err := applyRepoMetadata(tx, &renamed, desired); err != nil { + return fmt.Errorf("failed to apply metadata after rename: %w", err) + } + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit rename tx: %w", err) + } + + newRepo, err := db.GetRepo(i.Db, + orm.FilterEq("did", e.Did), + orm.FilterEq("rkey", e.Commit.RKey), + ) + if err != nil { + l.Warn("failed to fetch repo after rename for notification", "err", err) + return nil + } + i.Notifier.RenameRepo(ctx, syntax.DID(e.Did), &oldRepo, newRepo) + return nil + } + + rkey := e.Commit.RKey + name := derefString(record.Name) + if name == "" { + name = rkey + } + + repo := &models.Repo{ + Did: e.Did, + Name: name, + Knot: record.Knot, + Rkey: rkey, + Description: derefString(record.Description), + Website: derefString(record.Website), + Topics: append([]string(nil), record.Topics...), + Source: derefString(record.Source), + Spindle: derefString(record.Spindle), + Labels: append([]string(nil), record.Labels...), + RepoDid: repoDid, + } + + tx, err := i.Db.Begin() + if err != nil { + return fmt.Errorf("failed to begin insert tx: %w", err) + } + defer tx.Rollback() + + if err := db.AddRepo(tx, repo); err != nil { + return fmt.Errorf("failed to insert repo: %w", err) + } + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit insert tx: %w", err) + } + + i.Notifier.NewRepo(ctx, repo) + return nil +} + +func (i *Ingester) ingestRepoUpdate(ctx context.Context, e *jmodels.Event) error { + l := i.Logger.With("handler", "ingestRepoUpdate", "did", e.Did, "rkey", e.Commit.RKey) + + record := tangled.Repo{} + if err := json.Unmarshal(json.RawMessage(e.Commit.Record), &record); err != nil { + l.Error("invalid record", "err", err) + return err + } + + if record.RepoDid == nil || *record.RepoDid == "" { + l.Info("skipping repo update from non-DID-migrated knot") + return nil + } + + proceed, err := i.verifyOwnership(ctx, l, *record.RepoDid, e.Did, record.Knot) + if err != nil { + return err + } + if !proceed { + return nil + } + + current, err := db.GetRepo(i.Db, + orm.FilterEq("did", e.Did), + orm.FilterEq("rkey", e.Commit.RKey), + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + l.Info("skipping repo update for unknown row") + return nil + } + return fmt.Errorf("failed to fetch repo for ingest: %w", err) + } + + if current.RepoDid != "" && current.RepoDid != *record.RepoDid { + l.Warn("rejecting repo update: repoDid is immutable", + "currentRepoDid", current.RepoDid, + "recordRepoDid", *record.RepoDid, + ) + return nil + } + + desired := repoFromRecord(current, &record) + + if current.Source != desired.Source { + l.Warn("source field changed but mutation is unsupported, ignoring", + "current", current.Source, "desired", desired.Source) + } + + if !repoMetadataChanged(current, &desired) { + return nil + } + + tx, err := i.Db.Begin() + if err != nil { + return fmt.Errorf("failed to begin tx: %w", err) + } + defer tx.Rollback() + + if err := applyRepoMetadata(tx, current, desired); err != nil { + return fmt.Errorf("failed to apply repo metadata: %w", err) + } + return tx.Commit() +} + +func (i *Ingester) ingestRepoDelete(ctx context.Context, e *jmodels.Event) error { + l := i.Logger.With("handler", "ingestRepoDelete", "did", e.Did, "rkey", e.Commit.RKey) + + repo, err := db.GetRepo(i.Db, + orm.FilterEq("did", e.Did), + orm.FilterEq("rkey", e.Commit.RKey), + ) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + l.Info("skipping repo delete for unknown row") + return nil + } + return fmt.Errorf("failed to fetch repo for delete: %w", err) + } + + if err := db.RemoveRepo(i.Db, e.Did, e.Commit.RKey); err != nil { + return fmt.Errorf("failed to delete repo: %w", err) + } + + i.Notifier.DeleteRepo(ctx, repo) + l.Info("deleted repo row") + return nil +} + +func applyRepoMetadata(tx *sql.Tx, current *models.Repo, desired models.Repo) error { + if err := db.PutRepo(tx, desired); err != nil { + return err + } + + if current.Spindle != desired.Spindle { + var spindlePtr *string + if desired.Spindle != "" { + spindlePtr = &desired.Spindle + } + if err := db.UpdateSpindle(tx, desired.RepoDid, spindlePtr); err != nil { + return err + } + } + + if !labelsEqual(current.Labels, desired.Labels) { + if err := reconcileLabels(tx, current, desired); err != nil { + return err + } + } + + return nil +} + +func reconcileLabels(tx *sql.Tx, current *models.Repo, desired models.Repo) error { + added := filterOut(desired.Labels, current.Labels) + removed := filterOut(current.Labels, desired.Labels) + + if err := applyEach(added, func(l string) error { + return db.SubscribeLabel(tx, &models.RepoLabel{ + RepoDid: syntax.DID(desired.RepoDid), + LabelAt: syntax.ATURI(l), + }) + }); err != nil { + return err + } + + return applyEach(removed, func(l string) error { + return db.UnsubscribeLabel(tx, + orm.FilterEq("repo_did", desired.RepoDid), + orm.FilterEq("label_at", l), + ) + }) +} + +func filterOut(items, exclude []string) []string { + return slices.DeleteFunc(slices.Clone(items), func(s string) bool { + return slices.Contains(exclude, s) + }) +} + +func applyEach(items []string, fn func(string) error) error { + for _, item := range items { + if err := fn(item); err != nil { + return err + } + } + return nil +} + +func labelsEqual(a, b []string) bool { + if len(a) != len(b) { + return false + } + aSorted := append([]string(nil), a...) + bSorted := append([]string(nil), b...) + slices.Sort(aSorted) + slices.Sort(bSorted) + return slices.Equal(aSorted, bSorted) +} + +func repoFromRecord(current *models.Repo, record *tangled.Repo) models.Repo { + out := *current + out.Name = derefString(record.Name) + if out.Name == "" { + out.Name = current.Rkey + } + out.Knot = record.Knot + out.Description = derefString(record.Description) + out.Website = derefString(record.Website) + out.Topics = append([]string(nil), record.Topics...) + out.Spindle = derefString(record.Spindle) + out.Source = derefString(record.Source) + out.Labels = append([]string(nil), record.Labels...) + if record.RepoDid != nil { + out.RepoDid = *record.RepoDid + } + return out +} + +func repoMetadataChanged(current *models.Repo, desired *models.Repo) bool { + return current.Name != desired.Name || + current.Knot != desired.Knot || + current.Description != desired.Description || + current.Website != desired.Website || + current.TopicStr() != desired.TopicStr() || + current.Spindle != desired.Spindle || + !labelsEqual(current.Labels, desired.Labels) +} + +func derefString(s *string) string { + if s == nil { + return "" + } + return *s +} + +func (i *Ingester) verifyOwnership(ctx context.Context, l *slog.Logger, repoDid, eventDid, recordKnot string) (bool, error) { + if i.Verifier == nil { + return false, fmt.Errorf("ingester has no repo ownership verifier configured") + } + rd, err := repoverify.NewRepoDid(repoDid) + if err != nil { + l.Warn("rejecting repo event: invalid repoDid on record", "repoDid", repoDid, "err", err) + return false, nil + } + result, err := i.Verifier(ctx, rd) + if err != nil { + return false, fmt.Errorf("verify repo ownership: %w", err) + } + if result.OwnerDid == "" { + l.Warn("knot lacks RepoDescribeRepo, skipping owner check; upgrade knot to 1.14+", + "repoDid", repoDid, "knot", result.KnotURL.String()) + } else if result.OwnerDid.String() != eventDid { + l.Warn("rejecting repo event: owner mismatch", + "repoDid", repoDid, + "claimedOwner", eventDid, + "knotOwner", result.OwnerDid.String(), + "knot", result.KnotURL.String(), + ) + return false, nil + } + if !strings.EqualFold(recordKnot, result.KnotURL.Host) { + l.Warn("rejecting repo event: record knot does not match DID-doc endpoint", + "repoDid", repoDid, + "recordKnot", recordKnot, + "canonicalKnot", result.KnotURL.Host, + ) + return false, nil + } + return true, nil +} diff --git a/appview/ingester_repo_test.go b/appview/ingester_repo_test.go new file mode 100644 --- /dev/null +++ b/appview/ingester_repo_test.go @@ -0,0 +1,754 @@ +package appview + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "io" + "log/slog" + "net/url" + "path/filepath" + "testing" + + "github.com/bluesky-social/indigo/atproto/syntax" + jmodels "github.com/bluesky-social/jetstream/pkg/models" + "tangled.org/core/api/tangled" + "tangled.org/core/appview/db" + "tangled.org/core/appview/models" + "tangled.org/core/appview/notify" + "tangled.org/core/appview/repoverify" + "tangled.org/core/orm" +) + +func mustKnotURL(t *testing.T, raw string) *url.URL { + t.Helper() + u, err := repoverify.ParseKnotEndpoint(raw, true) + if err != nil { + t.Fatalf("ParseKnotEndpoint(%q): %v", raw, err) + } + return u +} + +func acceptOwner(t *testing.T, e *jmodels.Event) repoverify.Verifier { + t.Helper() + knot := mustKnotURL(t, "https://knot.example") + return func(_ context.Context, repoDid repoverify.RepoDid) (repoverify.Result, error) { + return repoverify.Result{ + RepoDid: repoDid, + OwnerDid: repoverify.OwnerDid(e.Did), + KnotURL: knot, + }, nil + } +} + +func stubVerifier(result repoverify.Result, err error) repoverify.Verifier { + return func(_ context.Context, _ repoverify.RepoDid) (repoverify.Result, error) { + return result, err + } +} + +type spyNotifier struct { + notify.BaseNotifier + creates int + deletes int + renames int +} + +func (s *spyNotifier) NewRepo(_ context.Context, _ *models.Repo) { s.creates++ } +func (s *spyNotifier) DeleteRepo(_ context.Context, _ *models.Repo) { s.deletes++ } +func (s *spyNotifier) RenameRepo(_ context.Context, _ syntax.DID, _, _ *models.Repo) { + s.renames++ +} + +func newTestIngester(t *testing.T) (*Ingester, *spyNotifier) { + t.Helper() + path := filepath.Join(t.TempDir(), "test.db") + d, err := db.Make(context.Background(), path) + if err != nil { + t.Fatalf("db.Make: %v", err) + } + t.Cleanup(func() { d.Close() }) + + spy := &spyNotifier{} + ing := &Ingester{ + Db: d, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + Notifier: spy, + } + return ing, spy +} + +func withVerifier(ing *Ingester, v repoverify.Verifier) *Ingester { + ing.Verifier = v + return ing +} + +func ingestAcceptingOwner(t *testing.T, ing *Ingester, e *jmodels.Event) error { + t.Helper() + ing.Verifier = acceptOwner(t, e) + return ing.ingestRepo(context.Background(), e) +} + +func seedRepoRow(t *testing.T, ing *Ingester, did, knot, name, rkey, repoDid string) *models.Repo { + t.Helper() + tx, err := ing.Db.Begin() + if err != nil { + t.Fatalf("Begin: %v", err) + } + repo := &models.Repo{ + Did: did, + Name: name, + Knot: knot, + Rkey: rkey, + RepoDid: repoDid, + } + if err := db.AddRepo(tx, repo); err != nil { + t.Fatalf("AddRepo: %v", err) + } + if err := tx.Commit(); err != nil { + t.Fatalf("Commit: %v", err) + } + return repo +} + +func ptr[T any](v T) *T { return &v } + +func makeEvent(t *testing.T, op string, did, rkey string, record tangled.Repo) *jmodels.Event { + t.Helper() + raw, err := json.Marshal(record) + if err != nil { + t.Fatalf("marshal record: %v", err) + } + return &jmodels.Event{ + Did: did, + Kind: jmodels.EventKindCommit, + Commit: &jmodels.Commit{ + Operation: op, + Collection: tangled.RepoNSID, + RKey: rkey, + Record: raw, + }, + } +} + +func makeDeleteEvent(did, rkey string) *jmodels.Event { + return &jmodels.Event{ + Did: did, + Kind: jmodels.EventKindCommit, + Commit: &jmodels.Commit{ + Operation: jmodels.CommitOperationDelete, + Collection: tangled.RepoNSID, + RKey: rkey, + }, + } +} + +func loadRepo(t *testing.T, ing *Ingester, did, rkey string) *models.Repo { + t.Helper() + r, err := db.GetRepo(ing.Db, + orm.FilterEq("did", did), + orm.FilterEq("rkey", rkey), + ) + if err != nil { + t.Fatalf("GetRepo: %v", err) + } + return r +} + +func TestIngestRepo_CreateInsertsNewRow(t *testing.T) { + ing, spy := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "knot.example", + Name: ptr("MyRepo"), + Description: ptr("a test repo"), + RepoDid: ptr("did:plc:repo1"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "myrepo") + if r.Name != "MyRepo" { + t.Errorf("name = %q, want %q", r.Name, "MyRepo") + } + if r.Description != "a test repo" { + t.Errorf("description = %q", r.Description) + } + if r.RepoDid != "did:plc:repo1" { + t.Errorf("repoDid = %q", r.RepoDid) + } + if spy.creates != 1 { + t.Errorf("NewRepo called %d times, want 1", spy.creates) + } +} + +func TestIngestRepo_CreateSkipsIfRowExists(t *testing.T) { + ing, spy := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "myrepo", "myrepo", "did:plc:repo1") + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "knot.example", + Name: ptr("myrepo"), + RepoDid: ptr("did:plc:repo1"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + if spy.creates != 0 { + t.Errorf("row already exists, NewRepo should not be called but was called %d times", spy.creates) + } +} + +func TestIngestRepo_CreateCascadesRename(t *testing.T) { + ing, spy := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "oldname", "oldname", "did:plc:repo1") + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "newname", tangled.Repo{ + Knot: "knot.example", + Name: ptr("NewName"), + RepoDid: ptr("did:plc:repo1"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + _, err := db.GetRepo(ing.Db, + orm.FilterEq("did", "did:plc:akshay"), + orm.FilterEq("rkey", "oldname"), + ) + if !errors.Is(err, sql.ErrNoRows) { + t.Errorf("old rkey row should be gone, got err = %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "newname") + if r.Name != "NewName" { + t.Errorf("name = %q, want %q", r.Name, "NewName") + } + if r.RepoDid != "did:plc:repo1" { + t.Errorf("repoDid = %q", r.RepoDid) + } + + hint, err := db.LookupRepoRename(ing.Db, "did:plc:akshay", "oldname") + if err != nil { + t.Fatalf("LookupRepoRename: %v", err) + } + if hint == nil { + t.Fatal("expected rename history, got nil") + } + + if spy.renames != 1 { + t.Errorf("RenameRepo called %d times, want 1", spy.renames) + } + if spy.creates != 0 { + t.Errorf("rename should not create: NewRepo called %d times, want 0", spy.creates) + } +} + +func TestIngestRepo_CreateNoRepoDidSkipped(t *testing.T) { + ing, spy := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "knot.example", + Name: ptr("myrepo"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + if spy.creates != 0 { + t.Errorf("NewRepo called %d times, want 0", spy.creates) + } +} + +func TestIngestRepo_UpdateMetadata(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "foo", "foo", "did:plc:repo1") + + e := makeEvent(t, jmodels.CommitOperationUpdate, "did:plc:akshay", "foo", tangled.Repo{ + Knot: "knot.example", + Name: ptr("foo"), + Description: ptr("updated description"), + Website: ptr("https://example.com"), + Topics: []string{"go", "test"}, + RepoDid: ptr("did:plc:repo1"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "foo") + if r.Description != "updated description" { + t.Errorf("description = %q", r.Description) + } + if r.Website != "https://example.com" { + t.Errorf("website = %q", r.Website) + } + if got := r.TopicStr(); got != "go test" { + t.Errorf("topics = %q", got) + } +} + +func TestIngestRepo_UpdateDisplayName(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "foo", "foo", "did:plc:repo1") + + e := makeEvent(t, jmodels.CommitOperationUpdate, "did:plc:akshay", "foo", tangled.Repo{ + Knot: "knot.example", + Name: ptr("Foo"), + RepoDid: ptr("did:plc:repo1"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "foo") + if r.Name != "Foo" { + t.Errorf("name = %q, want %q", r.Name, "Foo") + } + if r.Rkey != "foo" { + t.Errorf("rkey should be unchanged but got %q, want %q", r.Rkey, "foo") + } +} + +func TestIngestRepo_UpdateNothingChangedNoOp(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "foo", "foo", "did:plc:repo1") + + e := makeEvent(t, jmodels.CommitOperationUpdate, "did:plc:akshay", "foo", tangled.Repo{ + Knot: "knot.example", + Name: ptr("foo"), + RepoDid: ptr("did:plc:repo1"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "foo") + if r.Name != "foo" { + t.Errorf("name = %q, want unchanged %q", r.Name, "foo") + } +} + +func TestIngestRepo_UnknownRowSkipped(t *testing.T) { + ops := []string{jmodels.CommitOperationUpdate, jmodels.CommitOperationDelete} + for _, op := range ops { + t.Run(op, func(t *testing.T) { + ing, _ := newTestIngester(t) + + var e *jmodels.Event + switch op { + case jmodels.CommitOperationUpdate: + e = makeEvent(t, op, "did:plc:nobody", "ghost", tangled.Repo{ + Knot: "knot.example", + Name: ptr("ghost"), + RepoDid: ptr("did:plc:nope"), + }) + case jmodels.CommitOperationDelete: + e = makeDeleteEvent("did:plc:nobody", "ghost") + } + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + }) + } +} + +func TestIngestRepo_UpdateNoRepoDidSkipped(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "foo", "foo", "did:plc:repo1") + + e := makeEvent(t, jmodels.CommitOperationUpdate, "did:plc:akshay", "foo", tangled.Repo{ + Knot: "knot.example", + Name: ptr("bar"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "foo") + if r.Name != "foo" { + t.Errorf("name = %q, want unchanged %q", r.Name, "foo") + } +} + +func TestIngestRepo_DeleteRemovesRow(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "foo", "foo", "did:plc:repo1") + + e := makeDeleteEvent("did:plc:akshay", "foo") + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + _, err := db.GetRepo(ing.Db, + orm.FilterEq("did", "did:plc:akshay"), + orm.FilterEq("rkey", "foo"), + ) + if !errors.Is(err, sql.ErrNoRows) { + t.Errorf("expected row to be deleted, got err = %v", err) + } +} + +func TestIngestRepo_MalformedRecord(t *testing.T) { + ing, _ := newTestIngester(t) + + e := &jmodels.Event{ + Did: "did:plc:akshay", + Kind: jmodels.EventKindCommit, + Commit: &jmodels.Commit{ + Operation: jmodels.CommitOperationUpdate, + Collection: tangled.RepoNSID, + RKey: "rkey1", + Record: json.RawMessage("{not json"), + }, + } + + if err := ingestAcceptingOwner(t, ing, e); err == nil { + t.Errorf("ingestRepo with malformed record: err = nil, want error") + } +} + +func TestIngestRepo_RenameDeleteSequenceNoTornState(t *testing.T) { + ing, spy := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "oldname", "oldname", "did:plc:repo1") + + if _, err := ing.Db.Exec( + `insert into stars (did, rkey, subject_type, subject) values (?, ?, ?, ?)`, + "did:plc:boltless", "star1", "repo", "did:plc:repo1", + ); err != nil { + t.Fatalf("seed star: %v", err) + } + + createEvt := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "newname", tangled.Repo{ + Knot: "knot.example", + Name: ptr("NewName"), + RepoDid: ptr("did:plc:repo1"), + }) + if err := ingestAcceptingOwner(t, ing, createEvt); err != nil { + t.Fatalf("ingest create: %v", err) + } + + deleteEvt := makeDeleteEvent("did:plc:akshay", "oldname") + if err := ingestAcceptingOwner(t, ing, deleteEvt); err != nil { + t.Fatalf("ingest delete: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "newname") + if r.Name != "NewName" { + t.Errorf("name = %q, want %q", r.Name, "NewName") + } + if r.RepoDid != "did:plc:repo1" { + t.Errorf("repoDid = %q, want %q", r.RepoDid, "did:plc:repo1") + } + + _, err := db.GetRepo(ing.Db, + orm.FilterEq("did", "did:plc:akshay"), + orm.FilterEq("rkey", "oldname"), + ) + if !errors.Is(err, sql.ErrNoRows) { + t.Errorf("old rkey should be gone, got err = %v", err) + } + + var starSubject string + if err := ing.Db.QueryRow(`select subject from stars where did = ?`, "did:plc:boltless").Scan(&starSubject); err != nil { + t.Fatalf("query star: %v", err) + } + if starSubject != "did:plc:repo1" { + t.Errorf("star subject = %q, want %q", starSubject, "did:plc:repo1") + } + + if spy.renames != 1 { + t.Errorf("RenameRepo called %d times, want 1", spy.renames) + } + if spy.creates != 0 { + t.Errorf("rename should not create: NewRepo called %d times, want 0", spy.creates) + } + if spy.deletes != 0 { + t.Errorf("old rkey already gone, DeleteRepo should not be called but was called %d times", spy.deletes) + } +} + +func TestIngestRepo_CreateFallsBackToRkeyForName(t *testing.T) { + ing, _ := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:repo1"), + }) + + if err := ingestAcceptingOwner(t, ing, e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "myrepo") + if r.Name != "myrepo" { + t.Errorf("name should fall back to rkey: got %q, want %q", r.Name, "myrepo") + } +} + +func TestIngestRepo_CreateSquatRejected(t *testing.T) { + ing, spy := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:boltless", "squatrepo", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:akshays-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{ + RepoDid: "did:plc:akshays-repo", + OwnerDid: "did:plc:akshay", + KnotURL: mustKnotURL(t, "https://knot.example"), + }, nil)) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + if _, err := db.GetRepo(ing.Db, + orm.FilterEq("did", "did:plc:boltless"), + orm.FilterEq("rkey", "squatrepo"), + ); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("boltless's squat row should not exist, got err=%v", err) + } + if spy.creates != 0 { + t.Errorf("NewRepo called %d times despite rejection", spy.creates) + } +} + +func TestIngestRepo_CreateHijackExistingRepoRejected(t *testing.T) { + ing, spy := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "myrepo", "akshayskey", "did:plc:akshays-repo") + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:boltless", "takeover", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:akshays-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{ + RepoDid: "did:plc:akshays-repo", + OwnerDid: "did:plc:akshay", + KnotURL: mustKnotURL(t, "https://knot.example"), + }, nil)) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + akshay := loadRepo(t, ing, "did:plc:akshay", "akshayskey") + if akshay.Did != "did:plc:akshay" || akshay.Rkey != "akshayskey" { + t.Errorf("akshay's row mutated: %+v", akshay) + } + if spy.renames != 0 { + t.Errorf("RenameRepo called %d times despite rejection", spy.renames) + } +} + +func TestIngestRepo_CreateRenameIgnoresRkeyDrift(t *testing.T) { + ing, spy := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "oldname", "oldrkey", "did:plc:akshays-repo") + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "newrkey", tangled.Repo{ + Knot: "knot.example", + Name: ptr("newname"), + RepoDid: ptr("did:plc:akshays-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{ + RepoDid: "did:plc:akshays-repo", + OwnerDid: "did:plc:akshay", + KnotURL: mustKnotURL(t, "https://knot.example"), + }, nil)) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + r := loadRepo(t, ing, "did:plc:akshay", "newrkey") + if r.Name != "newname" { + t.Errorf("rename did not apply despite matching owner: name=%q", r.Name) + } + if spy.renames != 1 { + t.Errorf("RenameRepo called %d times, want 1", spy.renames) + } +} + +func TestIngestRepo_CreateVerifierTransientErrorPropagates(t *testing.T) { + ing, spy := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:akshays-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{}, errors.New("knot unreachable"))) + + err := ing.ingestRepo(context.Background(), e) + if err == nil { + t.Fatalf("expected error on transient verifier failure, got nil") + } + if spy.creates != 0 { + t.Errorf("NewRepo called %d times despite verifier error", spy.creates) + } +} + +func TestIngestRepo_UpdateRejectsOwnerMismatch(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "myrepo", "akshayskey", "did:plc:akshays-repo") + + e := makeEvent(t, jmodels.CommitOperationUpdate, "did:plc:boltless", "akshayskey", tangled.Repo{ + Knot: "knot.example", + Description: ptr("boltless hijacks metadata"), + RepoDid: ptr("did:plc:akshays-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{ + RepoDid: "did:plc:akshays-repo", + OwnerDid: "did:plc:akshay", + KnotURL: mustKnotURL(t, "https://knot.example"), + }, nil)) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + + akshay := loadRepo(t, ing, "did:plc:akshay", "akshayskey") + if akshay.Description == "boltless hijacks metadata" { + t.Errorf("update by non-owner applied: %+v", akshay) + } +} + +func TestIngestRepo_CreateInvalidRepoDidRejected(t *testing.T) { + ing, spy := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:"), + }) + + verifierCalled := false + withVerifier(ing, func(_ context.Context, _ repoverify.RepoDid) (repoverify.Result, error) { + verifierCalled = true + return repoverify.Result{}, nil + }) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + if verifierCalled { + t.Errorf("verifier was called with an invalid repoDid") + } + if spy.creates != 0 { + t.Errorf("NewRepo called %d times despite invalid repoDid", spy.creates) + } +} + +func TestIngestRepo_NilVerifierFailsClosed(t *testing.T) { + ing, spy := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:akshays-repo"), + }) + + err := ing.ingestRepo(context.Background(), e) + if err == nil { + t.Fatalf("expected error when Verifier is nil, got nil") + } + if spy.creates != 0 { + t.Errorf("NewRepo called %d times despite nil verifier", spy.creates) + } +} + +func TestIngestRepo_CreateRejectsKnotMismatch(t *testing.T) { + ing, spy := newTestIngester(t) + + e := makeEvent(t, jmodels.CommitOperationCreate, "did:plc:akshay", "myrepo", tangled.Repo{ + Knot: "evil.example", + RepoDid: ptr("did:plc:akshays-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{ + RepoDid: "did:plc:akshays-repo", + OwnerDid: "did:plc:akshay", + KnotURL: mustKnotURL(t, "https://knot.example"), + }, nil)) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + if _, err := db.GetRepo(ing.Db, + orm.FilterEq("did", "did:plc:akshay"), + orm.FilterEq("rkey", "myrepo"), + ); !errors.Is(err, sql.ErrNoRows) { + t.Fatalf("row should not be created for spoofed knot, err=%v", err) + } + if spy.creates != 0 { + t.Errorf("NewRepo called %d times despite knot mismatch", spy.creates) + } +} + +func TestIngestRepo_UpdateRejectsKnotMismatch(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "myrepo", "akshayskey", "did:plc:akshays-repo") + + e := makeEvent(t, jmodels.CommitOperationUpdate, "did:plc:akshay", "akshayskey", tangled.Repo{ + Knot: "evil.example", + Description: ptr("redirected clone target"), + RepoDid: ptr("did:plc:akshays-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{ + RepoDid: "did:plc:akshays-repo", + OwnerDid: "did:plc:akshay", + KnotURL: mustKnotURL(t, "https://knot.example"), + }, nil)) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + akshay := loadRepo(t, ing, "did:plc:akshay", "akshayskey") + if akshay.Description == "redirected clone target" { + t.Errorf("update with spoofed knot applied: %+v", akshay) + } + if akshay.Knot != "knot.example" { + t.Errorf("row knot mutated to %q, want knot.example", akshay.Knot) + } +} + +func TestIngestRepo_UpdateRejectsRepoDidMutation(t *testing.T) { + ing, _ := newTestIngester(t) + seedRepoRow(t, ing, "did:plc:akshay", "knot.example", "myrepo", "akshayskey", "did:plc:akshays-repo") + + e := makeEvent(t, jmodels.CommitOperationUpdate, "did:plc:akshay", "akshayskey", tangled.Repo{ + Knot: "knot.example", + Description: ptr("sneaky repoDid swap"), + RepoDid: ptr("did:plc:other-repo"), + }) + + withVerifier(ing, stubVerifier(repoverify.Result{ + RepoDid: "did:plc:other-repo", + OwnerDid: "did:plc:akshay", + KnotURL: mustKnotURL(t, "https://knot.example"), + }, nil)) + + if err := ing.ingestRepo(context.Background(), e); err != nil { + t.Fatalf("ingestRepo: %v", err) + } + akshay := loadRepo(t, ing, "did:plc:akshay", "akshayskey") + if akshay.RepoDid != "did:plc:akshays-repo" { + t.Errorf("repoDid mutated to %q, want did:plc:akshays-repo", akshay.RepoDid) + } + if akshay.Description == "sneaky repoDid swap" { + t.Errorf("metadata from repoDid-mutating update applied: %+v", akshay) + } +} diff --git a/appview/issues/issues.go b/appview/issues/issues.go --- a/appview/issues/issues.go +++ b/appview/issues/issues.go @@ -885,7 +885,7 @@ searchOpts := models.IssueSearchOptions{ Keywords: tf.Keywords, Phrases: tf.Phrases, - RepoAt: f.RepoAt().String(), + RepoDid: f.RepoDid, IsOpen: isOpen, AuthorDid: authorDid, Labels: labels, @@ -945,7 +945,7 @@ } } } else { filters := []orm.Filter{ - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), } if isOpen != nil { openInt := 0 @@ -1039,7 +1039,7 @@ body := r.FormValue("body") mentions, references := rp.mentionsResolver.Resolve(r.Context(), body) issue := &models.Issue{ - RepoAt: f.RepoAt(), + RepoDid: syntax.DID(f.RepoDid), Rkey: tid.TID(), Title: r.FormValue("title"), Body: body, diff --git a/appview/labels/labels.go b/appview/labels/labels.go --- a/appview/labels/labels.go +++ b/appview/labels/labels.go @@ -100,7 +100,7 @@ return } // find all the labels that this repo subscribes to - repoLabels, err := db.GetRepoLabels(l.db, orm.FilterEq("repo_at", repoAt)) + repoLabels, err := db.GetRepoLabels(l.db, orm.FilterEq("repo_did", repo.RepoDid)) if err != nil { fail("Failed to get labels for this repository.", err) return diff --git a/appview/metrics/metrics.go b/appview/metrics/metrics.go new file mode 100644 --- /dev/null +++ b/appview/metrics/metrics.go @@ -0,0 +1,19 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +var ( + HttpRequestsTotal = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "appview_http_requests_total", + Help: "Total number of HTTP requests", + }, []string{"method", "path", "status"}) + + HttpRequestDuration = promauto.NewHistogramVec(prometheus.HistogramOpts{ + Name: "appview_http_request_duration_seconds", + Help: "HTTP request duration in seconds", + Buckets: prometheus.DefBuckets, + }, []string{"method", "path", "status"}) +) diff --git a/appview/metrics/middleware.go b/appview/metrics/middleware.go new file mode 100644 --- /dev/null +++ b/appview/metrics/middleware.go @@ -0,0 +1,40 @@ +package metrics + +import ( + "fmt" + "net/http" + "time" + + "github.com/go-chi/chi/v5" +) + +type statusRecorder struct { + http.ResponseWriter + status int +} + +func (r *statusRecorder) WriteHeader(status int) { + r.status = status + r.ResponseWriter.WriteHeader(status) +} + +func Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK} + start := time.Now() + + next.ServeHTTP(rec, r) + + // use the matched route pattern to avoid high cardinality + routePattern := chi.RouteContext(r.Context()).RoutePattern() + if routePattern == "" { + routePattern = "unknown" + } + + status := fmt.Sprintf("%d", rec.status) + duration := time.Since(start).Seconds() + + HttpRequestsTotal.WithLabelValues(r.Method, routePattern, status).Inc() + HttpRequestDuration.WithLabelValues(r.Method, routePattern, status).Observe(duration) + }) +} diff --git a/appview/middleware/middleware.go b/appview/middleware/middleware.go --- a/appview/middleware/middleware.go +++ b/appview/middleware/middleware.go @@ -2,6 +2,8 @@ package middleware import ( "context" + "database/sql" + "errors" "fmt" "log/slog" "net/http" @@ -234,6 +236,7 @@ return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { l := mw.logger.With("middleware", "ResolveRepo") repoName := chi.URLParam(req, "repo") repoName = strings.TrimSuffix(repoName, ".git") + rkey := strings.ToLower(repoName) id, ok := req.Context().Value("resolvedId").(identity.Identity) if !ok { @@ -245,10 +248,53 @@ repo, err := db.GetRepo( mw.db, orm.FilterEq("did", id.DID.String()), - orm.FilterEq("name", repoName), + orm.FilterEq("rkey", rkey), ) if err != nil { - l.Error("failed to resolve repo", "err", err) + if !errors.Is(err, sql.ErrNoRows) { + l.Error("failed to resolve repo", "err", err) + http.Error(w, "internal server error", http.StatusInternalServerError) + return + } + hint, hintErr := db.LookupRepoRename(mw.db, id.DID.String(), rkey) + if hintErr != nil && !errors.Is(hintErr, sql.ErrNoRows) { + l.Error("failed to lookup repo rename hint", "err", hintErr) + } + if hint != nil { + parts := strings.SplitN(strings.TrimPrefix(req.URL.Path, "/"), "/", 3) + target := "/" + parts[0] + "/" + hint.Rkey + if len(parts) == 3 { + target += "/" + parts[2] + } + if req.URL.RawQuery != "" { + target += "?" + req.URL.RawQuery + } + http.Redirect(w, req, target, http.StatusMovedPermanently) + return + } + nameRepos, nameErr := db.GetRepos( + mw.db, + orm.FilterEq("did", id.DID.String()), + orm.FilterEq("name", repoName), + ) + if nameErr == nil && len(nameRepos) == 1 && nameRepos[0].RepoDid != "" { + nameRepo := &nameRepos[0] + if _, tidErr := syntax.ParseTID(nameRepo.Rkey); tidErr == nil { + ctx := context.WithValue(req.Context(), "repo", nameRepo) + next.ServeHTTP(w, req.WithContext(ctx)) + return + } + parts := strings.SplitN(strings.TrimPrefix(req.URL.Path, "/"), "/", 3) + target := "/" + nameRepo.RepoDid + if len(parts) == 3 { + target += "/" + parts[2] + } + if req.URL.RawQuery != "" { + target += "?" + req.URL.RawQuery + } + http.Redirect(w, req, target, http.StatusFound) + return + } w.WriteHeader(http.StatusNotFound) mw.pages.ErrorKnot404(w) return @@ -281,7 +327,7 @@ mw.pages.Error404(w) return } - pr, err := db.GetPull(mw.db, orm.FilterEq("repo_at", f.RepoAt()), orm.FilterEq("pull_id", prIdInt)) + pr, err := db.GetPull(mw.db, orm.FilterEq("repo_did", f.RepoDid), orm.FilterEq("pull_id", prIdInt)) if err != nil { l.Error("failed to get pull and comments", "err", err) mw.pages.Error404(w) @@ -324,7 +370,7 @@ mw.pages.Error404(w) return } - issue, err := db.GetIssue(mw.db, f.RepoAt(), issueId) + issue, err := db.GetIssue(mw.db, f.RepoDid, issueId) if err != nil { l.Error("failed to get issues", "err", err) mw.pages.Error404(w) @@ -360,7 +406,7 @@ if r.Header.Get("User-Agent") == "Go-http-client/1.1" { if r.URL.Query().Get("go-get") == "1" { modulePath := userutil.FlattenDid(fullName) if strings.Contains(modulePath, ":") { - modulePath = userutil.FlattenDid(f.Did) + "/" + f.Name + modulePath = userutil.FlattenDid(f.Did) + "/" + f.Rkey } html := fmt.Sprintf( ` diff --git a/appview/migration/migrate_add_repo_did.go b/appview/migration/migrate_add_repo_did.go --- a/appview/migration/migrate_add_repo_did.go +++ b/appview/migration/migrate_add_repo_did.go @@ -2,6 +2,7 @@ package migration import ( "context" + "encoding/json" "fmt" "strings" @@ -14,7 +15,10 @@ "tangled.org/core/appview/db" ) func (s *Migration) migrateAddRepoDid(ctx context.Context, client *atclient.APIClient, did syntax.DID, record syntax.ATURI) error { - // TODO: use agnostic.RepoGetRecord instead + if record.Collection().String() == tangled.FeedStarNSID { + return s.migrateAddRepoDidStar(ctx, client, did, record) + } + ex, err := comatproto.RepoGetRecord(ctx, client, "", record.Collection().String(), did.String(), record.RecordKey().String()) if err != nil { return fmt.Errorf("pds: %w", err) @@ -22,7 +26,7 @@ } val := ex.Value.Val - switch record.Collection() { + switch record.Collection().String() { case tangled.RepoNSID: rec, ok := val.(*tangled.Repo) if !ok { @@ -39,35 +43,35 @@ rec, ok := val.(*tangled.RepoIssue) if !ok { return fmt.Errorf("unexpected type for issue record") } - if rec.Repo != nil { - repoAt := *rec.Repo - repo, err := db.GetRepoByAtUri(s.db, repoAt) - if err != nil { - return fmt.Errorf("db: failed to query repo: %w", err) - } - rec.RepoDid = &repo.RepoDid + if strings.HasPrefix(rec.Repo, "did:") { + return nil + } + repo, err := db.GetRepoByAtUri(s.db, rec.Repo) + if err != nil { + return fmt.Errorf("db: failed to query repo by at_uri %q: %w", rec.Repo, err) } + rec.Repo = repo.RepoDid case tangled.RepoPullNSID: rec, ok := val.(*tangled.RepoPull) if !ok { return fmt.Errorf("unexpected type for pull record") } - if rec.Target != nil && rec.Target.Repo != nil { - repoAt := *rec.Target.Repo - repo, err := db.GetRepoByAtUri(s.db, repoAt) + if rec.Target == nil { + return fmt.Errorf("pull record has nil target") + } + if !strings.HasPrefix(rec.Target.Repo, "did:") { + repo, err := db.GetRepoByAtUri(s.db, rec.Target.Repo) if err != nil { - return fmt.Errorf("db: failed to query repo: %w", err) + return fmt.Errorf("db: failed to query target repo by at_uri %q: %w", rec.Target.Repo, err) } - rec.Target.RepoDid = &repo.RepoDid + rec.Target.Repo = repo.RepoDid } - if rec.Source != nil && rec.Source.Repo != nil { - repoAt := *rec.Source.Repo - repo, err := db.GetRepoByAtUri(s.db, repoAt) - if err != nil { - return fmt.Errorf("db: failed to query repo: %w", err) + if rec.Source != nil && rec.Source.Repo != nil && !strings.HasPrefix(*rec.Source.Repo, "did:") { + sourceRepo, srcErr := db.GetRepoByAtUri(s.db, *rec.Source.Repo) + if srcErr == nil && sourceRepo.RepoDid != "" { + rec.Source.Repo = &sourceRepo.RepoDid } - rec.Source.RepoDid = &repo.RepoDid } case tangled.RepoCollaboratorNSID: @@ -75,14 +79,14 @@ rec, ok := val.(*tangled.RepoCollaborator) if !ok { return fmt.Errorf("unexpected type for collaborator record") } - if rec.Repo != nil { - repoAt := *rec.Repo - repo, err := db.GetRepoByAtUri(s.db, repoAt) - if err != nil { - return fmt.Errorf("db: failed to query repo: %w", err) - } - rec.RepoDid = &repo.RepoDid + if strings.HasPrefix(rec.Repo, "did:") { + return nil + } + repo, err := db.GetRepoByAtUri(s.db, rec.Repo) + if err != nil { + return fmt.Errorf("db: failed to query repo by at_uri %q: %w", rec.Repo, err) } + rec.Repo = repo.RepoDid case tangled.RepoArtifactNSID: rec, ok := val.(*tangled.RepoArtifact) @@ -90,28 +94,13 @@ if !ok { return fmt.Errorf("unexpected type for artifact record") } if rec.Repo != nil { - repoAt := *rec.Repo - repo, err := db.GetRepoByAtUri(s.db, repoAt) + repo, err := db.GetRepoByAtUri(s.db, *rec.Repo) if err != nil { - return fmt.Errorf("db: failed to query repo: %w", err) + return fmt.Errorf("db: failed to query repo by at_uri %q: %w", *rec.Repo, err) } rec.RepoDid = &repo.RepoDid } - case tangled.FeedStarNSID: - rec, ok := val.(*tangled.FeedStar) - if !ok { - return fmt.Errorf("unexpected type for star record") - } - if rec.Subject != nil { - repoAt := *rec.Subject - repo, err := db.GetRepoByAtUri(s.db, repoAt) - if err != nil { - return fmt.Errorf("db: failed to query repo: %w", err) - } - rec.SubjectDid = &repo.RepoDid - } - case tangled.ActorProfileNSID: rec, ok := val.(*tangled.ActorProfile) if !ok { @@ -149,3 +138,57 @@ } return nil } + +func (s *Migration) migrateAddRepoDidStar(ctx context.Context, client *atclient.APIClient, did syntax.DID, record syntax.ATURI) error { + var raw struct { + Cid *string `json:"cid,omitempty"` + Uri string `json:"uri"` + Value json.RawMessage `json:"value"` + } + params := map[string]any{ + "collection": record.Collection().String(), + "repo": did.String(), + "rkey": record.RecordKey().String(), + } + if err := client.LexDo(ctx, lexutil.Query, "", "com.atproto.repo.getRecord", params, nil, &raw); err != nil { + return fmt.Errorf("get record: %w", err) + } + + var legacy struct { + CreatedAt string `json:"createdAt"` + Subject *string `json:"subject,omitempty"` + } + if err := json.Unmarshal(raw.Value, &legacy); err != nil { + return fmt.Errorf("decode old star fields: %w", err) + } + if legacy.Subject == nil { + return fmt.Errorf("star record has no subject field") + } + + repo, err := db.GetRepoByAtUri(s.db, *legacy.Subject) + if err != nil { + return fmt.Errorf("db: failed to query repo by at_uri %q: %w", *legacy.Subject, err) + } + if repo.RepoDid == "" { + return fmt.Errorf("repo has no repoDid: %s", *legacy.Subject) + } + + newRecord := &tangled.FeedStar{ + CreatedAt: legacy.CreatedAt, + Subject: &tangled.FeedStar_Subject{ + FeedStar_Repo: &tangled.FeedStar_Repo{Did: repo.RepoDid}, + }, + } + + _, err = comatproto.RepoPutRecord(ctx, client, &comatproto.RepoPutRecord_Input{ + Repo: did.String(), + Collection: record.Collection().String(), + Rkey: record.RecordKey().String(), + SwapRecord: raw.Cid, + Record: &lexutil.LexiconTypeDecoder{Val: newRecord}, + }) + if err != nil { + return fmt.Errorf("put record: %w", err) + } + return nil +} diff --git a/appview/models/artifact.go b/appview/models/artifact.go --- a/appview/models/artifact.go +++ b/appview/models/artifact.go @@ -15,7 +15,7 @@ Id uint64 Did string Rkey string - RepoAt syntax.ATURI + RepoDid syntax.DID Tag plumbing.Hash CreatedAt time.Time diff --git a/appview/models/collaborator.go b/appview/models/collaborator.go --- a/appview/models/collaborator.go +++ b/appview/models/collaborator.go @@ -14,7 +14,7 @@ Rkey string // content SubjectDid syntax.DID - RepoAt syntax.ATURI + RepoDid syntax.DID // meta Created time.Time diff --git a/appview/models/issue.go b/appview/models/issue.go --- a/appview/models/issue.go +++ b/appview/models/issue.go @@ -13,7 +13,7 @@ type Issue struct { Id int64 Did string Rkey string - RepoAt syntax.ATURI + RepoDid syntax.DID IssueId int Created time.Time Edited *time.Time @@ -44,17 +44,13 @@ references := make([]string, len(i.References)) for i, uri := range i.References { references[i] = string(uri) } - repoAtStr := i.RepoAt.String() rec := tangled.RepoIssue{ - Repo: &repoAtStr, + Repo: string(i.RepoDid), Title: i.Title, Body: &i.Body, Mentions: mentions, References: references, CreatedAt: i.Created.Format(time.RFC3339), - } - if i.Repo != nil && i.Repo.RepoDid != "" { - rec.RepoDid = &i.Repo.RepoDid } return rec } @@ -166,13 +162,8 @@ if record.Body != nil { body = *record.Body } - var repoAt syntax.ATURI - if record.Repo != nil { - repoAt = syntax.ATURI(*record.Repo) - } - return Issue{ - RepoAt: repoAt, + RepoDid: syntax.DID(record.Repo), Did: did, Rkey: rkey, Created: created, diff --git a/appview/models/language.go b/appview/models/language.go --- a/appview/models/language.go +++ b/appview/models/language.go @@ -1,12 +1,10 @@ package models -import ( - "github.com/bluesky-social/indigo/atproto/syntax" -) +import "github.com/bluesky-social/indigo/atproto/syntax" type RepoLanguage struct { Id int64 - RepoAt syntax.ATURI + RepoDid syntax.DID Ref string IsDefaultRef bool Language string diff --git a/appview/models/pull.go b/appview/models/pull.go --- a/appview/models/pull.go +++ b/appview/models/pull.go @@ -61,7 +61,7 @@ ID int PullId int // at ids - RepoAt syntax.ATURI + RepoDid syntax.DID OwnerDid string Rkey string @@ -97,14 +97,6 @@ for i, uri := range p.References { references[i] = string(uri) } - var targetRepoAt, targetRepoDid *string - targetRepoAt = new(string) - *targetRepoAt = p.RepoAt.String() - if p.Repo != nil && p.Repo.RepoDid != "" { - targetRepoDid = new(string) - *targetRepoDid = p.Repo.RepoDid - } - rounds := make([]*tangled.RepoPull_Round, len(p.Submissions)) for i, submission := range p.Submissions { rounds[i] = submission.AsRecord() @@ -123,9 +115,8 @@ Mentions: mentions, References: references, CreatedAt: p.Created.Format(time.RFC3339), Target: &tangled.RepoPull_Target{ - Repo: targetRepoAt, - RepoDid: targetRepoDid, - Branch: p.TargetBranch, + Repo: string(p.RepoDid), + Branch: p.TargetBranch, }, Rounds: rounds, Source: p.PullSource.AsRecord(), @@ -151,16 +142,14 @@ mentions = append(mentions, did) } } - var targetRepoAt syntax.ATURI + var targetRepoDid syntax.DID var targetBranch string if record.Target != nil { - if record.Target.Repo != nil { - uri, err := syntax.ParseATURI(*record.Target.Repo) - if err != nil { - return nil, fmt.Errorf("invalid target.repo aturi: %w", err) - } - targetRepoAt = uri + did, err := syntax.ParseDID(record.Target.Repo) + if err != nil { + return nil, fmt.Errorf("invalid target.repo did: %w", err) } + targetRepoDid = did targetBranch = record.Target.Branch } @@ -171,16 +160,9 @@ Branch: record.Source.Branch, } if record.Source.Repo != nil { - uri, err := syntax.ParseATURI(*record.Source.Repo) + did, err := syntax.ParseDID(*record.Source.Repo) if err != nil { - return nil, fmt.Errorf("invalid source.repo aturi: %w", err) - } - pullSource.RepoAt = &uri - } - if record.Source.RepoDid != nil { - did, err := syntax.ParseDID(*record.Source.RepoDid) - if err != nil { - return nil, fmt.Errorf("invalid source.repoDid did: %w", err) + return nil, fmt.Errorf("invalid source.repo did: %w", err) } pullSource.RepoDid = &did } @@ -209,7 +191,7 @@ submissions = append(submissions, submission) } return &Pull{ - RepoAt: targetRepoAt, + RepoDid: targetRepoDid, OwnerDid: did, Rkey: rkey, Title: record.Title, @@ -260,7 +242,6 @@ } type PullSource struct { Branch string - RepoAt *syntax.ATURI RepoDid *syntax.DID // optionally populate this for reverse mappings @@ -271,19 +252,14 @@ func (s *PullSource) AsRecord() *tangled.RepoPull_Source { if s == nil { return nil } - var repoAt, repoDid *string - if s.RepoAt != nil { - repoAt = new(string) - *repoAt = s.RepoAt.String() - } + var repo *string if s.RepoDid != nil { - repoDid = new(string) - *repoDid = s.RepoDid.String() + r := s.RepoDid.String() + repo = &r } return &tangled.RepoPull_Source{ - Branch: s.Branch, - Repo: repoAt, - RepoDid: repoDid, + Branch: s.Branch, + Repo: repo, } } @@ -313,7 +289,7 @@ PullId int SubmissionId int // at ids - RepoAt string + RepoDid string OwnerDid string CommentAt string @@ -366,21 +342,20 @@ } func (p *Pull) IsBranchBased() bool { if p.PullSource != nil { - if p.PullSource.RepoAt != nil { - return p.PullSource.RepoAt == &p.RepoAt - } else { - // no repo specified - return true + if p.PullSource.RepoDid != nil { + return *p.PullSource.RepoDid == p.RepoDid } + // no repo specified + return true } return false } func (p *Pull) IsForkBased() bool { if p.PullSource != nil { - if p.PullSource.RepoAt != nil { + if p.PullSource.RepoDid != nil { // make sure repos are different - return p.PullSource.RepoAt != &p.RepoAt + return *p.PullSource.RepoDid != p.RepoDid } } return false diff --git a/appview/models/repo.go b/appview/models/repo.go --- a/appview/models/repo.go +++ b/appview/models/repo.go @@ -57,7 +57,7 @@ } return tangled.Repo{ Knot: r.Knot, - Name: r.Name, + Name: r.cosmeticName(), Description: description, Website: website, Topics: r.Topics, @@ -69,6 +69,13 @@ RepoDid: repoDid, } } +func (r *Repo) cosmeticName() *string { + if r.Name == "" || r.Name == r.Rkey { + return nil + } + return &r.Name +} + func (r Repo) RepoAt() syntax.ATURI { return syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", r.Did, tangled.RepoNSID, r.Rkey)) } @@ -77,7 +84,7 @@ func (r Repo) RepoIdentifier() string { if r.RepoDid != "" { return r.RepoDid } - p, _ := securejoin.SecureJoin(r.Did, r.Name) + p, _ := securejoin.SecureJoin(r.Did, r.Rkey) return p } @@ -97,6 +104,7 @@ Language string StarCount int IssueCount IssueCount PullCount PullCount + ForkCount int } type IssueCount struct { @@ -113,8 +121,57 @@ } type RepoLabel struct { Id int64 - RepoAt syntax.ATURI + RepoDid syntax.DID LabelAt syntax.ATURI +} + +var reservedRepoNames = map[string]struct{}{ + "self": {}, +} + +func ValidateRepoName(name string) error { + if len(name) == 0 { + return fmt.Errorf("Repository name cannot be empty") + } + if len(name) > 100 { + return fmt.Errorf("Repository name must be 100 characters or fewer") + } + + // check for path traversal attempts + if strings.Contains(name, "/") || strings.Contains(name, "\\") { + return fmt.Errorf("Repository name contains invalid path characters") + } + + // check for sequences that could be used for traversal when normalized + if strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".") { + return fmt.Errorf("Repository name contains invalid path sequence") + } + + // then continue with character validation + for _, char := range name { + if !((char >= 'a' && char <= 'z') || + (char >= 'A' && char <= 'Z') || + (char >= '0' && char <= '9') || + char == '-' || char == '_' || char == '.') { + return fmt.Errorf("Repository name can only contain alphanumeric characters, periods, hyphens, and underscores") + } + } + + // additional check to prevent multiple sequential dots + if strings.Contains(name, "..") { + return fmt.Errorf("Repository name cannot contain sequential dots") + } + + if _, reserved := reservedRepoNames[strings.ToLower(name)]; reserved { + return fmt.Errorf("Repository name %q is reserved", name) + } + + // if all checks pass + return nil +} + +func StripGitExt(name string) string { + return strings.TrimSuffix(name, ".git") } type RepoGroup struct { diff --git a/appview/models/repo_test.go b/appview/models/repo_test.go new file mode 100644 --- /dev/null +++ b/appview/models/repo_test.go @@ -0,0 +1,86 @@ +package models + +import ( + "strings" + "testing" +) + +func TestValidateRepoName_ValidRkeys(t *testing.T) { + valid := []string{ + "myrepo", + "MyRepo", + "my-repo", + "my_repo", + "my.repo", + "a", + "repo123", + strings.Repeat("a", 100), + } + for _, name := range valid { + if err := ValidateRepoName(name); err != nil { + t.Errorf("ValidateRepoName(%q) = %v, want nil", name, err) + } + } +} + +func TestValidateRepoName_InvalidRkeys(t *testing.T) { + cases := []struct { + input string + substr string + }{ + {"", "empty"}, + {strings.Repeat("a", 101), "100 characters"}, + {"has space", "alphanumeric"}, + {"has/slash", "invalid path"}, + {"has\\backslash", "invalid path"}, + {".dotprefix", "invalid path"}, + {"dotsuffix.", "invalid path"}, + {"two..dots", "sequential dots"}, + {"../traversal", "invalid path"}, + {"self", "reserved"}, + {"SELF", "reserved"}, + } + for _, tc := range cases { + err := ValidateRepoName(tc.input) + if err == nil { + t.Errorf("ValidateRepoName(%q) = nil, want error containing %q", tc.input, tc.substr) + continue + } + if !strings.Contains(strings.ToLower(err.Error()), strings.ToLower(tc.substr)) { + t.Errorf("ValidateRepoName(%q) = %q, want substring %q", tc.input, err.Error(), tc.substr) + } + } +} + +func TestStripGitExt(t *testing.T) { + cases := []struct{ in, want string }{ + {"repo.git", "repo"}, + {"repo", "repo"}, + {"repo.git.git", "repo.git"}, + {".git", ""}, + } + for _, tc := range cases { + if got := StripGitExt(tc.in); got != tc.want { + t.Errorf("StripGitExt(%q) = %q, want %q", tc.in, got, tc.want) + } + } +} + +func TestCosmeticName_NilWhenMatchesRkey(t *testing.T) { + r := Repo{Name: "myrepo", Rkey: "myrepo"} + rec := r.AsRecord() + if rec.Name != nil { + t.Errorf("cosmeticName should be nil when Name == Rkey, got %q", *rec.Name) + } +} + +func TestCosmeticName_PresentWhenDiffers(t *testing.T) { + r := Repo{Name: "MyRepo", Rkey: "myrepo", Knot: "k"} + rec := r.AsRecord() + if rec.Name == nil { + t.Fatal("cosmeticName should be non-nil when Name != Rkey") + } + if *rec.Name != "MyRepo" { + t.Errorf("cosmeticName = %q, want %q", *rec.Name, "MyRepo") + } +} diff --git a/appview/models/search.go b/appview/models/search.go --- a/appview/models/search.go +++ b/appview/models/search.go @@ -5,7 +5,7 @@ type IssueSearchOptions struct { Keywords []string Phrases []string - RepoAt string + RepoDid string IsOpen *bool AuthorDid string Labels []string @@ -31,7 +31,7 @@ type PullSearchOptions struct { Keywords []string Phrases []string - RepoAt string + RepoDid string State *PullState AuthorDid string Labels []string diff --git a/appview/models/search_test.go b/appview/models/search_test.go --- a/appview/models/search_test.go +++ b/appview/models/search_test.go @@ -16,8 +16,8 @@ opts: IssueSearchOptions{}, want: false, }, { - name: "non-filter fields only (RepoAt, IsOpen, Page) return false", - opts: IssueSearchOptions{RepoAt: "at://did:plc:abc/repo"}, + name: "non-filter fields only (RepoDid, IsOpen, Page) return false", + opts: IssueSearchOptions{RepoDid: "did:plc:abc"}, want: false, }, { @@ -93,8 +93,8 @@ opts: PullSearchOptions{}, want: false, }, { - name: "non-filter fields only (RepoAt, State, Page) return false", - opts: PullSearchOptions{RepoAt: "at://did:plc:abc/repo"}, + name: "non-filter fields only (RepoDid, State, Page) return false", + opts: PullSearchOptions{RepoDid: "did:plc:abc"}, want: false, }, { diff --git a/appview/models/site_deploy.go b/appview/models/site_deploy.go --- a/appview/models/site_deploy.go +++ b/appview/models/site_deploy.go @@ -1,6 +1,10 @@ package models -import "time" +import ( + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) type SiteDeployStatus string @@ -29,7 +33,7 @@ } type SiteDeploy struct { Id int64 - RepoAt string + RepoDid syntax.DID Branch string Dir string CommitSHA string diff --git a/appview/models/sites.go b/appview/models/sites.go --- a/appview/models/sites.go +++ b/appview/models/sites.go @@ -1,6 +1,10 @@ package models -import "time" +import ( + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) type DomainClaim struct { ID int64 @@ -11,8 +15,8 @@ } type RepoSite struct { ID int64 - RepoAt string - RepoName string // populated when joined with repos table + RepoDid syntax.DID + RepoRkey string // populated when joined with repos table Branch string Dir string IsIndex bool diff --git a/appview/models/star.go b/appview/models/star.go --- a/appview/models/star.go +++ b/appview/models/star.go @@ -2,15 +2,21 @@ package models import ( "time" +) - "github.com/bluesky-social/indigo/atproto/syntax" +type StarSubjectType string + +const ( + StarSubjectRepo StarSubjectType = "repo" + StarSubjectString StarSubjectType = "string" ) type Star struct { - Did string - RepoAt syntax.ATURI - Created time.Time - Rkey string + Did string + SubjectType StarSubjectType + Subject string + Created time.Time + Rkey string } // RepoStar is used for reverse mapping to repos diff --git a/appview/models/webhook.go b/appview/models/webhook.go --- a/appview/models/webhook.go +++ b/appview/models/webhook.go @@ -10,12 +10,13 @@ type WebhookEvent string const ( - WebhookEventPush WebhookEvent = "push" + WebhookEventPush WebhookEvent = "push" + WebhookEventRepoRenamed WebhookEvent = "repository:renamed" ) type Webhook struct { Id int64 - RepoAt syntax.ATURI + RepoDid syntax.DID Url string Secret string Active bool @@ -72,3 +73,11 @@ // WebhookUser represents user information in webhook payload type WebhookUser struct { Did string `json:"did"` } + +// WebhookRenamePayload represents the payload for a repository:renamed event +type WebhookRenamePayload struct { + OldName string `json:"old_name"` + NewName string `json:"new_name"` + Repository WebhookRepository `json:"repository"` + Sender WebhookUser `json:"sender"` +} diff --git a/appview/notify/db/db.go b/appview/notify/db/db.go --- a/appview/notify/db/db.go +++ b/appview/notify/db/db.go @@ -5,7 +5,6 @@ "context" "slices" "github.com/bluesky-social/indigo/atproto/syntax" - "tangled.org/core/api/tangled" "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/notify" @@ -40,15 +39,17 @@ func (n *databaseNotifier) DeleteRepo(ctx context.Context, repo *models.Repo) { // no-op for now } +func (n *databaseNotifier) RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) { +} + func (n *databaseNotifier) NewStar(ctx context.Context, star *models.Star) { l := log.FromContext(ctx) - if star.RepoAt.Collection().String() != tangled.RepoNSID { - // skip string stars for now + if star.SubjectType != models.StarSubjectRepo { return } - var err error - repo, err := db.GetRepo(n.db, orm.FilterEq("at_uri", string(star.RepoAt))) + + repo, err := db.GetRepo(n.db, orm.FilterEq("repo_did", star.Subject)) if err != nil { l.Error("failed to get repos", "err", err) return @@ -58,7 +59,7 @@ actorDid := syntax.DID(star.Did) recipients := sets.Singleton(syntax.DID(repo.Did)) eventType := models.NotificationTypeRepoStarred entityType := "repo" - entityId := star.RepoAt.String() + entityId := star.Subject repoId := &repo.Id var issueId *int64 var pullId *int64 @@ -83,7 +84,7 @@ func (n *databaseNotifier) NewIssue(ctx context.Context, issue *models.Issue, mentions []syntax.DID) { l := log.FromContext(ctx) - collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_at", issue.Repo.RepoAt())) + collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_did", string(issue.RepoDid))) if err != nil { l.Error("failed to fetch collaborators", "err", err) return @@ -240,12 +241,12 @@ func (n *databaseNotifier) NewPull(ctx context.Context, pull *models.Pull) { l := log.FromContext(ctx) - repo, err := db.GetRepo(n.db, orm.FilterEq("at_uri", string(pull.RepoAt))) + repo, err := db.GetRepo(n.db, orm.FilterEq("repo_did", string(pull.RepoDid))) if err != nil { l.Error("failed to get repos", "err", err) return } - collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_at", repo.RepoAt())) + collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_did", string(pull.RepoDid))) if err != nil { l.Error("failed to fetch collaborators", "err", err) return @@ -285,7 +286,7 @@ func (n *databaseNotifier) NewPullComment(ctx context.Context, comment *models.PullComment, mentions []syntax.DID) { l := log.FromContext(ctx) pull, err := db.GetPull(n.db, - orm.FilterEq("repo_at", syntax.ATURI(comment.RepoAt)), + orm.FilterEq("repo_did", comment.RepoDid), orm.FilterEq("pull_id", comment.PullId), ) if err != nil { @@ -293,7 +294,7 @@ l.Error("failed to get pulls", "err", err) return } - repo, err := db.GetRepo(n.db, orm.FilterEq("at_uri", comment.RepoAt)) + repo, err := db.GetRepo(n.db, orm.FilterEq("repo_did", comment.RepoDid)) if err != nil { l.Error("failed to get repos", "err", err) return @@ -371,7 +372,7 @@ func (n *databaseNotifier) NewIssueState(ctx context.Context, actor syntax.DID, issue *models.Issue) { l := log.FromContext(ctx) - collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_at", issue.Repo.RepoAt())) + collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_did", string(issue.RepoDid))) if err != nil { l.Error("failed to fetch collaborators", "err", err) return @@ -419,13 +420,13 @@ func (n *databaseNotifier) NewPullState(ctx context.Context, actor syntax.DID, pull *models.Pull) { l := log.FromContext(ctx) // Get repo details - repo, err := db.GetRepo(n.db, orm.FilterEq("at_uri", string(pull.RepoAt))) + repo, err := db.GetRepo(n.db, orm.FilterEq("repo_did", string(pull.RepoDid))) if err != nil { l.Error("failed to get repos", "err", err) return } - collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_at", repo.RepoAt())) + collaborators, err := db.GetCollaborators(n.db, orm.FilterEq("repo_did", string(pull.RepoDid))) if err != nil { l.Error("failed to fetch collaborators", "err", err) return diff --git a/appview/notify/logging/notifier.go b/appview/notify/logging/notifier.go --- a/appview/notify/logging/notifier.go +++ b/appview/notify/logging/notifier.go @@ -31,6 +31,11 @@ ctx = tlog.IntoContext(ctx, tlog.SubLogger(l.logger, "DeleteRepo")) l.inner.DeleteRepo(ctx, repo) } +func (l *loggingNotifier) RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) { + ctx = tlog.IntoContext(ctx, tlog.SubLogger(l.logger, "RenameRepo")) + l.inner.RenameRepo(ctx, actor, oldRepo, newRepo) +} + func (l *loggingNotifier) NewStar(ctx context.Context, star *models.Star) { ctx = tlog.IntoContext(ctx, tlog.SubLogger(l.logger, "NewStar")) l.inner.NewStar(ctx, star) diff --git a/appview/notify/merged_notifier.go b/appview/notify/merged_notifier.go --- a/appview/notify/merged_notifier.go +++ b/appview/notify/merged_notifier.go @@ -38,6 +38,10 @@ func (m *mergedNotifier) DeleteRepo(ctx context.Context, repo *models.Repo) { m.fanout(func(n Notifier) { n.DeleteRepo(ctx, repo) }) } +func (m *mergedNotifier) RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) { + m.fanout(func(n Notifier) { n.RenameRepo(ctx, actor, oldRepo, newRepo) }) +} + func (m *mergedNotifier) NewStar(ctx context.Context, star *models.Star) { m.fanout(func(n Notifier) { n.NewStar(ctx, star) }) } diff --git a/appview/notify/notifier.go b/appview/notify/notifier.go --- a/appview/notify/notifier.go +++ b/appview/notify/notifier.go @@ -10,6 +10,7 @@ type Notifier interface { NewRepo(ctx context.Context, repo *models.Repo) DeleteRepo(ctx context.Context, repo *models.Repo) + RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) NewStar(ctx context.Context, star *models.Star) DeleteStar(ctx context.Context, star *models.Star) @@ -47,6 +48,8 @@ var _ Notifier = &BaseNotifier{} func (m *BaseNotifier) NewRepo(ctx context.Context, repo *models.Repo) {} func (m *BaseNotifier) DeleteRepo(ctx context.Context, repo *models.Repo) {} +func (m *BaseNotifier) RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) { +} func (m *BaseNotifier) NewStar(ctx context.Context, star *models.Star) {} func (m *BaseNotifier) DeleteStar(ctx context.Context, star *models.Star) {} diff --git a/appview/notify/posthog/notifier.go b/appview/notify/posthog/notifier.go --- a/appview/notify/posthog/notifier.go +++ b/appview/notify/posthog/notifier.go @@ -35,11 +35,30 @@ log.Println("failed to enqueue posthog event:", err) } } +func (n *posthogNotifier) RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) { + err := n.client.Enqueue(posthog.Capture{ + DistinctId: actor.String(), + Event: "repo_renamed", + Properties: posthog.Properties{ + "repo_at": newRepo.RepoAt(), + "owner": newRepo.Did, + "old_name": oldRepo.Name, + "new_name": newRepo.Name, + }, + }) + if err != nil { + log.Println("failed to enqueue posthog event:", err) + } +} + func (n *posthogNotifier) NewStar(ctx context.Context, star *models.Star) { err := n.client.Enqueue(posthog.Capture{ DistinctId: star.Did, Event: "star", - Properties: posthog.Properties{"repo_at": star.RepoAt.String()}, + Properties: posthog.Properties{ + "subject_type": string(star.SubjectType), + "subject": star.Subject, + }, }) if err != nil { log.Println("failed to enqueue posthog event:", err) @@ -50,7 +69,10 @@ func (n *posthogNotifier) DeleteStar(ctx context.Context, star *models.Star) { err := n.client.Enqueue(posthog.Capture{ DistinctId: star.Did, Event: "unstar", - Properties: posthog.Properties{"repo_at": star.RepoAt.String()}, + Properties: posthog.Properties{ + "subject_type": string(star.SubjectType), + "subject": star.Subject, + }, }) if err != nil { log.Println("failed to enqueue posthog event:", err) @@ -62,7 +84,7 @@ err := n.client.Enqueue(posthog.Capture{ DistinctId: issue.Did, Event: "new_issue", Properties: posthog.Properties{ - "repo_at": issue.RepoAt.String(), + "repo_did": string(issue.RepoDid), "issue_id": issue.IssueId, "mentions": mentions, }, @@ -77,8 +99,8 @@ err := n.client.Enqueue(posthog.Capture{ DistinctId: pull.OwnerDid, Event: "new_pull", Properties: posthog.Properties{ - "repo_at": pull.RepoAt, - "pull_id": pull.PullId, + "repo_did": string(pull.RepoDid), + "pull_id": pull.PullId, }, }) if err != nil { @@ -91,7 +113,7 @@ err := n.client.Enqueue(posthog.Capture{ DistinctId: comment.OwnerDid, Event: "new_pull_comment", Properties: posthog.Properties{ - "repo_at": comment.RepoAt, + "repo_did": comment.RepoDid, "pull_id": comment.PullId, "mentions": mentions, }, @@ -106,8 +128,8 @@ err := n.client.Enqueue(posthog.Capture{ DistinctId: pull.OwnerDid, Event: "pull_closed", Properties: posthog.Properties{ - "repo_at": pull.RepoAt, - "pull_id": pull.PullId, + "repo_did": string(pull.RepoDid), + "pull_id": pull.PullId, }, }) if err != nil { @@ -216,7 +238,7 @@ err := n.client.Enqueue(posthog.Capture{ DistinctId: issue.Did, Event: event, Properties: posthog.Properties{ - "repo_at": issue.RepoAt.String(), + "repo_did": string(issue.RepoDid), "actor": actor, "issue_id": issue.IssueId, }, @@ -243,9 +265,9 @@ err := n.client.Enqueue(posthog.Capture{ DistinctId: pull.OwnerDid, Event: event, Properties: posthog.Properties{ - "repo_at": pull.RepoAt, - "pull_id": pull.PullId, - "actor": actor, + "repo_did": string(pull.RepoDid), + "pull_id": pull.PullId, + "actor": actor, }, }) if err != nil { diff --git a/appview/notify/webhook/notifier.go b/appview/notify/webhook/notifier.go --- a/appview/notify/webhook/notifier.go +++ b/appview/notify/webhook/notifier.go @@ -14,6 +14,7 @@ "net/http" "time" "github.com/avast/retry-go/v4" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/google/uuid" "tangled.org/core/appview/db" "tangled.org/core/appview/models" @@ -41,57 +42,85 @@ var _ notify.Notifier = &Notifier{} func (w *Notifier) Push(ctx context.Context, repo *models.Repo, ref, oldSha, newSha, committerDid string) { - webhooks, err := db.GetActiveWebhooksForRepo(w.db, repo.RepoAt()) + webhooks, err := w.activeWebhooksForEvent(repo.RepoDid, models.WebhookEventPush) + if err != nil { + w.logger.Error("failed to get webhooks for repo", "repo_did", repo.RepoDid, "err", err) + return + } + if len(webhooks) == 0 { + return + } + + payload := w.buildPushPayload(repo, ref, oldSha, newSha, committerDid) + payloadBytes, err := json.Marshal(payload) if err != nil { - w.logger.Error("failed to get webhooks for repo", "repo", repo.RepoAt(), "err", err) + w.logger.Error("failed to marshal push payload", "repo_did", repo.RepoDid, "err", err) return } - var pushWebhooks []models.Webhook + userAgent := "Tangled-Hook/" + newSha[:7] for _, webhook := range webhooks { - if webhook.HasEvent(models.WebhookEventPush) { - pushWebhooks = append(pushWebhooks, webhook) - } + go w.sendWebhook(ctx, webhook, string(models.WebhookEventPush), payload.Repository.FullName, userAgent, payloadBytes) } +} - if len(pushWebhooks) == 0 { +func (w *Notifier) RenameRepo(ctx context.Context, actor syntax.DID, oldRepo, newRepo *models.Repo) { + webhooks, err := w.activeWebhooksForEvent(newRepo.RepoDid, models.WebhookEventRepoRenamed) + if err != nil { + w.logger.Error("failed to get webhooks for repo", "repo_did", newRepo.RepoDid, "err", err) + return + } + if len(webhooks) == 0 { return } - payload, err := w.buildPushPayload(repo, ref, oldSha, newSha, committerDid) + payload := &models.WebhookRenamePayload{ + OldName: oldRepo.Name, + NewName: newRepo.Name, + Repository: buildWebhookRepository(newRepo), + Sender: models.WebhookUser{Did: actor.String()}, + } + payloadBytes, err := json.Marshal(payload) if err != nil { - w.logger.Error("failed to build push payload", "repo", repo.RepoAt(), "err", err) + w.logger.Error("failed to marshal rename payload", "repo_did", newRepo.RepoDid, "err", err) return } - for _, webhook := range pushWebhooks { - go w.sendWebhook(ctx, webhook, string(models.WebhookEventPush), payload) + userAgent := "Tangled-Hook/rename" + for _, webhook := range webhooks { + go w.sendWebhook(ctx, webhook, string(models.WebhookEventRepoRenamed), payload.Repository.FullName, userAgent, payloadBytes) } } -func (w *Notifier) buildPushPayload(repo *models.Repo, ref, oldSha, newSha, committerDid string) (*models.WebhookPayload, error) { - owner := repo.Did - - pusher := committerDid - if committerDid == "" { - pusher = owner +func (w *Notifier) activeWebhooksForEvent(repoDid string, event models.WebhookEvent) ([]models.Webhook, error) { + webhooks, err := db.GetActiveWebhooksForRepo(w.db, repoDid) + if err != nil { + return nil, err } + var matching []models.Webhook + for _, webhook := range webhooks { + if webhook.HasEvent(event) { + matching = append(matching, webhook) + } + } + return matching, nil +} +func buildWebhookRepository(repo *models.Repo) models.WebhookRepository { repository := models.WebhookRepository{ Name: repo.Name, - FullName: fmt.Sprintf("%s/%s", repo.Did, repo.Name), + FullName: fmt.Sprintf("%s/%s", repo.Did, repo.Rkey), Description: repo.Description, Fork: repo.Source != "", - HtmlUrl: fmt.Sprintf("https://%s/%s/%s", repo.Knot, repo.Did, repo.Name), - CloneUrl: fmt.Sprintf("https://%s/%s/%s", repo.Knot, repo.Did, repo.Name), - SshUrl: fmt.Sprintf("ssh://git@%s/%s/%s", repo.Knot, repo.Did, repo.Name), + HtmlUrl: fmt.Sprintf("https://%s/%s/%s", repo.Knot, repo.Did, repo.Rkey), + CloneUrl: fmt.Sprintf("https://%s/%s/%s", repo.Knot, repo.Did, repo.Rkey), + SshUrl: fmt.Sprintf("ssh://git@%s/%s/%s", repo.Knot, repo.Did, repo.Rkey), CreatedAt: repo.Created.Format(time.RFC3339), UpdatedAt: repo.Created.Format(time.RFC3339), Owner: models.WebhookUser{ - Did: owner, + Did: repo.Did, }, } - if repo.Website != "" { repository.Website = repo.Website } @@ -99,43 +128,40 @@ if repo.RepoStats != nil { repository.StarsCount = repo.RepoStats.StarCount repository.OpenIssues = repo.RepoStats.IssueCount.Open } + return repository +} - payload := &models.WebhookPayload{ +func (w *Notifier) buildPushPayload(repo *models.Repo, ref, oldSha, newSha, committerDid string) *models.WebhookPayload { + pusher := committerDid + if committerDid == "" { + pusher = repo.Did + } + return &models.WebhookPayload{ Ref: ref, Before: oldSha, After: newSha, - Repository: repository, + Repository: buildWebhookRepository(repo), Pusher: models.WebhookUser{ Did: pusher, }, } - - return payload, nil } -func (w *Notifier) sendWebhook(ctx context.Context, webhook models.Webhook, event string, payload *models.WebhookPayload) { +func (w *Notifier) sendWebhook(ctx context.Context, webhook models.Webhook, event, repoFullName, userAgent string, payloadBytes []byte) { deliveryId := uuid.New().String() - payloadBytes, err := json.Marshal(payload) - if err != nil { - w.logger.Error("failed to marshal webhook payload", "webhook_id", webhook.Id, "err", err) - return - } - req, err := http.NewRequestWithContext(ctx, "POST", webhook.Url, bytes.NewReader(payloadBytes)) if err != nil { w.logger.Error("failed to create webhook request", "webhook_id", webhook.Id, "err", err) return } - shortSha := payload.After[:7] - req.Header.Set("Content-Type", "application/json") - req.Header.Set("User-Agent", "Tangled-Hook/"+shortSha) + req.Header.Set("User-Agent", userAgent) req.Header.Set("X-Tangled-Event", event) req.Header.Set("X-Tangled-Hook-ID", fmt.Sprintf("%d", webhook.Id)) req.Header.Set("X-Tangled-Delivery", deliveryId) - req.Header.Set("X-Tangled-Repo", payload.Repository.FullName) + req.Header.Set("X-Tangled-Repo", repoFullName) if webhook.Secret != "" { signature := w.computeSignature(payloadBytes, webhook.Secret) diff --git a/appview/pages/pages.go b/appview/pages/pages.go --- a/appview/pages/pages.go +++ b/appview/pages/pages.go @@ -1557,6 +1557,20 @@ params.Active = "overview" return p.executeRepo("repo/stars", w, params) } +type RepoForksParams struct { + LoggedInUser *oauth.MultiAccountUser + RepoInfo repoinfo.RepoInfo + Active string + Forks []models.Repo + Page pagination.Page + TotalCount int +} + +func (p *Pages) RepoForks(w io.Writer, params RepoForksParams) error { + params.Active = "overview" + return p.executeRepo("repo/forks", w, params) +} + type PipelinesParams struct { LoggedInUser *oauth.MultiAccountUser RepoInfo repoinfo.RepoInfo diff --git a/appview/pages/repoinfo/repoinfo.go b/appview/pages/repoinfo/repoinfo.go --- a/appview/pages/repoinfo/repoinfo.go +++ b/appview/pages/repoinfo/repoinfo.go @@ -20,7 +20,7 @@ } } func (r RepoInfo) FullName() string { - return path.Join(r.owner(), r.Name) + return path.Join(r.owner(), r.Rkey) } func (r RepoInfo) RepoIdentifier() string { @@ -39,7 +39,7 @@ } } func (r RepoInfo) FullNameWithoutAt() string { - return path.Join(r.ownerWithoutAt(), r.Name) + return path.Join(r.ownerWithoutAt(), r.Rkey) } func (r RepoInfo) GetTabs() [][]string { diff --git a/appview/pages/templates/fragments/starBtn.html b/appview/pages/templates/fragments/starBtn.html --- a/appview/pages/templates/fragments/starBtn.html +++ b/appview/pages/templates/fragments/starBtn.html @@ -22,7 +22,7 @@ {{ else }} {{ i "star" "w-4 h-4 inline group-[.htmx-request]:hidden" }} {{ end }} {{ i "loader-circle" "w-4 h-4 animate-spin hidden group-[.htmx-request]:inline" }} - {{ if .IsStarred }}unstar{{ else }}star{{ end }} + {{ if .IsStarred }}unstar{{ else }}star{{ end }} {{ if .RepoName }} {{ $did := .SubjectAt.Authority | string }} diff --git a/appview/pages/templates/goodfirstissues/index.html b/appview/pages/templates/goodfirstissues/index.html --- a/appview/pages/templates/goodfirstissues/index.html +++ b/appview/pages/templates/goodfirstissues/index.html @@ -46,7 +46,7 @@ {{ else }} {{ i "book-marked" "w-4 h-4 mr-1.5 shrink-0" }} {{ end }} {{ $repoOwner := resolve .Repo.Did }} - {{ $repoOwner }}/{{ .Repo.Name }} + {{ $repoOwner }}/{{ .Repo.Name }} @@ -90,7 +90,7 @@ {{ if gt (len .Issues) 0 }}
{{ range .Issues }} - +
diff --git a/appview/pages/templates/knots/dashboard.html b/appview/pages/templates/knots/dashboard.html --- a/appview/pages/templates/knots/dashboard.html +++ b/appview/pages/templates/knots/dashboard.html @@ -77,7 +77,7 @@ {{ $repos := index $.Repos . }} {{ range $repos }}
{{ i "book-marked" "size-4" }} - + {{ .Name }}
diff --git a/appview/pages/templates/layouts/fragments/topbar.html b/appview/pages/templates/layouts/fragments/topbar.html --- a/appview/pages/templates/layouts/fragments/topbar.html +++ b/appview/pages/templates/layouts/fragments/topbar.html @@ -61,7 +61,22 @@ hx-target="#topbar-search-results" hx-swap="innerHTML" hx-indicator="#topbar-search-indicator" /> - ⌘K + + {{ i "loader-circle" "size-4 text-gray-400 animate-spin" }} diff --git a/appview/pages/templates/layouts/repobase.html b/appview/pages/templates/layouts/repobase.html --- a/appview/pages/templates/layouts/repobase.html +++ b/appview/pages/templates/layouts/repobase.html @@ -78,7 +78,7 @@ {{ $sourceOwner := resolve .RepoInfo.Source.Did }}
{{ i "git-fork" "w-3 h-3 shrink-0" }} forked from - + {{ $sourceOwner }}/{{ .RepoInfo.Source.Name }}
@@ -117,15 +117,24 @@ (dict "SubjectAt" .RepoInfo.RepoAt "IsStarred" .RepoInfo.IsStarred "StarCount" .RepoInfo.Stats.StarCount "RepoName" .RepoInfo.Name) }} - - {{ i "git-fork" "w-4 h-4" }} - fork - {{ i "loader-circle" "w-4 h-4 animate-spin hidden group-[.htmx-request]:inline" }} - +
+ + {{ i "git-fork" "w-4 h-4" }} + fork + {{ i "loader-circle" "w-4 h-4 animate-spin hidden group-[.htmx-request]:inline" }} + + + {{ .RepoInfo.Stats.ForkCount }} + +
{{ template "repo/fragments/feedDropdown" . }}
{{ end }} diff --git a/appview/pages/templates/notifications/fragments/item.html b/appview/pages/templates/notifications/fragments/item.html --- a/appview/pages/templates/notifications/fragments/item.html +++ b/appview/pages/templates/notifications/fragments/item.html @@ -76,11 +76,11 @@ {{ define "notificationUrl" }} {{ $url := "" }} {{ if eq .Type "repo_starred" }} - {{$url = printf "/%s/%s" (resolve .Repo.Did) .Repo.Name}} + {{$url = printf "/%s/%s" (resolve .Repo.Did) .Repo.Rkey}} {{ else if .Issue }} - {{$url = printf "/%s/%s/issues/%d" (resolve .Repo.Did) .Repo.Name .Issue.IssueId}} + {{$url = printf "/%s/%s/issues/%d" (resolve .Repo.Did) .Repo.Rkey .Issue.IssueId}} {{ else if .Pull }} - {{$url = printf "/%s/%s/pulls/%d" (resolve .Repo.Did) .Repo.Name .Pull.PullId}} + {{$url = printf "/%s/%s/pulls/%d" (resolve .Repo.Did) .Repo.Rkey .Pull.PullId}} {{ else if eq .Type "followed" }} {{$url = printf "/%s" (resolve .ActorDid)}} {{ else }} diff --git a/appview/pages/templates/repo/commit.html b/appview/pages/templates/repo/commit.html --- a/appview/pages/templates/repo/commit.html +++ b/appview/pages/templates/repo/commit.html @@ -28,8 +28,8 @@ {{ end }}
-
-

+

+

{{ template "attribution" . }} diff --git a/appview/pages/templates/repo/empty.html b/appview/pages/templates/repo/empty.html --- a/appview/pages/templates/repo/empty.html +++ b/appview/pages/templates/repo/empty.html @@ -35,7 +35,7 @@ {{ $bullet := "mx-2 text-xs bg-gray-200 dark:bg-gray-600 rounded-full size-5 flex items-center justify-center font-mono inline-flex align-middle" }}

1First, generate a new SSH key pair.

2Then add the public key to your account from the keys page in your settings.

-

3Configure your remote to git@{{ $knot | stripPort }}:{{ resolve .RepoInfo.OwnerDid }}/{{ .RepoInfo.Name }}

+

3Configure your remote to git@{{ $knot | stripPort }}:{{ .RepoInfo.RepoDid }}

4Push!

diff --git a/appview/pages/templates/repo/fork.html b/appview/pages/templates/repo/fork.html --- a/appview/pages/templates/repo/fork.html +++ b/appview/pages/templates/repo/fork.html @@ -1,52 +1,164 @@ {{ define "title" }}fork · {{ .RepoInfo.FullName }}{{ end }} {{ define "content" }} -
-

Fork {{ .RepoInfo.FullName }}

-
-
-
+
+
+

+ {{ i "git-fork" "size-5 shrink-0" }} + Fork + + {{ template "user/fragments/picHandleLink" .RepoInfo.OwnerDid }} + / + {{ .RepoInfo.Name }} + +

+
+ +
+ + {{/* Info banner */}} +
+ {{ i "badge-info" "size-6 text-gray-500 dark:text-gray-400 mt-0.5" }} +

+ A fork is a copy of a repository. +
+ Forking gives you your own copy of a repository to work on independently. +

+
+ + + + {{/* Step 1 — Details */}} +
+ {{ template "fork/numberCircle" 1 }} +
+

Details

-
- Repository name - -
+
-
- Select a knot to fork into -
-
- {{ range .Knots }} -
- + +
+ + - +
+
+ + {{/* Description */}} +
+ + +

{{ len .RepoInfo.Description }}/140 characters

+
+
- {{ else }} -

No knots available.

- {{ end }}
-

A knot hosts repository data. Learn how to register your own knot.

- + + {{/* Step 2 — Knot selection */}} +
+ {{ template "fork/numberCircle" 2 }} +
+

Knot selection

+ +
+ + {{/* Knot info box */}} + + + {{/* Knot list */}} +
+ +
+ {{ range .Knots }} +
+ + +
+ {{ else }} +

No knots available.

+ {{ end }} +
+
+ +
+
+
-
- -
-
- + +
+ + + +
+ + +{{ end }} + +{{ define "fork/numberCircle" }} +
+
+ {{.}} +
+
{{ end }} diff --git a/appview/pages/templates/repo/forks.html b/appview/pages/templates/repo/forks.html new file mode 100644 --- /dev/null +++ b/appview/pages/templates/repo/forks.html @@ -0,0 +1,36 @@ +{{ define "title" }}forks · {{ .RepoInfo.FullName }}{{ end }} + {{ define "repoContent" }} +
+

Forked by

+
+ {{ range .Forks }} +
+ {{ template "forkOwnerAndName" . }} +

+ forked {{ .Created | relTimeFmt }} +

+
+ {{ end }} + {{ if eq .TotalCount 0 }} +

No forks yet.

+ {{ end }} +
+ {{ if gt .TotalCount .Page.Limit }} + {{ template "fragments/pagination" (dict + "Page" .Page + "TotalCount" .TotalCount + "BasePath" (printf "/%s/forks" .RepoInfo.FullName) + "QueryParams" (queryParams) + ) }} + {{ end }} +
+{{ end }} + +{{ define "forkOwnerAndName" }} +
+ {{ template "user/fragments/picHandleLink" .Did }} + / + {{ $repoOwner := resolve .Did }} + {{ .Name }} +
+{{ end }} diff --git a/appview/pages/templates/repo/fragments/cloneDropdown.html b/appview/pages/templates/repo/fragments/cloneDropdown.html --- a/appview/pages/templates/repo/fragments/cloneDropdown.html +++ b/appview/pages/templates/repo/fragments/cloneDropdown.html @@ -38,15 +38,15 @@ {{ template "cloneUrlItem" ( dict "Label" "HTTPS" - "HandleUrl" (printf "https://tangled.org/%s/%s" $repoOwnerHandle .RepoInfo.Name) - "PermaUrl" (printf "https://tangled.org/%s/%s" .RepoInfo.OwnerDid .RepoInfo.Name) + "HandleUrl" (printf "https://tangled.org/%s/%s" $repoOwnerHandle .RepoInfo.Rkey) + "PermaUrl" (printf "https://tangled.org/%s" .RepoInfo.RepoDid) ) }} {{ template "cloneUrlItem" ( dict "Label" "SSH" - "HandleUrl" (printf "git@%s:%s/%s" (stripPort $knot) $repoOwnerHandle .RepoInfo.Name) - "PermaUrl" (printf "git@%s:%s/%s" (stripPort $knot) .RepoInfo.OwnerDid .RepoInfo.Name) + "HandleUrl" (printf "git@%s:%s/%s" (stripPort $knot) $repoOwnerHandle .RepoInfo.Rkey) + "PermaUrl" (printf "git@%s:%s" (stripPort $knot) .RepoInfo.RepoDid) ) }}

diff --git a/appview/pages/templates/repo/index.html b/appview/pages/templates/repo/index.html --- a/appview/pages/templates/repo/index.html +++ b/appview/pages/templates/repo/index.html @@ -361,3 +361,4 @@ {{- if or .HTMLReadme .Readme -}} {{ template "repo/fragments/readme" . }} {{- end -}} {{ end }} + diff --git a/appview/pages/templates/repo/needsUpgrade.html b/appview/pages/templates/repo/needsUpgrade.html --- a/appview/pages/templates/repo/needsUpgrade.html +++ b/appview/pages/templates/repo/needsUpgrade.html @@ -53,6 +53,11 @@

{{ i "triangle-alert" "size-5 inline-flex items-center align-middle" }} The knot hosting this repository needs an upgrade. This repository is currently unavailable. +
+ + Read the upgrade guide + +
diff --git a/appview/pages/templates/repo/pulls/fragments/pullActions.html b/appview/pages/templates/repo/pulls/fragments/pullActions.html --- a/appview/pages/templates/repo/pulls/fragments/pullActions.html +++ b/appview/pages/templates/repo/pulls/fragments/pullActions.html @@ -34,7 +34,7 @@ comment {{ if .BranchDeleteStatus }}
+{{ end }} + +{{ define "renameRepo" }} + {{ if and .RepoInfo.Roles.IsOwner .RepoInfo.RepoDid }} +
+

Rename Repository

+

+ Existing git remotes that use the old name will break. Use the + stable, DID-based URLs below to avoid breakage on future renames. +

+ + +
+
+ +
+
+ {{ end }} {{ end }} {{ define "deleteRepo" }} diff --git a/appview/pages/templates/repo/settings/hooks.html b/appview/pages/templates/repo/settings/hooks.html --- a/appview/pages/templates/repo/settings/hooks.html +++ b/appview/pages/templates/repo/settings/hooks.html @@ -225,6 +225,10 @@
Push events
+
+ + Repository renamed +

Additional event types (pull requests, issues) will be available in future updates.

@@ -294,12 +298,18 @@
{{ $hasPush := false }} + {{ $hasRepoRenamed := false }} {{ range $webhook.Events }} {{ if eq . "push" }}{{ $hasPush = true }}{{ end }} + {{ if eq . "repository:renamed" }}{{ $hasRepoRenamed = true }}{{ end }} {{ end }}
Push events +
+
+ + Repository renamed

Additional event types (pull requests, issues) will be available in future updates. diff --git a/appview/pages/templates/repo/settings/sites.html b/appview/pages/templates/repo/settings/sites.html --- a/appview/pages/templates/repo/settings/sites.html +++ b/appview/pages/templates/repo/settings/sites.html @@ -32,7 +32,7 @@

{{ else }}
{{ i "circle-check" "size-4 shrink-0" }} - live at {{ .OwnerClaim.Domain }}/{{ .RepoInfo.Name }} + live at {{ .OwnerClaim.Domain }}/{{ .RepoInfo.Rkey }}
{{ end }} {{ else if and .SiteConfig (not .OwnerClaim) }} @@ -157,9 +157,9 @@
sub-path site

{{ if .OwnerClaim }} - {{ .OwnerClaim.Domain }}/{{ $.RepoInfo.Name }} + {{ .OwnerClaim.Domain }}/{{ $.RepoInfo.Rkey }} {{ else }} - e.g. you.tngl.page/{{ $.RepoInfo.Name }} + e.g. you.tngl.page/{{ $.RepoInfo.Rkey }} {{ end }}

diff --git a/appview/pages/templates/spindles/dashboard.html b/appview/pages/templates/spindles/dashboard.html --- a/appview/pages/templates/spindles/dashboard.html +++ b/appview/pages/templates/spindles/dashboard.html @@ -69,7 +69,7 @@ {{ $repos := index $.Repos . }} {{ range $repos }}
{{ i "book-marked" "size-4" }} - + {{ .Name }}
diff --git a/appview/pages/templates/timeline/fragments/preview.html b/appview/pages/templates/timeline/fragments/preview.html --- a/appview/pages/templates/timeline/fragments/preview.html +++ b/appview/pages/templates/timeline/fragments/preview.html @@ -121,14 +121,14 @@ {{ template "user/fragments/picHandleLink" $repo.Did }} {{ with $source }} {{ $sourceDid := resolve .Did }} forked - + {{ $sourceDid }}/{{ .Name }} to - {{ $repo.Name }} + {{ $repo.Name }} {{ else }} created - + {{ $repo.Name }} {{ end }} @@ -159,7 +159,7 @@ {{ else }} starred {{ end }} {{ template "user/fragments/pic" (list $repoOwnerHandle "size-6") }} - + {{ $repoOwnerHandle | truncateAt30 }}/{{ .Repo.Name }}
diff --git a/appview/pages/templates/timeline/fragments/timeline.html b/appview/pages/templates/timeline/fragments/timeline.html --- a/appview/pages/templates/timeline/fragments/timeline.html +++ b/appview/pages/templates/timeline/fragments/timeline.html @@ -39,14 +39,14 @@ {{ template "user/fragments/picHandleLink" $repo.Did }} {{ with $source }} {{ $sourceDid := resolve .Did }} forked - + {{ $sourceDid }}/{{ .Name }} to - {{ $repo.Name }} + {{ $repo.Name }} {{ else }} created - + {{ $repo.Name }} {{ end }} @@ -74,7 +74,7 @@ {{ else }} starred {{ end }} - + {{ $repoOwnerHandle | truncateAt30 }}/{{ .Repo.Name }} {{ template "repo/fragments/time" .Created }} diff --git a/appview/pages/templates/user/fragments/issueEvent.html b/appview/pages/templates/user/fragments/issueEvent.html --- a/appview/pages/templates/user/fragments/issueEvent.html +++ b/appview/pages/templates/user/fragments/issueEvent.html @@ -1,6 +1,7 @@ {{ define "user/fragments/issueEvent" }} {{ $repoOwner := resolve .Repo.Did }} - {{ $repoUrl := printf "%s/%s" $repoOwner .Repo.Name }} + {{ $repoUrl := printf "%s/%s" $repoOwner .Repo.Rkey }} + {{ $repoDisplay := printf "%s/%s" $repoOwner .Repo.Name }}
{{ if .Open }} @@ -15,7 +16,7 @@ #{{ .IssueId }} {{- .Title -}} on - {{ $repoUrl }} + {{ $repoDisplay }}
{{ end }} diff --git a/appview/pages/templates/user/fragments/pullEvent.html b/appview/pages/templates/user/fragments/pullEvent.html --- a/appview/pages/templates/user/fragments/pullEvent.html +++ b/appview/pages/templates/user/fragments/pullEvent.html @@ -1,6 +1,7 @@ {{ define "user/fragments/pullEvent" }} {{ $repoOwner := resolve .Repo.Did }} - {{ $repoUrl := printf "%s/%s" $repoOwner .Repo.Name }} + {{ $repoUrl := printf "%s/%s" $repoOwner .Repo.Rkey }} + {{ $repoDisplay := printf "%s/%s" $repoOwner .Repo.Name }}
{{ if .State.IsOpen }} @@ -19,7 +20,7 @@ #{{ .PullId }} {{- .Title -}} on - {{ $repoUrl }} + {{ $repoDisplay }}
{{ end }} diff --git a/appview/pages/templates/user/fragments/repoCard.html b/appview/pages/templates/user/fragments/repoCard.html --- a/appview/pages/templates/user/fragments/repoCard.html +++ b/appview/pages/templates/user/fragments/repoCard.html @@ -27,9 +27,9 @@ {{ i "book-marked" "w-4 h-4 mr-1.5 shrink-0" }} {{ end }} {{ $repoOwner := resolve .Did }} {{- if $fullName -}} - {{ $repoOwner }}/{{ .Name }} + {{ $repoOwner }}/{{ .Name }} {{- else -}} - {{ .Name }} + {{ .Name }} {{- end -}} {{ if and $starButton $root.LoggedInUser }} @@ -62,6 +62,12 @@ {{ end }} {{ with .StarCount }}
{{ i "star" "w-3 h-3 fill-current" }} + {{ . }} +
+ {{ end }} + {{ with .ForkCount }} +
+ {{ i "git-fork" "w-3 h-3 fill-current" }} {{ . }}
{{ end }} diff --git a/appview/pages/templates/user/overview.html b/appview/pages/templates/user/overview.html --- a/appview/pages/templates/user/overview.html +++ b/appview/pages/templates/user/overview.html @@ -79,7 +79,7 @@ {{ else }} {{ i "book-plus" "w-4 h-4" }} {{ end }} - + {{- .Repo.Name -}} diff --git a/appview/pipelines/pipelines.go b/appview/pipelines/pipelines.go --- a/appview/pipelines/pipelines.go +++ b/appview/pipelines/pipelines.go @@ -89,7 +89,7 @@ filterKind := r.URL.Query().Get("trigger") filters := []orm.Filter{ orm.FilterEq("p.repo_owner", f.Did), - orm.FilterEq("p.repo_name", f.Name), + orm.FilterEq("p.repo_name", f.Rkey), orm.FilterEq("p.knot", f.Knot), } switch filterKind { @@ -153,7 +153,7 @@ ps, err := db.GetPipelineStatuses( p.db, 1, orm.FilterEq("p.repo_owner", f.Did), - orm.FilterEq("p.repo_name", f.Name), + orm.FilterEq("p.repo_name", f.Rkey), orm.FilterEq("p.knot", f.Knot), orm.FilterEq("p.id", pipelineId), ) @@ -220,7 +220,7 @@ ps, err := db.GetPipelineStatuses( p.db, 1, orm.FilterEq("p.repo_owner", f.Did), - orm.FilterEq("p.repo_name", f.Name), + orm.FilterEq("p.repo_name", f.Rkey), orm.FilterEq("p.knot", f.Knot), orm.FilterEq("p.id", pipelineId), ) @@ -369,7 +369,7 @@ ps, err := db.GetPipelineStatuses( p.db, 1, orm.FilterEq("p.repo_owner", f.Did), - orm.FilterEq("p.repo_name", f.Name), + orm.FilterEq("p.repo_name", f.Rkey), orm.FilterEq("p.knot", f.Knot), orm.FilterEq("p.id", pipelineId), ) diff --git a/appview/pulls/comment.go b/appview/pulls/comment.go --- a/appview/pulls/comment.go +++ b/appview/pulls/comment.go @@ -103,7 +103,7 @@ } comment := &models.PullComment{ OwnerDid: user.Did, - RepoAt: f.RepoAt().String(), + RepoDid: string(f.RepoDid), PullId: pull.PullId, Body: body, CommentAt: atResp.Uri, diff --git a/appview/pulls/compose_helpers_test.go b/appview/pulls/compose_helpers_test.go --- a/appview/pulls/compose_helpers_test.go +++ b/appview/pulls/compose_helpers_test.go @@ -181,7 +181,7 @@ } } func TestComposeCanonicalURL(t *testing.T) { - repo := repoinfo.RepoInfo{OwnerDid: "did:plc:abc", Name: "demo"} + repo := repoinfo.RepoInfo{OwnerDid: "did:plc:abc", Name: "demo", Rkey: "demo"} cases := []struct { name string p pages.RepoNewPullParams diff --git a/appview/pulls/create.go b/appview/pulls/create.go --- a/appview/pulls/create.go +++ b/appview/pulls/create.go @@ -11,6 +11,7 @@ "strings" "time" "tangled.org/core/api/tangled" + "tangled.org/core/appview/compat113" "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/oauth" @@ -98,13 +99,13 @@ l := s.logger.With("handler", "handleForkBasedPull", "user", userDid, "fork_repo", forkRepo, "target_branch", targetBranch, "source_branch", sourceBranch, "is_stacked", isStacked) repoString := strings.SplitN(forkRepo, "/", 2) forkOwnerDid := repoString[0] - repoName := repoString[1] - fork, err := db.GetForkByDid(s.db, forkOwnerDid, repoName) + forkRkey := strings.ToLower(repoString[1]) + fork, err := db.GetForkByDid(s.db, forkOwnerDid, forkRkey) if errors.Is(err, sql.ErrNoRows) { s.pages.Notice(w, "pull", "No such fork.") return } else if err != nil { - l.Error("failed to fetch fork", "err", err, "fork_owner_did", forkOwnerDid, "repo_name", repoName) + l.Error("failed to fetch fork", "err", err, "fork_owner_did", forkOwnerDid, "fork_rkey", forkRkey) s.pages.Notice(w, "pull", "Failed to fetch fork.") return } @@ -182,17 +183,10 @@ s.pages.Notice(w, "pull", "Invalid patch format. Please provide a valid diff.") return } - forkAtUri := fork.RepoAt() - var forkDid *syntax.DID - if fork.RepoDid != "" { - forkDid = new(syntax.DID) - *forkDid = syntax.DID(fork.RepoDid) - } - + forkDid := syntax.DID(fork.RepoDid) pullSource := &models.PullSource{ Branch: sourceBranch, - RepoAt: &forkAtUri, - RepoDid: forkDid, + RepoDid: &forkDid, } s.createPullRequest(w, r, repo, userDid, title, body, targetBranch, patch, combined, sourceRev, pullSource, isStacked, stackTitles, stackBodies) @@ -284,7 +278,7 @@ Title: title, Body: body, TargetBranch: targetBranch, OwnerDid: userDid.String(), - RepoAt: repo.RepoAt(), + RepoDid: syntax.DID(repo.RepoDid), Rkey: rkey, Mentions: mentions, References: references, @@ -308,9 +302,7 @@ _, err = comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{ Collection: tangled.RepoPullNSID, Repo: userDid.String(), Rkey: rkey, - Record: &lexutil.LexiconTypeDecoder{ - Val: &record, - }, + Record: compat113.Pull(&record), }) if err != nil { l.Error("failed to create pull request", "err", err) @@ -324,7 +316,7 @@ l.Error("failed to create pull request in database", "err", err) s.pages.Notice(w, "pull", "Failed to create pull request. Try again later.") return } - pullId, err := db.NextPullId(tx, repo.RepoAt()) + pullId, err := db.NextPullId(tx, repo.RepoDid) if err != nil { s.logger.Error("failed to get pull id", "err", err) s.pages.Notice(w, "pull", "Failed to create pull request. Try again later.") @@ -410,9 +402,7 @@ writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{ RepoApplyWrites_Create: &comatproto.RepoApplyWrites_Create{ Collection: tangled.RepoPullNSID, Rkey: &p.Rkey, - Value: &lexutil.LexiconTypeDecoder{ - Val: &record, - }, + Value: compat113.Pull(&record), }, }) } @@ -502,7 +492,7 @@ Title: title, Body: body, TargetBranch: targetBranch, OwnerDid: userDid.String(), - RepoAt: repo.RepoAt(), + RepoDid: syntax.DID(repo.RepoDid), Rkey: rkey, Mentions: mentions, References: references, diff --git a/appview/pulls/lifecycle.go b/appview/pulls/lifecycle.go --- a/appview/pulls/lifecycle.go +++ b/appview/pulls/lifecycle.go @@ -66,7 +66,7 @@ p.State = models.PullClosed } err = db.ClosePulls( tx, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", string(f.RepoDid)), orm.FilterIn("at_uri", atUris), ) if err != nil { @@ -143,7 +143,7 @@ p.State = models.PullOpen } err = db.ReopenPulls( tx, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", string(f.RepoDid)), orm.FilterIn("at_uri", atUris), ) if err != nil { diff --git a/appview/pulls/list.go b/appview/pulls/list.go --- a/appview/pulls/list.go +++ b/appview/pulls/list.go @@ -103,7 +103,7 @@ searchOpts := models.PullSearchOptions{ Keywords: tf.Keywords, Phrases: tf.Phrases, - RepoAt: f.RepoAt().String(), + RepoDid: f.RepoDid, State: state, AuthorDid: authorDid, Labels: labels, @@ -175,7 +175,7 @@ } } } else { filters := []orm.Filter{ - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), } if state != nil { filters = append(filters, orm.FilterEq("state", *state)) @@ -195,10 +195,10 @@ for _, p := range pulls { var pullSourceRepo *models.Repo if p.PullSource != nil { - if p.PullSource.RepoAt != nil { - pullSourceRepo, err = db.GetRepoByAtUri(s.db, p.PullSource.RepoAt.String()) + if p.PullSource.RepoDid != nil { + pullSourceRepo, err = db.GetRepoByDid(s.db, string(*p.PullSource.RepoDid)) if err != nil { - l.Error("failed to get repo by at uri", "err", err, "repo_at", p.PullSource.RepoAt.String()) + l.Error("failed to get repo by did", "err", err, "repo_did", p.PullSource.RepoDid.String()) continue } else { p.PullSource.Repo = pullSourceRepo @@ -265,7 +265,7 @@ ps, err := db.GetPipelineStatuses( s.db, len(shas), orm.FilterEq("p.repo_owner", f.Did), - orm.FilterEq("p.repo_name", f.Name), + orm.FilterEq("p.repo_name", f.Rkey), orm.FilterEq("p.knot", f.Knot), orm.FilterIn("p.sha", shas), ) diff --git a/appview/pulls/merge.go b/appview/pulls/merge.go --- a/appview/pulls/merge.go +++ b/appview/pulls/merge.go @@ -118,7 +118,7 @@ for _, p := range pullsToMerge { atUris = append(atUris, p.AtUri()) p.State = models.PullMerged } - err = db.MergePulls(tx, orm.FilterEq("repo_at", f.RepoAt()), orm.FilterIn("at_uri", atUris)) + err = db.MergePulls(tx, orm.FilterEq("repo_did", string(f.RepoDid)), orm.FilterIn("at_uri", atUris)) if err != nil { l.Error("failed to update pull request status in database", "err", err) s.pages.Notice(w, "pull-merge-error", "Failed to merge pull request. Try again later.") diff --git a/appview/pulls/resubmit.go b/appview/pulls/resubmit.go --- a/appview/pulls/resubmit.go +++ b/appview/pulls/resubmit.go @@ -7,6 +7,7 @@ "net/http" "time" "tangled.org/core/api/tangled" + "tangled.org/core/appview/compat113" "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/oauth" @@ -185,9 +186,9 @@ l.Error("failed to get repo and knot", "err", err) return } - forkRepo, err := db.GetRepoByAtUri(s.db, pull.PullSource.RepoAt.String()) + forkRepo, err := db.GetRepoByDid(s.db, string(*pull.PullSource.RepoDid)) if err != nil { - l.Error("failed to get source repo", "err", err, "repo_at", pull.PullSource.RepoAt.String()) + l.Error("failed to get source repo", "err", err, "repo_did", pull.PullSource.RepoDid.String()) s.pages.Notice(w, "resubmit-error", "Failed to create pull request. Try again later.") return } @@ -330,9 +331,7 @@ Collection: tangled.RepoPullNSID, Repo: userDid.String(), Rkey: pull.Rkey, SwapRecord: ex.Cid, - Record: &lexutil.LexiconTypeDecoder{ - Val: &record, - }, + Record: compat113.Pull(&record), }) if err != nil { l.Error("failed to update record on PDS", "err", err, "rkey", pull.Rkey) @@ -480,7 +479,7 @@ if p.State == models.PullMerged { continue } - err := db.AbandonPulls(tx, orm.FilterEq("repo_at", p.RepoAt), orm.FilterEq("at_uri", p.AtUri())) + err := db.AbandonPulls(tx, orm.FilterEq("repo_did", string(p.RepoDid)), orm.FilterEq("at_uri", p.AtUri())) if err != nil { l.Error("failed to delete pull", "err", err, "pull_id", p.PullId) s.pages.Notice(w, "pull-resubmit-error", "Failed to resubmit pull request. Try again later.") @@ -521,9 +520,7 @@ writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{ RepoApplyWrites_Create: &comatproto.RepoApplyWrites_Create{ Collection: tangled.RepoPullNSID, Rkey: &p.Rkey, - Value: &lexutil.LexiconTypeDecoder{ - Val: &record, - }, + Value: compat113.Pull(&record), }, }) } @@ -581,9 +578,7 @@ writes = append(writes, &comatproto.RepoApplyWrites_Input_Writes_Elem{ RepoApplyWrites_Update: &comatproto.RepoApplyWrites_Update{ Collection: tangled.RepoPullNSID, Rkey: op.Rkey, - Value: &lexutil.LexiconTypeDecoder{ - Val: &record, - }, + Value: compat113.Pull(&record), }, }) } diff --git a/appview/pulls/single.go b/appview/pulls/single.go --- a/appview/pulls/single.go +++ b/appview/pulls/single.go @@ -149,7 +149,7 @@ ps, err := db.GetPipelineStatuses( s.db, len(shas), orm.FilterEq("p.repo_owner", f.Did), - orm.FilterEq("p.repo_name", f.Name), + orm.FilterEq("p.repo_name", f.Rkey), orm.FilterEq("p.knot", f.Knot), orm.FilterIn("p.sha", shas), ) @@ -369,15 +369,15 @@ if pull.State == models.PullMerged || pull.State == models.PullAbandoned || pull.PullSource == nil { return pages.Unknown } - var sourceRepo syntax.ATURI - if pull.PullSource.RepoAt != nil { - sourceRepo = *pull.PullSource.RepoAt + var sourceRepoDid string + if pull.PullSource.RepoDid != nil { + sourceRepoDid = string(*pull.PullSource.RepoDid) } else { - sourceRepo = repo.RepoAt() + sourceRepoDid = repo.RepoDid } xrpcc := &indigoxrpc.Client{Host: s.config.KnotMirror.Url} - branchResp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, pull.PullSource.Branch, sourceRepo.String()) + branchResp, err := tangled.GitTempGetBranch(r.Context(), xrpcc, pull.PullSource.Branch, sourceRepoDid) if err != nil { if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { s.logger.Error("failed to call XRPC repo.branches", "xrpcerr", xrpcerr, "err", err, "pull_id", pull.PullId, "branch", pull.PullSource.Branch) diff --git a/appview/repo/artifact.go b/appview/repo/artifact.go --- a/appview/repo/artifact.go +++ b/appview/repo/artifact.go @@ -20,6 +20,7 @@ "tangled.org/core/types" "tangled.org/core/xrpc" comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" lexutil "github.com/bluesky-social/indigo/lex/util" indigoxrpc "github.com/bluesky-social/indigo/xrpc" "github.com/dustin/go-humanize" @@ -102,7 +103,7 @@ artifact := models.Artifact{ Did: user.Did, Rkey: rkey, - RepoAt: f.RepoAt(), + RepoDid: syntax.DID(f.RepoDid), Tag: tag.Tag.Hash, CreatedAt: createdAt, BlobCid: cid.Cid(uploadBlobResp.Blob.Ref), @@ -154,7 +155,7 @@ } artifacts, err := db.GetArtifact( rp.db, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), orm.FilterEq("tag", tag.Tag.Hash[:]), orm.FilterEq("name", filename), ) @@ -235,7 +236,7 @@ tag := plumbing.NewHash(tagParam) artifacts, err := db.GetArtifact( rp.db, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), orm.FilterEq("tag", tag[:]), orm.FilterEq("name", filename), ) @@ -277,7 +278,7 @@ } defer tx.Rollback() err = db.DeleteArtifact(tx, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), orm.FilterEq("tag", artifact.Tag[:]), orm.FilterEq("name", filename), ) diff --git a/appview/repo/feed.go b/appview/repo/feed.go --- a/appview/repo/feed.go +++ b/appview/repo/feed.go @@ -74,7 +74,7 @@ } // fetch and add pull requests if requested if opts.IncludePulls { - pulls, err := db.GetPullsPaginated(rp.db, feedPagePerType, orm.FilterEq("repo_at", repo.RepoAt())) + pulls, err := db.GetPullsPaginated(rp.db, feedPagePerType, orm.FilterEq("repo_did", repo.RepoDid)) if err != nil { return nil, err } @@ -93,7 +93,7 @@ if opts.IncludeIssues { issues, err := db.GetIssuesPaginated( rp.db, feedPagePerType, - orm.FilterEq("repo_at", repo.RepoAt()), + orm.FilterEq("repo_did", repo.RepoDid), ) if err != nil { return nil, err @@ -315,7 +315,7 @@ if !ok || repoOwnerId.Handle.IsInvalidHandle() { rp.logger.Error("failed to get resolved repo owner id") return } - ownerSlashRepo := repoOwnerId.Handle.String() + "/" + f.Name + ownerSlashRepo := repoOwnerId.Handle.String() + "/" + f.Rkey opts := parseFeedOpts(r) feed, err := rp.getRepoFeed(r.Context(), f, ownerSlashRepo, opts) diff --git a/appview/repo/index.go b/appview/repo/index.go --- a/appview/repo/index.go +++ b/appview/repo/index.go @@ -15,6 +15,7 @@ "context" "encoding/json" + "github.com/bluesky-social/indigo/atproto/syntax" indigoxrpc "github.com/bluesky-social/indigo/xrpc" "github.com/go-git/go-git/v5/plumbing" "tangled.org/core/api/tangled" @@ -158,7 +159,7 @@ ) ([]types.RepoLanguageDetails, error) { // first attempt to fetch from db langs, err := db.GetRepoLanguages( rp.db, - orm.FilterEq("repo_at", repo.RepoAt()), + orm.FilterEq("repo_did", repo.RepoDid), orm.FilterEq("ref", currentRef), ) @@ -181,7 +182,7 @@ } for _, lang := range ls.Languages { langs = append(langs, models.RepoLanguage{ - RepoAt: repo.RepoAt(), + RepoDid: syntax.DID(repo.RepoDid), Ref: currentRef, IsDefaultRef: isDefaultRef, Language: lang.Name, @@ -196,7 +197,7 @@ } defer tx.Rollback() // update appview's cache - err = db.UpdateRepoLanguages(tx, repo.RepoAt(), currentRef, langs) + err = db.UpdateRepoLanguages(tx, syntax.DID(repo.RepoDid), currentRef, langs) if err != nil { // non-fatal l.Error("failed to cache lang results", "err", err) diff --git a/appview/repo/opengraph.go b/appview/repo/opengraph.go --- a/appview/repo/opengraph.go +++ b/appview/repo/opengraph.go @@ -27,7 +27,7 @@ var languageStats []types.RepoLanguageDetails langs, err := db.GetRepoLanguages( rp.db, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), orm.FilterEq("is_default_ref", 1), ) if err != nil { diff --git a/appview/repo/rename_test.go b/appview/repo/rename_test.go new file mode 100644 --- /dev/null +++ b/appview/repo/rename_test.go @@ -0,0 +1,53 @@ +package repo + +import ( + "strings" + "testing" +) + +func TestValidateRenameInput(t *testing.T) { + const validTID = "3jzfcijpj2z2a" + cases := []struct { + name string + currentName string + currentRkey string + raw string + wantName string + wantErrSub string + }{ + {"happy path", "foo", "", "bar", "bar", ""}, + {"trims surrounding whitespace", "foo", "", " bar ", "bar", ""}, + {"strips .git suffix", "foo", "", "bar.git", "bar", ""}, + {"empty after trim", "foo", "", " ", "", "cannot be empty"}, + {"raw empty", "foo", "", "", "", "cannot be empty"}, + {"path traversal slash", "foo", "", "../bar", "", "invalid path"}, + {"invalid character", "foo", "", "ba r", "", "alphanumeric"}, + {"same name as current with non-TID rkey", "foo", "foo", "foo", "", "matches the current name"}, + {"same name as current with TID rkey allowed", "foo", validTID, "foo", "foo", ""}, + {"case-only diff is not a no-op", "foo", "", "Foo", "Foo", ""}, + {"strip-git collides with current", "foo", "", "foo.git", "", "matches the current name"}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, err := validateRenameInput(tc.currentName, tc.currentRkey, tc.raw) + if tc.wantErrSub == "" { + if err != nil { + t.Fatalf("err = %v, want nil", err) + } + if got != tc.wantName { + t.Errorf("name = %q, want %q", got, tc.wantName) + } + return + } + if err == nil { + t.Fatalf("err = nil, want error containing %q", tc.wantErrSub) + } + if got != "" { + t.Errorf("name = %q, want empty on error", got) + } + if !strings.Contains(err.Error(), tc.wantErrSub) { + t.Errorf("err = %q, want substring %q", err.Error(), tc.wantErrSub) + } + }) + } +} diff --git a/appview/repo/repo.go b/appview/repo/repo.go --- a/appview/repo/repo.go +++ b/appview/repo/repo.go @@ -15,6 +15,7 @@ "tangled.org/core/appview/cloudflare" "tangled.org/core/api/tangled" + "tangled.org/core/appview/compat113" "tangled.org/core/appview/config" "tangled.org/core/appview/db" "tangled.org/core/appview/models" @@ -23,6 +24,7 @@ "tangled.org/core/appview/oauth" "tangled.org/core/appview/pages" "tangled.org/core/appview/pagination" "tangled.org/core/appview/reporesolver" + "tangled.org/core/appview/sites" "tangled.org/core/appview/validator" xrpcclient "tangled.org/core/appview/xrpcclient" "tangled.org/core/eventconsumer" @@ -140,7 +142,7 @@ newRepo.Spindle = "" } // optimistic update - err = db.UpdateSpindle(rp.db, newRepo.RepoAt().String(), spindlePtr) + err = db.UpdateSpindle(rp.db, newRepo.RepoDid, spindlePtr) if err != nil { fail("Failed to update spindle. Try again later.", err) return @@ -320,7 +322,7 @@ return } if err = db.SubscribeLabel(tx, &models.RepoLabel{ - RepoAt: f.RepoAt(), + RepoDid: syntax.DID(f.RepoDid), LabelAt: label.AtUri(), }); err != nil { fail("Failed to subscribe to label.", err) @@ -423,7 +425,7 @@ defer tx.Rollback() err = db.UnsubscribeLabel( tx, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), orm.FilterEq("label_at", removedAt), ) if err != nil { @@ -515,7 +517,7 @@ defer tx.Rollback() for _, l := range labelAts { err = db.SubscribeLabel(tx, &models.RepoLabel{ - RepoAt: f.RepoAt(), + RepoDid: syntax.DID(f.RepoDid), LabelAt: syntax.ATURI(l), }) if err != nil { @@ -596,7 +598,7 @@ }) err = db.UnsubscribeLabel( rp.db, - orm.FilterEq("repo_at", f.RepoAt()), + orm.FilterEq("repo_did", f.RepoDid), orm.FilterIn("label_at", labelAts), ) if err != nil { @@ -758,9 +760,7 @@ resp, err := comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{ Collection: tangled.RepoCollaboratorNSID, Repo: currentUser.Did, Rkey: rkey, - Record: &lexutil.LexiconTypeDecoder{ - Val: repoCollaboratorRecord(f, collaboratorIdent.DID.String(), createdAt), - }, + Record: compat113.Collaborator(repoCollaboratorRecord(f, collaboratorIdent.DID.String(), createdAt)), }) // invalid record if err != nil { @@ -805,7 +805,7 @@ err = db.AddCollaborator(tx, models.Collaborator{ Did: syntax.DID(currentUser.Did), Rkey: rkey, SubjectDid: collaboratorIdent.DID, - RepoAt: f.RepoAt(), + RepoDid: syntax.DID(f.RepoDid), Created: createdAt, }) if err != nil { @@ -831,6 +831,210 @@ rp.pages.HxRefresh(w) } +func (rp *Repo) RenameRepo(w http.ResponseWriter, r *http.Request) { + l := rp.logger.With("handler", "RenameRepo") + noticeId := "rename-repo-error" + + user := rp.oauth.GetMultiAccountUser(r) + f, err := rp.repoResolver.Resolve(r) + if err != nil { + l.Error("failed to get repo and knot", "err", err) + rp.pages.Notice(w, noticeId, "Failed to load repository.") + return + } + l = l.With("did", user.Did, "rkey", f.Rkey, "oldName", f.Name) + + if f.RepoDid == "" { + rp.pages.Notice(w, noticeId, "This repository's knot has not completed the DID migration; rename is unavailable.") + return + } + + if !compat113.KnotSupports114(r.Context(), f.Knot, rp.config.Core.Dev) { + rp.pages.Notice(w, noticeId, "This repository's knot is below v1.14 and does not yet support renames. Ask the knot operator to upgrade.") + return + } + + newName, err := validateRenameInput(f.Name, f.Rkey, r.FormValue("name")) + if err != nil { + rp.pages.Notice(w, noticeId, err.Error()) + return + } + newRkey := strings.ToLower(newName) + l = l.With("newName", newName, "newRkey", newRkey) + + atpClient, err := rp.oauth.AuthorizedClient(r) + if err != nil { + l.Error("failed to get authorized client", "err", err) + rp.pages.Notice(w, noticeId, "Failed to authorize. Try again later.") + return + } + + newRepo := *f + newRepo.Name = newName + newRepo.Rkey = newRkey + newRepo.Created = time.Now() + record := newRepo.AsRecord() + + if newRkey == f.Rkey { + ex, err := comatproto.RepoGetRecord(r.Context(), atpClient, "", tangled.RepoNSID, f.Did, f.Rkey) + if err != nil { + l.Error("failed to fetch existing record", "err", err) + rp.pages.Notice(w, noticeId, "Failed to read repository record from PDS.") + return + } + + _, err = comatproto.RepoPutRecord(r.Context(), atpClient, &comatproto.RepoPutRecord_Input{ + Collection: tangled.RepoNSID, + Repo: f.Did, + Rkey: f.Rkey, + SwapRecord: ex.Cid, + Record: &lexutil.LexiconTypeDecoder{ + Val: &record, + }, + }) + if err != nil { + l.Error("failed to update display name on PDS", "err", err) + rp.pages.Notice(w, noticeId, "Failed to save display name to PDS.") + return + } + l.Info("updated display name on PDS") + + if err := db.UpdateRepoDisplayName(rp.db, f.Did, f.Rkey, newName); err != nil { + l.Error("optimistic display name update failed", "err", err) + } + } else { + ex, getErr := comatproto.RepoGetRecord(r.Context(), atpClient, "", tangled.RepoNSID, f.Did, newRkey) + switch { + case getErr != nil: + _, err = comatproto.RepoCreateRecord(r.Context(), atpClient, &comatproto.RepoCreateRecord_Input{ + Collection: tangled.RepoNSID, + Repo: f.Did, + Rkey: &newRkey, + Record: &lexutil.LexiconTypeDecoder{Val: &record}, + }) + if err != nil { + l.Error("failed to write rename to PDS", "err", err) + rp.pages.Notice(w, noticeId, "Failed to save renamed repository to PDS.") + return + } + l.Info("wrote rename-create to PDS; old record retained as alias") + + default: + existing, ok := ex.Value.Val.(*tangled.Repo) + if !ok || existing.RepoDid == nil || *existing.RepoDid != f.RepoDid { + rp.pages.Notice(w, noticeId, fmt.Sprintf("You already have a repository named %q.", newRkey)) + return + } + _, err = comatproto.RepoPutRecord(r.Context(), atpClient, &comatproto.RepoPutRecord_Input{ + Collection: tangled.RepoNSID, + Repo: f.Did, + Rkey: newRkey, + SwapRecord: ex.Cid, + Record: &lexutil.LexiconTypeDecoder{Val: &record}, + }) + if err != nil { + l.Error("failed to rewrite rename-back record on PDS", "err", err) + rp.pages.Notice(w, noticeId, "Failed to save renamed repository to PDS.") + return + } + l.Info("rewrote rename-back record on PDS over prior alias") + } + + tx, err := rp.db.Begin() + if err != nil { + l.Error("failed to begin rename tx", "err", err) + rp.pages.HxLocation(w, fmt.Sprintf("/%s", f.RepoDid)) + return + } + defer tx.Rollback() + + if err := db.RenameRepo(tx, f.Did, f.Rkey, newRkey, newName); err != nil { + l.Error("optimistic rename failed", "err", err) + rp.pages.HxLocation(w, fmt.Sprintf("/%s", f.RepoDid)) + return + } + if err := db.RecordRepoRename(tx, f.Did, f.Rkey, f.RepoDid); err != nil { + l.Error("failed to record rename history", "err", err) + } + if err := db.DeleteRepoRename(tx, f.Did, newRkey); err != nil { + l.Error("failed to clear stale rename hint", "err", err) + } + if err := tx.Commit(); err != nil { + l.Error("failed to commit rename tx", "err", err) + rp.pages.HxLocation(w, fmt.Sprintf("/%s", f.RepoDid)) + return + } + } + + oldRepo := *f + rp.notifier.RenameRepo(r.Context(), syntax.DID(user.Did), &oldRepo, &newRepo) + + if newRkey != f.Rkey { + rp.migrateSiteOnRename(r.Context(), f, newRkey) + } + + rp.pages.HxLocation(w, fmt.Sprintf("/%s", f.RepoDid)) +} + +func validateRenameInput(currentName, currentRkey, raw string) (string, error) { + newName := strings.TrimSpace(raw) + if newName == "" { + return "", errors.New("Repository name cannot be empty.") + } + if err := models.ValidateRepoName(newName); err != nil { + return "", err + } + newName = models.StripGitExt(newName) + if newName == currentName { + if _, tidErr := syntax.ParseTID(currentRkey); tidErr == nil { + return newName, nil + } + return "", errors.New("New name matches the current name.") + } + return newName, nil +} + +func (rp *Repo) migrateSiteOnRename(ctx context.Context, oldRepo *models.Repo, newRkey string) { + l := rp.logger.With("handler", "migrateSiteOnRename", "repo_did", oldRepo.RepoDid) + + siteConfig, err := db.GetRepoSiteConfig(rp.db, oldRepo.RepoDid) + if err != nil || siteConfig == nil { + return + } + + if !rp.cfClient.Enabled() { + return + } + + ownerClaim, _ := db.GetActiveDomainClaimForDid(rp.db, oldRepo.Did) + + go func() { + bgCtx := context.Background() + oldRkey := oldRepo.Rkey + + if err := sites.Delete(bgCtx, rp.cfClient, oldRepo.Did, oldRkey); err != nil { + l.Error("sites: failed to delete old R2 prefix", "oldRkey", oldRkey, "err", err) + } + + newRepo := *oldRepo + newRepo.Rkey = newRkey + if deployErr := sites.Deploy(bgCtx, rp.cfClient, rp.config, &newRepo, siteConfig.Branch, siteConfig.Dir); deployErr != nil { + l.Error("sites: redeploy after rename failed", "err", deployErr) + } + + if ownerClaim != nil { + if err := sites.DeleteDomainMapping(bgCtx, rp.cfClient, ownerClaim.Domain, oldRkey); err != nil { + l.Error("sites: failed to remove old KV mapping", "oldRkey", oldRkey, "err", err) + } + if err := sites.PutDomainMapping(bgCtx, rp.cfClient, ownerClaim.Domain, oldRepo.Did, newRkey, siteConfig.IsIndex); err != nil { + l.Error("sites: failed to write new KV mapping", "newRkey", newRkey, "err", err) + } + } + + l.Info("sites: migrated on rename", "oldRkey", oldRkey, "newRkey", newRkey) + }() +} + func (rp *Repo) DeleteRepo(w http.ResponseWriter, r *http.Request) { user := rp.oauth.GetMultiAccountUser(r) l := rp.logger.With("handler", "DeleteRepo") @@ -921,7 +1125,7 @@ return } // remove repo from db - err = db.RemoveRepo(tx, f.Did, f.Name) + err = db.RemoveRepo(tx, f.Did, f.Rkey) if err != nil { rp.pages.Notice(w, noticeId, "Failed to update appview") return @@ -1040,7 +1244,7 @@ return } // choose a name for a fork - forkName := r.FormValue("repo_name") + forkName := strings.ToLower(r.FormValue("repo_name")) if forkName == "" { rp.pages.Notice(w, "repo", "Repository name cannot be empty.") return @@ -1074,7 +1278,7 @@ forkSourceUrl := fmt.Sprintf("%s://%s/%s", uri, f.Knot, f.RepoIdentifier()) l = l.With("cloneUrl", forkSourceUrl) - rkey := tid.TID() + rkey := strings.ToLower(forkName) // TODO: this could coordinate better with the knot to receive a clone status client, err := rp.oauth.ServiceClient( @@ -1092,7 +1296,7 @@ } forkInput := &tangled.RepoCreate_Input{ Rkey: rkey, - Name: forkName, + Name: rkey, Source: &forkSourceUrl, } createResp, err := tangled.RepoCreate( @@ -1123,7 +1327,7 @@ } repo := &models.Repo{ Did: user.Did, - Name: forkName, + Name: rkey, Knot: targetKnot, Rkey: rkey, Source: forkSource, @@ -1276,15 +1480,15 @@ if page.Limit > 30 || page.Limit <= 0 { page.Limit = 30 } - starrers, err := db.GetStars(rp.db, f.RepoAt(), page) + starrers, err := db.GetStars(rp.db, string(f.RepoDid), page) if err != nil { - l.Error("failed to fetch starrers", "err", err, "repoAt", f.RepoAt()) + l.Error("failed to fetch starrers", "err", err, "repoDid", f.RepoDid) return } - totalCount, err := db.GetStarCount(rp.db, f.RepoAt()) + totalCount, err := db.GetStarCount(rp.db, models.StarSubjectRepo, string(f.RepoDid)) if err != nil { - l.Error("failed to fetch star count", "err", err, "repoAt", f.RepoAt()) + l.Error("failed to fetch star count", "err", err, "repoDid", f.RepoDid) return } @@ -1297,6 +1501,42 @@ TotalCount: totalCount, }) } +func (rp *Repo) Forks(w http.ResponseWriter, r *http.Request) { + l := rp.logger.With("handler", "Forks") + + user := rp.oauth.GetMultiAccountUser(r) + f, err := rp.repoResolver.Resolve(r) + if err != nil { + l.Error("failed to resolve source repo", "err", err) + return + } + + page := pagination.FromContext(r.Context()) + + forks, err := db.GetReposPaginated(rp.db, page, orm.FilterEq("source", f.RepoDid)) + if err != nil { + l.Error("failed to fetch forks", "err", err, "repoAt", f.RepoAt()) + return + } + + totalCount, err := db.GetForkCount(rp.db, f.RepoDid) + if err != nil { + l.Error("failed to fetch fork count", "err", err, "repoAt", f.RepoAt()) + return + } + + err = rp.pages.RepoForks(w, pages.RepoForksParams{ + LoggedInUser: user, + RepoInfo: rp.repoResolver.GetRepoInfo(r, user), + Forks: forks, + Page: page, + TotalCount: totalCount, + }) + if err != nil { + l.Error("failed to render page", "err", err) + } +} + // this is used to rollback changes made to the PDS // // it is a no-op if the provided ATURI is empty @@ -1320,14 +1560,9 @@ return err } func repoCollaboratorRecord(f *models.Repo, subject string, createdAt time.Time) *tangled.RepoCollaborator { - rec := &tangled.RepoCollaborator{ + return &tangled.RepoCollaborator{ Subject: subject, CreatedAt: createdAt.Format(time.RFC3339), + Repo: f.RepoDid, } - s := string(f.RepoAt()) - rec.Repo = &s - if f.RepoDid != "" { - rec.RepoDid = &f.RepoDid - } - return rec } diff --git a/appview/repo/repo_util.go b/appview/repo/repo_util.go --- a/appview/repo/repo_util.go +++ b/appview/repo/repo_util.go @@ -104,7 +104,7 @@ ps, err := db.GetPipelineStatuses( d, len(shas), orm.FilterEq("p.repo_owner", repo.Did), - orm.FilterEq("p.repo_name", repo.Name), + orm.FilterEq("p.repo_name", repo.Rkey), orm.FilterEq("p.knot", repo.Knot), orm.FilterIn("p.sha", shas), ) diff --git a/appview/repo/router.go b/appview/repo/router.go --- a/appview/repo/router.go +++ b/appview/repo/router.go @@ -48,6 +48,7 @@ // a file path r.Get("/archive/{ref}", rp.DownloadArchive) r.With(middleware.Paginate).Get("/stars", rp.Stars) + r.With(middleware.Paginate).Get("/forks", rp.Forks) r.Route("/fork", func(r chi.Router) { r.Use(middleware.AuthMiddleware(rp.oauth)) @@ -88,6 +89,7 @@ r.With(mw.RepoPermissionMiddleware("repo:owner")).Post("/label/subscribe", rp.SubscribeLabel) r.With(mw.RepoPermissionMiddleware("repo:owner")).Post("/label/unsubscribe", rp.UnsubscribeLabel) r.With(mw.RepoPermissionMiddleware("repo:invite")).Put("/collaborator", rp.AddCollaborator) r.With(mw.RepoPermissionMiddleware("repo:delete")).Delete("/delete", rp.DeleteRepo) + r.With(mw.RepoPermissionMiddleware("repo:owner")).Post("/rename", rp.RenameRepo) r.Put("/branches/default", rp.SetDefaultBranch) r.Put("/secrets", rp.Secrets) r.Delete("/secrets", rp.Secrets) diff --git a/appview/repo/settings.go b/appview/repo/settings.go --- a/appview/repo/settings.go +++ b/appview/repo/settings.go @@ -22,6 +22,7 @@ "tangled.org/core/orm" "tangled.org/core/types" comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/syntax" lexutil "github.com/bluesky-social/indigo/lex/util" indigoxrpc "github.com/bluesky-social/indigo/xrpc" ) @@ -195,7 +196,7 @@ } host := fmt.Sprintf("%s://%s", scheme, f.Knot) xrpcc := &indigoxrpc.Client{Host: host} - repo := fmt.Sprintf("%s/%s", f.Did, f.Name) + repo := fmt.Sprintf("%s/%s", f.Did, f.Rkey) xrpcBytes, err := tangled.RepoBranches(r.Context(), xrpcc, "", 0, repo) if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { l.Error("failed to call XRPC repo.branches", "xrpcerr", xrpcerr, "err", err) @@ -210,7 +211,7 @@ rp.pages.Error503(w) return } - siteConfig, err := db.GetRepoSiteConfig(rp.db, f.RepoAt().String()) + siteConfig, err := db.GetRepoSiteConfig(rp.db, f.RepoDid) if err != nil { l.Error("failed to get site config", "err", err) rp.pages.Error503(w) @@ -224,14 +225,14 @@ // non-fatal — just show no claim ownerClaim = nil } - deploys, err := db.GetSiteDeploys(rp.db, f.RepoAt().String(), 20) + deploys, err := db.GetSiteDeploys(rp.db, f.RepoDid, 20) if err != nil { l.Error("failed to get site deploys", "err", err) // non-fatal deploys = nil } - indexSiteTakenBy, err := db.GetIndexRepoAtForDid(rp.db, f.Did, f.RepoAt().String()) + indexSiteTakenBy, err := db.GetIndexRepoDidForDid(rp.db, f.Did, f.RepoDid) if err != nil { l.Error("failed to get index site owner", "err", err) // non-fatal @@ -281,7 +282,7 @@ } isIndex := r.FormValue("is_index") == "true" - if err := db.SetRepoSiteConfig(rp.db, f.RepoAt().String(), branch, dir, isIndex); err != nil { + if err := db.SetRepoSiteConfig(rp.db, f.RepoDid, branch, dir, isIndex); err != nil { l.Error("failed to save site config", "err", err) rp.pages.Notice(w, noticeId, "Failed to save site configuration.") return @@ -297,7 +298,7 @@ go func() { ctx := context.Background() deploy := &models.SiteDeploy{ - RepoAt: f.RepoAt().String(), + RepoDid: syntax.DID(f.RepoDid), Branch: branch, Dir: dir, Trigger: models.SiteDeployTriggerConfigChange, @@ -317,7 +318,7 @@ l.Error("sites: failed to record deploy", "repo", f.RepoIdentifier(), "err", err) } if deployErr == nil { - if err := sites.PutDomainMapping(ctx, rp.cfClient, ownerClaim.Domain, f.Did, f.Name, isIndex); err != nil { + if err := sites.PutDomainMapping(ctx, rp.cfClient, ownerClaim.Domain, f.Did, f.Rkey, isIndex); err != nil { l.Error("sites: KV write failed", "domain", ownerClaim.Domain, "err", err) } rp.logger.Info("site deployed to r2", "repo", f.RepoIdentifier(), "is_index", isIndex) @@ -344,9 +345,9 @@ } // Fetch the current config before deleting so we know the isIndex flag for // the KV key and the domain mapping to clean up. - existingConfig, _ := db.GetRepoSiteConfig(rp.db, f.RepoAt().String()) + existingConfig, _ := db.GetRepoSiteConfig(rp.db, f.RepoDid) - if err := db.DeleteRepoSiteConfig(rp.db, f.RepoAt().String()); err != nil { + if err := db.DeleteRepoSiteConfig(rp.db, f.RepoDid); err != nil { l.Error("failed to delete site config", "err", err) rp.pages.Notice(w, noticeId, "Failed to remove site configuration.") return @@ -358,11 +359,11 @@ ownerClaim, _ := db.GetActiveDomainClaimForDid(rp.db, f.Did) go func() { ctx := context.Background() - if err := sites.Delete(ctx, rp.cfClient, f.Did, f.Name); err != nil { + if err := sites.Delete(ctx, rp.cfClient, f.Did, f.Rkey); err != nil { l.Error("sites: R2 delete failed", "repo", f.RepoIdentifier(), "err", err) } if ownerClaim != nil { - if err := sites.DeleteDomainMapping(ctx, rp.cfClient, ownerClaim.Domain, f.Name); err != nil { + if err := sites.DeleteDomainMapping(ctx, rp.cfClient, ownerClaim.Domain, f.Rkey); err != nil { l.Error("sites: KV delete failed", "domain", ownerClaim.Domain, "err", err) } } diff --git a/appview/repo/tags.go b/appview/repo/tags.go --- a/appview/repo/tags.go +++ b/appview/repo/tags.go @@ -40,7 +40,7 @@ l.Error("failed to decode XRPC response", "err", err) rp.pages.Error503(w) return } - artifacts, err := db.GetArtifact(rp.db, orm.FilterEq("repo_at", f.RepoAt())) + artifacts, err := db.GetArtifact(rp.db, orm.FilterEq("repo_did", f.RepoDid)) if err != nil { l.Error("failed grab artifacts", "err", err) return @@ -124,7 +124,7 @@ rp.pages.Error503(w) return } - filters := []orm.Filter{orm.FilterEq("repo_at", f.RepoAt())} + filters := []orm.Filter{orm.FilterEq("repo_did", f.RepoDid)} if result.Tag.Tag != nil { filters = append(filters, orm.FilterEq("tag", result.Tag.Tag.Hash[:])) } diff --git a/appview/repo/webhooks.go b/appview/repo/webhooks.go --- a/appview/repo/webhooks.go +++ b/appview/repo/webhooks.go @@ -5,6 +5,7 @@ "net/http" "strconv" "strings" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/go-chi/chi/v5" "tangled.org/core/appview/db" "tangled.org/core/appview/models" @@ -24,7 +25,7 @@ } user := rp.oauth.GetMultiAccountUser(r) - webhooks, err := db.GetWebhooksForRepo(rp.db, f.RepoAt()) + webhooks, err := db.GetWebhooksForRepo(rp.db, f.RepoDid) if err != nil { l.Error("failed to get webhooks", "err", err) rp.pages.Notice(w, "webhooks-error", "Failed to load webhooks") @@ -82,18 +83,21 @@ events := []string{} if r.FormValue("event_push") == "on" { events = append(events, string(models.WebhookEventPush)) } + if r.FormValue("event_repo_renamed") == "on" { + events = append(events, string(models.WebhookEventRepoRenamed)) + } if len(events) == 0 { - rp.pages.Notice(w, "webhooks-error", "Push events must be enabled") + rp.pages.Notice(w, "webhooks-error", "At least one event must be enabled") return } webhook := &models.Webhook{ - RepoAt: f.RepoAt(), - Url: url, - Secret: secret, - Active: active, - Events: events, + RepoDid: syntax.DID(f.RepoDid), + Url: url, + Secret: secret, + Active: active, + Events: events, } tx, err := rp.db.Begin() @@ -146,8 +150,8 @@ return } // Verify webhook belongs to this repo - if webhook.RepoAt != f.RepoAt() { - l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoAt, "current_repo", f.RepoAt()) + if string(webhook.RepoDid) != f.RepoDid { + l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoDid, "current_repo", f.RepoDid) w.WriteHeader(http.StatusForbidden) return } @@ -168,10 +172,12 @@ } webhook.Active = r.FormValue("active") == "on" - // Parse events - only push events are supported for now events := []string{} if r.FormValue("event_push") == "on" { events = append(events, string(models.WebhookEventPush)) + } + if r.FormValue("event_repo_renamed") == "on" { + events = append(events, string(models.WebhookEventRepoRenamed)) } if len(events) > 0 { @@ -228,8 +234,8 @@ return } // Verify webhook belongs to this repo - if webhook.RepoAt != f.RepoAt() { - l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoAt, "current_repo", f.RepoAt()) + if string(webhook.RepoDid) != f.RepoDid { + l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoDid, "current_repo", f.RepoDid) w.WriteHeader(http.StatusForbidden) return } @@ -284,8 +290,8 @@ return } // Verify webhook belongs to this repo - if webhook.RepoAt != f.RepoAt() { - l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoAt, "current_repo", f.RepoAt()) + if string(webhook.RepoDid) != f.RepoDid { + l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoDid, "current_repo", f.RepoDid) w.WriteHeader(http.StatusForbidden) return } @@ -343,8 +349,8 @@ return } // Verify webhook belongs to this repo - if webhook.RepoAt != f.RepoAt() { - l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoAt, "current_repo", f.RepoAt()) + if string(webhook.RepoDid) != f.RepoDid { + l.Error("webhook does not belong to repo", "webhook_repo", webhook.RepoDid, "current_repo", f.RepoDid) w.WriteHeader(http.StatusForbidden) return } diff --git a/appview/reporesolver/resolver.go b/appview/reporesolver/resolver.go --- a/appview/reporesolver/resolver.go +++ b/appview/reporesolver/resolver.go @@ -76,27 +76,27 @@ // get dir/ref currentDir := extractCurrentDir(r.URL.EscapedPath()) ref := chi.URLParam(r, "ref") - repoAt := repo.RepoAt() + repoDid := repo.RepoDid isStarred := false roles := repoinfo.RolesInRepo{} if user != nil { - isStarred = db.GetStarStatus(rr.execer, user.Did, repoAt) + isStarred = db.GetStarStatus(rr.execer, user.Did, repoDid) roles.Roles = rr.enforcer.GetPermissionsInRepo(user.Did, repo.Knot, repo.RepoIdentifier()) } stats := repo.RepoStats if stats == nil { - starCount, starErr := db.GetStarCount(rr.execer, repoAt) + starCount, starErr := db.GetStarCount(rr.execer, models.StarSubjectRepo, repoDid) if starErr != nil { - log.Println("failed to get star count for ", repoAt) + log.Println("failed to get star count for ", repoDid) } - issueCount, err := db.GetIssueCount(rr.execer, repoAt) + issueCount, err := db.GetIssueCount(rr.execer, repoDid) if err != nil { - log.Println("failed to get issue count for ", repoAt) + log.Println("failed to get issue count for ", repoDid) } - pullCount, err := db.GetPullCount(rr.execer, repoAt) + pullCount, err := db.GetPullCount(rr.execer, repoDid) if err != nil { - log.Println("failed to get pull count for ", repoAt) + log.Println("failed to get pull count for ", repoDid) } stats = &models.RepoStats{ StarCount: starCount, diff --git a/appview/repoverify/verify.go b/appview/repoverify/verify.go new file mode 100644 --- /dev/null +++ b/appview/repoverify/verify.go @@ -0,0 +1,159 @@ +package repoverify + +import ( + "context" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "syscall" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" + indigoxrpc "github.com/bluesky-social/indigo/xrpc" + "tangled.org/core/api/tangled" + "tangled.org/core/appview/xrpcclient" + "tangled.org/core/idresolver" +) + +type RepoDid syntax.DID + +func (r RepoDid) String() string { return string(r) } + +func NewRepoDid(s string) (RepoDid, error) { + did, err := syntax.ParseDID(s) + if err != nil { + return "", fmt.Errorf("invalid repoDid %q: %w", s, err) + } + return RepoDid(did), nil +} + +type OwnerDid syntax.DID + +func (o OwnerDid) String() string { return string(o) } + +func NewOwnerDid(s string) (OwnerDid, error) { + did, err := syntax.ParseDID(s) + if err != nil { + return "", fmt.Errorf("invalid ownerDid %q: %w", s, err) + } + return OwnerDid(did), nil +} + +func ParseKnotEndpoint(raw string, dev bool) (*url.URL, error) { + if raw == "" { + return nil, fmt.Errorf("empty knot URL") + } + u, err := url.Parse(raw) + if err != nil { + return nil, fmt.Errorf("invalid knot URL %q: %w", raw, err) + } + if u.Host == "" { + return nil, fmt.Errorf("knot URL %q has no host", raw) + } + switch u.Scheme { + case "https": + case "http": + if !dev { + return nil, fmt.Errorf("knot URL %q must use https outside dev mode", raw) + } + default: + return nil, fmt.Errorf("knot URL %q has unsupported scheme %q", raw, u.Scheme) + } + return u, nil +} + +type Result struct { + RepoDid RepoDid + OwnerDid OwnerDid + KnotURL *url.URL +} + +type Verifier func(ctx context.Context, repoDid RepoDid) (Result, error) + +const verifyTimeout = 10 * time.Second + +func New(resolver *idresolver.Resolver, dev bool) Verifier { + transport := &http.Transport{ + DialContext: safeDialer(dev).DialContext, + } + httpClient := &http.Client{ + Timeout: verifyTimeout, + Transport: transport, + } + + return func(ctx context.Context, repoDid RepoDid) (Result, error) { + ctx, cancel := context.WithTimeout(ctx, verifyTimeout) + defer cancel() + return resolveAndDescribe(ctx, resolver, httpClient, repoDid, dev) + } +} + +func resolveAndDescribe( + ctx context.Context, + resolver *idresolver.Resolver, + httpClient *http.Client, + repoDid RepoDid, + dev bool, +) (Result, error) { + ident, err := resolver.ResolveIdent(ctx, repoDid.String()) + if err != nil { + return Result{}, fmt.Errorf("resolve repoDid %s: %w", repoDid, err) + } + + knot, err := ParseKnotEndpoint(ident.GetServiceEndpoint("atproto_pds"), dev) + if err != nil { + return Result{}, fmt.Errorf("repoDid %s: %w", repoDid, err) + } + + client := &indigoxrpc.Client{Host: knot.String(), Client: httpClient} + out, err := tangled.RepoDescribeRepo(ctx, client, repoDid.String()) + if xrpcErr := xrpcclient.HandleXrpcErr(err); xrpcErr != nil { + if errors.Is(xrpcErr, xrpcclient.ErrXrpcUnsupported) { + return Result{RepoDid: repoDid, KnotURL: knot}, nil + } + return Result{}, fmt.Errorf("describeRepo on %s: %w", knot, xrpcErr) + } + + if out.RepoDid != repoDid.String() { + return Result{}, fmt.Errorf("knot %s returned mismatched repoDid: got %q, want %q", knot, out.RepoDid, repoDid) + } + + ownerDid, err := NewOwnerDid(out.OwnerDid) + if err != nil { + return Result{}, fmt.Errorf("describeRepo on %s returned invalid ownerDid: %w", knot, err) + } + + return Result{ + RepoDid: repoDid, + OwnerDid: ownerDid, + KnotURL: knot, + }, nil +} + +func safeDialer(dev bool) *net.Dialer { + d := &net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 30 * time.Second, + } + if dev { + return d + } + d.Control = func(network, address string, _ syscall.RawConn) error { + host, _, err := net.SplitHostPort(address) + if err != nil { + return fmt.Errorf("invalid dial address %q: %w", address, err) + } + ip := net.ParseIP(host) + if ip == nil { + return fmt.Errorf("dial address %q did not resolve to IP", address) + } + if ip.IsLoopback() || ip.IsPrivate() || ip.IsLinkLocalUnicast() || + ip.IsLinkLocalMulticast() || ip.IsMulticast() || ip.IsUnspecified() { + return fmt.Errorf("refusing to dial %s: reserved or private address", ip) + } + return nil + } + return d +} diff --git a/appview/repoverify/verify_test.go b/appview/repoverify/verify_test.go new file mode 100644 --- /dev/null +++ b/appview/repoverify/verify_test.go @@ -0,0 +1,66 @@ +package repoverify + +import "testing" + +func TestNewRepoDid_RejectsInvalid(t *testing.T) { + if _, err := NewRepoDid(""); err == nil { + t.Error("NewRepoDid(\"\") err = nil, want error") + } +} + +func TestNewRepoDid_AcceptsValid(t *testing.T) { + raw := "did:plc:abc123abc123abc123abc123" + got, err := NewRepoDid(raw) + if err != nil { + t.Fatalf("NewRepoDid(%q): %v", raw, err) + } + if got.String() != raw { + t.Errorf("got %q, want %q", got, raw) + } +} + +func TestParseKnotEndpoint_RejectsHttpInProd(t *testing.T) { + if _, err := ParseKnotEndpoint("http://knot.example", false); err == nil { + t.Error("http:// knot URL accepted in prod") + } +} + +func TestParseKnotEndpoint_AllowsHttpInDev(t *testing.T) { + u, err := ParseKnotEndpoint("http://knot.example", true) + if err != nil { + t.Fatalf("dev mode should allow http: %v", err) + } + if u.Host != "knot.example" { + t.Errorf("Host = %q, want knot.example", u.Host) + } +} + +func TestParseKnotEndpoint_RejectsUnsupportedScheme(t *testing.T) { + if _, err := ParseKnotEndpoint("ftp://knot.example", true); err == nil { + t.Error("ParseKnotEndpoint accepted ftp:// in dev") + } + if _, err := ParseKnotEndpoint("ftp://knot.example", false); err == nil { + t.Error("ParseKnotEndpoint accepted ftp:// in prod") + } +} + +func TestParseKnotEndpoint_RejectsEmptyOrHostless(t *testing.T) { + cases := []string{"", "https://", "not a url at all"} + for _, raw := range cases { + t.Run(raw, func(t *testing.T) { + if _, err := ParseKnotEndpoint(raw, false); err == nil { + t.Errorf("ParseKnotEndpoint(%q) accepted bogus URL", raw) + } + }) + } +} + +func TestParseKnotEndpoint_HostPreservesPort(t *testing.T) { + u, err := ParseKnotEndpoint("http://localhost:3000", true) + if err != nil { + t.Fatalf("ParseKnotEndpoint: %v", err) + } + if u.Host != "localhost:3000" { + t.Errorf("Host = %q, want localhost:3000", u.Host) + } +} diff --git a/appview/settings/settings.go b/appview/settings/settings.go --- a/appview/settings/settings.go +++ b/appview/settings/settings.go @@ -224,8 +224,8 @@ ctx := context.Background() // Delete each repo's R2 objects. for _, sc := range siteConfigs { - if err := sites.Delete(ctx, s.CfClient, user.Did, sc.RepoName); err != nil { - s.Logger.Error("releaseSitesDomain: R2 delete failed", "did", user.Did, "repo", sc.RepoName, "err", err) + if err := sites.Delete(ctx, s.CfClient, user.Did, sc.RepoRkey); err != nil { + s.Logger.Error("releaseSitesDomain: R2 delete failed", "did", user.Did, "repo", sc.RepoRkey, "err", err) } } diff --git a/appview/sites/sites.go b/appview/sites/sites.go --- a/appview/sites/sites.go +++ b/appview/sites/sites.go @@ -158,7 +158,7 @@ if err != nil { return fmt.Errorf("walking deploy dir: %w", err) } - if err := cf.SyncFiles(ctx, prefix(f.Did, f.Name), files); err != nil { + if err := cf.SyncFiles(ctx, prefix(f.Did, f.Rkey), files); err != nil { return fmt.Errorf("syncing files to R2: %w", err) } diff --git a/appview/state/gfi.go b/appview/state/gfi.go --- a/appview/state/gfi.go +++ b/appview/state/gfi.go @@ -4,7 +4,6 @@ import ( "net/http" "sort" - "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/pages" @@ -48,7 +47,7 @@ } repoUris := make([]string, 0, len(repoLabels)) for _, rl := range repoLabels { - repoUris = append(repoUris, rl.RepoAt.String()) + repoUris = append(repoUris, string(rl.RepoDid)) } allIssues, err := db.GetIssuesPaginated( @@ -56,7 +55,7 @@ s.db, pagination.Page{ Limit: 500, }, - orm.FilterIn("repo_at", repoUris), + orm.FilterIn("repo_did", repoUris), orm.FilterEq("open", 1), ) if err != nil { @@ -72,12 +71,12 @@ goodFirstIssues = append(goodFirstIssues, issue) } } - repoGroups := make(map[syntax.ATURI]*models.RepoGroup) + repoGroups := make(map[string]*models.RepoGroup) for _, issue := range goodFirstIssues { - if group, exists := repoGroups[issue.Repo.RepoAt()]; exists { + if group, exists := repoGroups[issue.Repo.RepoDid]; exists { group.Issues = append(group.Issues, issue) } else { - repoGroups[issue.Repo.RepoAt()] = &models.RepoGroup{ + repoGroups[issue.Repo.RepoDid] = &models.RepoGroup{ Repo: issue.Repo, Issues: []models.Issue{issue}, } diff --git a/appview/state/knotstream.go b/appview/state/knotstream.go --- a/appview/state/knotstream.go +++ b/appview/state/knotstream.go @@ -7,6 +7,7 @@ "encoding/json" "errors" "fmt" "slices" + "strings" "time" "tangled.org/core/appview/cloudflare" @@ -72,7 +73,7 @@ func resolveRepo(d *db.DB, repoDid *string, ownerDid, repoName string) (*models.Repo, error) { if repoDid != nil && *repoDid != "" { return db.GetRepoByDid(d, *repoDid) } - repos, err := db.GetRepos(d, orm.FilterEq("did", ownerDid), orm.FilterEq("name", repoName)) + repos, err := db.GetRepos(d, orm.FilterEq("did", ownerDid), orm.FilterEq("rkey", strings.ToLower(repoName))) if err != nil { return nil, err } @@ -114,17 +115,11 @@ if !slices.Contains(knownKnots, source.Key()) { return fmt.Errorf("%s does not belong to %s, something is fishy", record.CommitterDid, source.Key()) } - ownerDid := "" - if record.OwnerDid != nil { - ownerDid = *record.OwnerDid - } else { - // handle legacy event - if record.RepoDid != nil { - ownerDid = *record.RepoDid - } + if record.Repo == "" { + return fmt.Errorf("gitRefUpdate from %s missing repo", source.Key()) } - repo, lookupErr := resolveRepo(d, record.RepoDid, ownerDid, record.RepoName) + repo, lookupErr := db.GetRepoByDid(d, record.Repo) if lookupErr != nil { return fmt.Errorf("failed to look up repo: %w", lookupErr) } @@ -167,17 +162,12 @@ return } pushedBranch := ref.Short() - ownerDid := "" - if record.OwnerDid != nil { - ownerDid = *record.OwnerDid - } - - repo, err := resolveRepo(d, record.RepoDid, ownerDid, record.RepoName) + repo, err := db.GetRepoByDid(d, record.Repo) if err != nil { return } - siteConfig, err := db.GetRepoSiteConfig(d, repo.RepoAt().String()) + siteConfig, err := db.GetRepoSiteConfig(d, repo.RepoDid) if err != nil || siteConfig == nil { return } @@ -186,7 +176,7 @@ return } deploy := &models.SiteDeploy{ - RepoAt: repo.RepoAt().String(), + RepoDid: syntax.DID(repo.RepoDid), Branch: siteConfig.Branch, Dir: siteConfig.Dir, CommitSHA: record.NewSha, @@ -249,15 +239,10 @@ } func updateRepoLanguages(d *db.DB, record tangled.GitRefUpdate) error { if record.Meta == nil || record.Meta.LangBreakdown == nil || record.Meta.LangBreakdown.Inputs == nil { - return fmt.Errorf("empty language data for repo: %v/%s", record.OwnerDid, record.RepoName) + return fmt.Errorf("empty language data for repo: %s", record.Repo) } - ownerDid := "" - if record.OwnerDid != nil { - ownerDid = *record.OwnerDid - } - - r, lookupErr := resolveRepo(d, record.RepoDid, ownerDid, record.RepoName) + r, lookupErr := db.GetRepoByDid(d, record.Repo) if lookupErr != nil { return fmt.Errorf("failed to look up repo: %w", lookupErr) } @@ -275,7 +260,7 @@ continue } langs = append(langs, models.RepoLanguage{ - RepoAt: repo.RepoAt(), + RepoDid: syntax.DID(repo.RepoDid), Ref: ref.Short(), IsDefaultRef: record.Meta.IsDefaultRef, Language: l.Lang, @@ -290,7 +275,7 @@ } defer tx.Rollback() // update appview's cache - err = db.UpdateRepoLanguages(tx, repo.RepoAt(), ref.Short(), langs) + err = db.UpdateRepoLanguages(tx, syntax.DID(repo.RepoDid), ref.Short(), langs) if err != nil { fmt.Printf("failed; %s\n", err) // non-fatal @@ -398,7 +383,7 @@ "repo_name", record.RepoName) repos, err := db.GetRepos(d, orm.FilterEq("did", record.OwnerDid), - orm.FilterEq("name", record.RepoName), + orm.FilterEq("rkey", strings.ToLower(record.RepoName)), ) if err != nil || len(repos) == 0 { logger.Warn("didAssign for unknown repo, skipping", @@ -443,7 +428,7 @@ if err := enforcer.AddRepo(record.OwnerDid, knot, record.RepoDid); err != nil { return fmt.Errorf("add RBAC policies for %s: %w", record.RepoDid, err) } - collabs, collabErr := db.GetCollaborators(d, orm.FilterEq("repo_at", repoAtUri)) + collabs, collabErr := db.GetCollaborators(d, orm.FilterEq("repo_did", record.RepoDid)) if collabErr != nil { return fmt.Errorf("get collaborators for RBAC update: %w", collabErr) } diff --git a/appview/state/profile.go b/appview/state/profile.go --- a/appview/state/profile.go +++ b/appview/state/profile.go @@ -725,7 +725,7 @@ func (s *State) createPullRequestItem(pull *models.Pull, owner *identity.Identity, author *feeds.Author) *feeds.Item { return &feeds.Item{ Title: fmt.Sprintf("%s created pull request '%s' in @%s/%s", author.Name, pull.Title, owner.Handle, pull.Repo.Name), - Link: &feeds.Link{Href: fmt.Sprintf("%s/@%s/%s/pulls/%d", s.config.Core.BaseUrl(), owner.Handle, pull.Repo.Name, pull.PullId), Type: "text/html", Rel: "alternate"}, + Link: &feeds.Link{Href: fmt.Sprintf("%s/@%s/%s/pulls/%d", s.config.Core.BaseUrl(), owner.Handle, pull.Repo.Rkey, pull.PullId), Type: "text/html", Rel: "alternate"}, Created: pull.Created, Author: author, } @@ -734,7 +734,7 @@ func (s *State) createIssueItem(issue *models.Issue, owner *identity.Identity, author *feeds.Author) *feeds.Item { return &feeds.Item{ Title: fmt.Sprintf("%s created issue '%s' in @%s/%s", author.Name, issue.Title, owner.Handle, issue.Repo.Name), - Link: &feeds.Link{Href: fmt.Sprintf("%s/@%s/%s/issues/%d", s.config.Core.BaseUrl(), owner.Handle, issue.Repo.Name, issue.IssueId), Type: "text/html", Rel: "alternate"}, + Link: &feeds.Link{Href: fmt.Sprintf("%s/@%s/%s/issues/%d", s.config.Core.BaseUrl(), owner.Handle, issue.Repo.Rkey, issue.IssueId), Type: "text/html", Rel: "alternate"}, Created: issue.Created, Author: author, } @@ -754,7 +754,7 @@ } return &feeds.Item{ Title: title, - Link: &feeds.Link{Href: fmt.Sprintf("%s/@%s/%s", s.config.Core.BaseUrl(), author.Name[1:], repo.Repo.Name), Type: "text/html", Rel: "alternate"}, // Remove @ prefix + Link: &feeds.Link{Href: fmt.Sprintf("%s/@%s/%s", s.config.Core.BaseUrl(), author.Name[1:], repo.Repo.Rkey), Type: "text/html", Rel: "alternate"}, // Remove @ prefix Created: repo.Repo.Created, Author: author, }, nil diff --git a/appview/state/router.go b/appview/state/router.go --- a/appview/state/router.go +++ b/appview/state/router.go @@ -11,6 +11,7 @@ "tangled.org/core/appview/db" "tangled.org/core/appview/issues" "tangled.org/core/appview/knots" "tangled.org/core/appview/labels" + "tangled.org/core/appview/metrics" "tangled.org/core/appview/middleware" "tangled.org/core/appview/migration" "tangled.org/core/appview/notifications" @@ -37,6 +38,8 @@ s.pages, s.rdb, s.logger, ) + + router.Use(metrics.Middleware) m := migration.NewMigration(s.db, s.oauth, s.idResolver.Directory(), s.logger) router.Use(m.BackgroundMigrationMiddleware) @@ -63,7 +66,7 @@ remaining := "" if len(pathParts) > 1 { remaining = "/" + pathParts[1] } - rewritten := "/" + repo.Did + "/" + repo.Name + remaining + rewritten := "/" + repo.Did + "/" + repo.Rkey + remaining r2 := r.Clone(r.Context()) r2.URL.Path = rewritten r2.URL.RawPath = rewritten diff --git a/appview/state/star.go b/appview/state/star.go --- a/appview/state/star.go +++ b/appview/state/star.go @@ -1,6 +1,8 @@ package state import ( + "fmt" + "log" "net/http" "time" @@ -11,9 +13,37 @@ "tangled.org/core/api/tangled" "tangled.org/core/appview/db" "tangled.org/core/appview/models" "tangled.org/core/appview/pages" - "tangled.org/core/orm" "tangled.org/core/tid" ) + +func resolveStarSubject(d db.Execer, subjectUri syntax.ATURI) (models.StarSubjectType, string, *tangled.FeedStar_Subject, error) { + collection := subjectUri.Collection() + + switch collection.String() { + case tangled.RepoNSID: + repo, err := db.GetRepoByAtUri(d, subjectUri.String()) + if err != nil { + return "", "", nil, err + } + if repo.RepoDid == "" { + return "", "", nil, fmt.Errorf("repo has no DID: %s", subjectUri) + } + subject := &tangled.FeedStar_Subject{ + FeedStar_Repo: &tangled.FeedStar_Repo{Did: repo.RepoDid}, + } + return models.StarSubjectRepo, repo.RepoDid, subject, nil + + case tangled.StringNSID: + uri := subjectUri.String() + subject := &tangled.FeedStar_Subject{ + FeedStar_String: &tangled.FeedStar_String{Uri: uri}, + } + return models.StarSubjectString, uri, subject, nil + + default: + return "", "", nil, fmt.Errorf("unsupported star subject collection: %s", collection) + } +} func (s *State) Star(w http.ResponseWriter, r *http.Request) { l := s.logger.With("handler", "Star") @@ -31,6 +61,12 @@ l.Warn("invalid form", "subject", subject, "err", err) return } + subjectType, subjectKey, starSubject, err := resolveStarSubject(s.db, subjectUri) + if err != nil { + log.Println("failed to resolve star subject", err) + return + } + client, err := s.oauth.AuthorizedClient(r) if err != nil { l.Error("failed to authorize client", "err", err) @@ -44,14 +80,9 @@ case http.MethodPost: createdAt := time.Now().Format(time.RFC3339) rkey := tid.TID() - subjectStr := subjectUri.String() starRecord := &tangled.FeedStar{ CreatedAt: createdAt, - Subject: &subjectStr, - } - repo, err := db.GetRepo(s.db, orm.FilterEq("at_uri", subjectUri.String())) - if err == nil && repo.RepoDid != "" { - starRecord.SubjectDid = &repo.RepoDid + Subject: starSubject, } resp, err := comatproto.RepoPutRecord(r.Context(), client, &comatproto.RepoPutRecord_Input{ @@ -67,9 +98,10 @@ } l.Info("created atproto record", "uri", resp.Uri) star := &models.Star{ - Did: currentUser.Did, - RepoAt: subjectUri, - Rkey: rkey, + Did: currentUser.Did, + SubjectType: subjectType, + Subject: subjectKey, + Rkey: rkey, } err = db.AddStar(s.db, star) @@ -78,9 +110,9 @@ l.Error("failed to star", "err", err) return } - starCount, err := db.GetStarCount(s.db, subjectUri) + starCount, err := db.GetStarCount(s.db, subjectType, subjectKey) if err != nil { - l.Error("failed to get star count", "subjectUri", subjectUri, "err", err) + l.Error("failed to get star count", "subject", subjectKey, "err", err) } s.notifier.NewStar(r.Context(), star) @@ -95,7 +127,7 @@ return case http.MethodDelete: // find the record in the db - star, err := db.GetStar(s.db, currentUser.Did, subjectUri) + star, err := db.GetStar(s.db, currentUser.Did, subjectKey) if err != nil { l.Error("failed to get star relationship", "err", err) return @@ -118,9 +150,9 @@ l.Warn("failed to delete star from DB", "err", err) // this is not an issue, the firehose event might have already done this } - starCount, err := db.GetStarCount(s.db, subjectUri) + starCount, err := db.GetStarCount(s.db, subjectType, subjectKey) if err != nil { - l.Error("failed to get star count", "subjectUri", subjectUri, "err", err) + l.Error("failed to get star count", "subject", subjectKey, "err", err) return } @@ -135,5 +167,4 @@ }) return } - } diff --git a/appview/state/state.go b/appview/state/state.go --- a/appview/state/state.go +++ b/appview/state/state.go @@ -29,6 +29,7 @@ whnotify "tangled.org/core/appview/notify/webhook" "tangled.org/core/appview/oauth" "tangled.org/core/appview/pages" "tangled.org/core/appview/reporesolver" + "tangled.org/core/appview/repoverify" "tangled.org/core/appview/validator" xrpcclient "tangled.org/core/appview/xrpcclient" "tangled.org/core/consts" @@ -39,7 +40,6 @@ "tangled.org/core/log" tlog "tangled.org/core/log" "tangled.org/core/orm" "tangled.org/core/rbac" - "tangled.org/core/tid" comatproto "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/atproto/atclient" @@ -118,7 +118,6 @@ repoResolver := reporesolver.New(config, enforcer, d, rdb) mentionsResolver := mentions.New(config, res, d, log.SubLogger(logger, "mentionsResolver")) - wrapper := db.DbWrapper{Execer: d} jc, err := jetstream.NewJetstreamClient( config.Jetstream.Endpoint, "appview", @@ -135,6 +134,7 @@ tangled.PublicKeyNSID, tangled.RepoArtifactNSID, tangled.RepoIssueCommentNSID, tangled.RepoIssueNSID, + tangled.RepoNSID, tangled.RepoPullNSID, tangled.SpindleMemberNSID, tangled.SpindleNSID, @@ -142,7 +142,7 @@ tangled.StringNSID, }, nil, tlog.SubLogger(logger, "jetstream"), - wrapper, + d, false, // in-memory filter is inapplicable to appview so @@ -157,20 +157,6 @@ if err := BackfillDefaultDefs(d, res, config.Label.DefaultLabelDefs); err != nil { return nil, fmt.Errorf("failed to backfill default label defs: %w", err) } - ingester := appview.Ingester{ - Db: wrapper, - Enforcer: enforcer, - IdResolver: res, - Cache: rdb, - Config: config, - Logger: log.SubLogger(logger, "ingester"), - Validator: validator, - } - err = jc.StartJetstream(ctx, ingester.Ingest()) - if err != nil { - return nil, fmt.Errorf("failed to start jetstream watcher: %w", err) - } - var notifiers []notify.Notifier // Always add the database notifier @@ -186,6 +172,22 @@ notifiers = append(notifiers, whnotify.NewNotifier(d)) notifier := notify.NewMergedNotifier(notifiers) notifier = lognotify.NewLoggingNotifier(notifier, tlog.SubLogger(logger, "notify")) + + ingester := appview.Ingester{ + Db: d, + Enforcer: enforcer, + IdResolver: res, + Cache: rdb, + Config: config, + Logger: log.SubLogger(logger, "ingester"), + Validator: validator, + Notifier: notifier, + Verifier: repoverify.New(res, config.Core.Dev), + } + err = jc.StartJetstream(ctx, ingester.Ingest()) + if err != nil { + return nil, fmt.Errorf("failed to start jetstream watcher: %w", err) + } var cfClient *cloudflare.Client if config.Cloudflare.ApiToken != "" { @@ -424,42 +426,6 @@ fmt.Fprintln(w, key) } } -func validateRepoName(name string) error { - // check for path traversal attempts - if name == "." || name == ".." || - strings.Contains(name, "/") || strings.Contains(name, "\\") { - return fmt.Errorf("Repository name contains invalid path characters") - } - - // check for sequences that could be used for traversal when normalized - if strings.Contains(name, "./") || strings.Contains(name, "../") || - strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".") { - return fmt.Errorf("Repository name contains invalid path sequence") - } - - // then continue with character validation - for _, char := range name { - if !((char >= 'a' && char <= 'z') || - (char >= 'A' && char <= 'Z') || - (char >= '0' && char <= '9') || - char == '-' || char == '_' || char == '.') { - return fmt.Errorf("Repository name can only contain alphanumeric characters, periods, hyphens, and underscores") - } - } - - // additional check to prevent multiple sequential dots - if strings.Contains(name, "..") { - return fmt.Errorf("Repository name cannot contain sequential dots") - } - - // if all checks pass - return nil -} - -func stripGitExt(name string) string { - return strings.TrimSuffix(name, ".git") -} - func (s *State) NewRepo(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: @@ -495,12 +461,13 @@ s.pages.Notice(w, "repo", "Repository name cannot be empty.") return } - if err := validateRepoName(repoName); err != nil { + if err := models.ValidateRepoName(repoName); err != nil { s.pages.Notice(w, "repo", err.Error()) return } - repoName = stripGitExt(repoName) - l = l.With("repoName", repoName) + repoName = models.StripGitExt(repoName) + rkey := strings.ToLower(repoName) + l = l.With("repoName", repoName, "rkey", rkey) defaultBranch := r.FormValue("branch") if defaultBranch == "" { @@ -526,7 +493,7 @@ // Check for existing repos existingRepo, err := db.GetRepo( s.db, orm.FilterEq("did", user.Did), - orm.FilterEq("name", repoName), + orm.FilterEq("rkey", rkey), ) if err == nil && existingRepo != nil { l.Info("repo exists") @@ -534,7 +501,18 @@ s.pages.Notice(w, "repo", fmt.Sprintf("You already have a repository by this name on %s", existingRepo.Knot)) return } - rkey := tid.TID() + atpClient, err := s.oauth.AuthorizedClient(r) + if err != nil { + l.Error("failed to get authorized client", "err", err) + s.pages.Notice(w, "repo", "Failed to authorize. Try again later.") + return + } + + if rkeyOccupied(r.Context(), atpClient, user.Did, rkey) { + l.Info("rkey occupied by prior rename alias") + s.pages.Notice(w, "repo", fmt.Sprintf("The name %q still has a record on your PDS from a prior rename. Pick a different name, or delete at://%s/%s/%s first.", rkey, user.Did, tangled.RepoNSID, rkey)) + return + } client, err := s.oauth.ServiceClient( r, @@ -550,7 +528,7 @@ } input := &tangled.RepoCreate_Input{ Rkey: rkey, - Name: repoName, + Name: rkey, DefaultBranch: &defaultBranch, } createResp, err := tangled.RepoCreate( @@ -604,7 +582,7 @@ } ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) if dErr := tangled.RepoDelete(ctx, deleteClient, &tangled.RepoDelete_Input{ Did: user.Did, - Name: repoName, + Name: rkey, Rkey: rkey, }); dErr != nil { cancel() @@ -620,18 +598,10 @@ "did", user.Did, "repo", repoName, "knot", domain) }() } - atpClient, err := s.oauth.AuthorizedClient(r) - if err != nil { - l.Info("PDS write failed", "err", err) - cleanupKnot() - s.pages.Notice(w, "repo", "Failed to write record to PDS.") - return - } - - atresp, err := comatproto.RepoPutRecord(r.Context(), atpClient, &comatproto.RepoPutRecord_Input{ + _, err = comatproto.RepoCreateRecord(r.Context(), atpClient, &comatproto.RepoCreateRecord_Input{ Collection: tangled.RepoNSID, Repo: user.Did, - Rkey: rkey, + Rkey: &rkey, Record: &lexutil.LexiconTypeDecoder{ Val: &record, }, @@ -639,11 +609,15 @@ }) if err != nil { l.Info("PDS write failed", "err", err) cleanupKnot() - s.pages.Notice(w, "repo", "Failed to announce repository creation.") + if rkeyOccupied(r.Context(), atpClient, user.Did, rkey) { + s.pages.Notice(w, "repo", fmt.Sprintf("You already have a repository named %q.", rkey)) + } else { + s.pages.Notice(w, "repo", "Failed to announce repository creation.") + } return } - aturi := atresp.Uri + aturi := fmt.Sprintf("at://%s/%s/%s", user.Did, tangled.RepoNSID, rkey) l = l.With("aturi", aturi) l.Info("wrote to PDS") @@ -710,9 +684,16 @@ case repoDid != "": s.pages.HxLocation(w, fmt.Sprintf("/%s", repoDid)) default: handle := s.pages.DisplayHandle(r.Context(), user.Did) - s.pages.HxLocation(w, fmt.Sprintf("/%s/%s", handle, repoName)) + s.pages.HxLocation(w, fmt.Sprintf("/%s/%s", handle, rkey)) } } +} + +func rkeyOccupied(ctx context.Context, client *atclient.APIClient, did, rkey string) bool { + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + resp, err := comatproto.RepoGetRecord(probeCtx, client, "", tangled.RepoNSID, did, rkey) + return err == nil && resp != nil } // this is used to rollback changes made to the PDS diff --git a/appview/strings/strings.go b/appview/strings/strings.go --- a/appview/strings/strings.go +++ b/appview/strings/strings.go @@ -149,14 +149,15 @@ renderToggle = true showRendered = r.URL.Query().Get("code") != "true" } - starCount, err := db.GetStarCount(s.Db, string.AtUri()) + stringUri := string.AtUri().String() + starCount, err := db.GetStarCount(s.Db, models.StarSubjectString, stringUri) if err != nil { l.Error("failed to get star count", "err", err) } user := s.OAuth.GetMultiAccountUser(r) isStarred := false if user != nil { - isStarred = db.GetStarStatus(s.Db, user.Did, string.AtUri()) + isStarred = db.GetStarStatus(s.Db, user.Did, stringUri) } s.Pages.SingleString(w, pages.SingleStringParams{ diff --git a/cmd/appview/main.go b/cmd/appview/main.go --- a/cmd/appview/main.go +++ b/cmd/appview/main.go @@ -5,6 +5,7 @@ "context" "net/http" "os" + "github.com/prometheus/client_golang/prometheus/promhttp" "tangled.org/core/appview/config" "tangled.org/core/appview/state" tlog "tangled.org/core/log" @@ -34,6 +35,14 @@ os.Exit(-1) } logger.Info("starting server", "address", c.Core.ListenAddr) + + go func() { + logger.Info("starting metrics server", "address", c.Core.MetricsListenAddr) + http.Handle("/metrics", promhttp.Handler()) + if err := http.ListenAndServe(c.Core.MetricsListenAddr, nil); err != nil { + logger.Error("failed to start metrics server", "err", err) + } + }() if err := http.ListenAndServe(c.Core.ListenAddr, state.Router()); err != nil { logger.Error("failed to start appview", "err", err) diff --git a/cmd/cborgen/cborgen.go b/cmd/cborgen/cborgen.go --- a/cmd/cborgen/cborgen.go +++ b/cmd/cborgen/cborgen.go @@ -17,6 +17,8 @@ "tangled", tangled.ActorProfile{}, tangled.FeedReaction{}, tangled.FeedStar{}, + tangled.FeedStar_Repo{}, + tangled.FeedStar_String{}, tangled.GitRefUpdate{}, tangled.GitRefUpdate_CommitCountBreakdown{}, tangled.GitRefUpdate_IndividualEmailCommitCount{}, diff --git a/cmd/knotmirror/main.go b/cmd/knotmirror/main.go --- a/cmd/knotmirror/main.go +++ b/cmd/knotmirror/main.go @@ -11,6 +11,8 @@ "github.com/carlmjohnson/versioninfo" "github.com/urfave/cli/v3" "tangled.org/core/knotmirror" "tangled.org/core/knotmirror/config" + "tangled.org/core/knotmirror/db" + "tangled.org/core/knotmirror/migrate" "tangled.org/core/log" ) @@ -42,6 +44,12 @@ Usage: "run the knotmirror daemon", Action: runKnotMirror, Flags: []cli.Flag{}, }, + { + Name: "migrate-disk", + Usage: "rename mirror dirs from {did}/{rkey} -> {repo_did}; daemon must be stopped", + Action: runMigrateDisk, + Flags: []cli.Flag{}, + }, } return app.Run(ctx, args) } @@ -56,3 +64,20 @@ logger.Debug("config loaded:", "config", cfg) return knotmirror.Run(ctx, cfg) } + +func runMigrateDisk(ctx context.Context, cmd *cli.Command) error { + logger := log.FromContext(ctx) + cfg, err := config.Load(ctx) + if err != nil { + return err + } + database, err := db.Make(ctx, cfg.DbUrl, 4) + if err != nil { + return err + } + defer database.Close() + + stats, err := migrate.RenameDisk(ctx, cfg.GitRepoBasePath, database, logger) + logger.Info("migrate-disk complete", "stats", stats.String()) + return err +} diff --git a/docs/DOCS.md b/docs/DOCS.md --- a/docs/DOCS.md +++ b/docs/DOCS.md @@ -1500,6 +1500,22 @@ This document is laid out in reverse-chronological order. Newer migration guides are listed first, and older guides are further down the page. +## Upgrading to v1.14.0-alpha + +Starting with v1.14.0-alpha, the fully knot uses the repoDID as its +canonical handle for repositories. This unlocks repository +renames from the appview UI and changes the wire format for +the following lexicons (`sh.tangled.repo.pull`, `sh.tangled.repo.collaborator`, +`sh.tangled.repo.issue`, `sh.tangled.git.refUpdate`). + +Knots that have not been upgraded may silently drop new push +events, pull requests, issues, and collaborator invites for +repositories they host until upgraded. So upgrade please!!! + +- Upgrade to the latest tag (v1.14.0 or above) +- Head to the [knot dashboard](https://tangled.org/settings/knots) and + hit the "retry" button to verify your knot + ## Upgrading to v1.13.0-alpha Starting with v1.13.0-alpha, every repository on a knot is diff --git a/hook/setup.go b/hook/setup.go --- a/hook/setup.go +++ b/hook/setup.go @@ -5,6 +5,8 @@ import ( "errors" "fmt" + "io/fs" + "log/slog" "os" "path/filepath" "strings" @@ -54,6 +56,9 @@ // did:web:repo1 func Setup(config config) error { // iterate over all directories in current directory: repoDirs, err := os.ReadDir(config.scanPath) + if errors.Is(err, fs.ErrNotExist) { + return os.MkdirAll(config.scanPath, 0755) + } if err != nil { return err } @@ -70,6 +75,10 @@ } userPath := filepath.Join(config.scanPath, did) if err := SetupRepo(config, userPath); err != nil { + if errors.Is(err, ErrNoGitRepo) { + slog.Warn("hook setup: skipping non-repo entry", "path", userPath, "err", err) + continue + } return err } } diff --git a/input.css b/input.css --- a/input.css +++ b/input.css @@ -92,13 +92,15 @@ label { @apply block text-gray-900 text-sm py-2 dark:text-gray-100; } - input:not([type="checkbox"]), + + input, textarea { @apply block rounded p-3 bg-gray-50 dark:bg-gray-800 dark:text-white border border-gray-300 dark:border-gray-600 focus:outline-none focus:ring-1 focus:ring-gray-400 dark:focus:ring-gray-500; } + input[type="checkbox"] { @apply appearance-none size-4 rounded bg-transparent border border-gray-200 @@ -111,8 +113,9 @@ disabled:checked:bg-gray-300 disabled:checked:border-transparent disabled:indeterminate:bg-gray-100 disabled:indeterminate:border-gray-200 dark:border-gray-600 dark:hover:bg-gray-700 - dark:checked:bg-gray-100 dark:checked:border-transparent + dark:checked:bg-gray-100 dark:checked:hover:bg-gray-200 + dark:checked:border-gray-400 dark:indeterminate:bg-gray-100 dark:indeterminate:border-transparent dark:disabled:bg-gray-700 dark:disabled:border-gray-600 dark:disabled:checked:bg-gray-500 dark:disabled:checked:border-transparent @@ -142,6 +145,49 @@ input[type="checkbox"]:disabled:indeterminate { background-image: url("data:image/svg+xml,"); } } + + input[type="radio"] { + @apply appearance-none size-4 p-0 rounded-full bg-transparent border + + border-gray-400 hover:border-gray-900 + hover:bg-gray-200 + + disabled:cursor-not-allowed + disabled:border-gray-300 + disabled:checked:border-gray-300 + disabled:hover:bg-inherit + + dark:border-gray-400 dark:hover:border-gray-100 + + dark:hover:bg-gray-600 + + dark:checked:border-gray-400 + + dark:disabled:border-gray-500 + dark:disabled:checked:border-gray-500 + + focus:outline-none focus:ring-1 + focus:ring-gray-400 dark:focus:ring-gray-500; + + background-position: center; + background-repeat: no-repeat; + background-size: 8px; + } + input[type="radio"]:checked { + background-image: url("data:image/svg+xml,"); + } + input[type="radio"]:disabled:checked { + background-image: url("data:image/svg+xml,"); + } + @media (prefers-color-scheme: dark) { + input[type="radio"]:checked { + background-image: url("data:image/svg+xml,"); + } + input[type="radio"]:disabled:checked { + background-image: url("data:image/svg+xml,"); + } + } + details summary::-webkit-details-marker { display: none; } diff --git a/jobs/src/index.ts b/jobs/src/index.ts --- a/jobs/src/index.ts +++ b/jobs/src/index.ts @@ -9,7 +9,7 @@ // --------------------------------------------------------------------------- // Job postings — add a new markdown file in src/postings/ and import it here // --------------------------------------------------------------------------- -import softwareEngineer from './postings/software-engineer.md'; +// import softwareEngineer from './postings/software-engineer.md'; interface Posting { slug: string; @@ -41,7 +41,7 @@ }; } const POSTINGS: Posting[] = [ - parsePosting('software-engineer', softwareEngineer as string), + // parsePosting('software-engineer', softwareEngineer as string), // add more here as you create markdown files ]; diff --git a/jobs/src/postings/software-engineer.md b/jobs/src/postings/archive/software-engineer.md rename from jobs/src/postings/software-engineer.md rename to jobs/src/postings/archive/software-engineer.md --- a/jobs/src/postings/software-engineer.md +++ b/jobs/src/postings/archive/software-engineer.md diff --git a/knotmirror/db/db.go b/knotmirror/db/db.go --- a/knotmirror/db/db.go +++ b/knotmirror/db/db.go @@ -7,6 +7,7 @@ "fmt" "time" _ "github.com/jackc/pgx/v5/stdlib" + "tangled.org/core/log" ) func Make(ctx context.Context, dbUrl string, maxConns int) (*sql.DB, error) { @@ -94,6 +95,10 @@ execute function set_updated_at(); `) if err != nil { return nil, fmt.Errorf("initializing db schema: %w", err) + } + + if err := RunMigrations(ctx, conn, log.FromContext(ctx), Migrations); err != nil { + return nil, fmt.Errorf("running migrations: %w", err) } return db, nil diff --git a/knotmirror/db/migrations.go b/knotmirror/db/migrations.go new file mode 100644 --- /dev/null +++ b/knotmirror/db/migrations.go @@ -0,0 +1,72 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + "log/slog" +) + +type MigrationFn = func(context.Context, *sql.Tx) error + +type Migration struct { + Name string + Fn MigrationFn +} + +func ensureMigrationsTable(ctx context.Context, conn *sql.Conn) error { + _, err := conn.ExecContext(ctx, ` + create table if not exists migrations ( + name text primary key, + applied_at timestamptz not null default now() + ); + `) + return err +} + +func RunMigration(ctx context.Context, conn *sql.Conn, logger *slog.Logger, m Migration) error { + logger = logger.With("migration", m.Name) + + tx, err := conn.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("begin migration tx: %w", err) + } + defer tx.Rollback() + + var exists bool + if err := tx.QueryRowContext(ctx, `select exists (select 1 from migrations where name = $1)`, m.Name).Scan(&exists); err != nil { + return fmt.Errorf("checking migration state: %w", err) + } + if exists { + logger.Debug("migration already applied") + return nil + } + + if err := m.Fn(ctx, tx); err != nil { + logger.Error("migration failed", "err", err) + return fmt.Errorf("running migration %s: %w", m.Name, err) + } + + if _, err := tx.ExecContext(ctx, `insert into migrations (name) values ($1)`, m.Name); err != nil { + return fmt.Errorf("recording migration: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commit migration: %w", err) + } + + logger.Info("migration applied") + return nil +} + +func RunMigrations(ctx context.Context, conn *sql.Conn, logger *slog.Logger, ms []Migration) error { + if err := ensureMigrationsTable(ctx, conn); err != nil { + return fmt.Errorf("ensuring migrations table: %w", err) + } + for _, m := range ms { + if err := RunMigration(ctx, conn, logger, m); err != nil { + return err + } + } + return nil +} diff --git a/knotmirror/db/migrations_list.go b/knotmirror/db/migrations_list.go new file mode 100644 --- /dev/null +++ b/knotmirror/db/migrations_list.go @@ -0,0 +1,56 @@ +package db + +import ( + "context" + "database/sql" + "fmt" + + "tangled.org/core/log" +) + +var Migrations = []Migration{ + { + Name: "repos_pk_to_repo_did", + Fn: reposPkToRepoDid, + }, +} + +func reposPkToRepoDid(ctx context.Context, tx *sql.Tx) error { + if _, err := tx.ExecContext(ctx, + `alter table repos add column if not exists repo_did text`, + ); err != nil { + return fmt.Errorf("adding repo_did column: %w", err) + } + var bad int + if err := tx.QueryRowContext(ctx, + `select count(*) from repos where repo_did is null or repo_did = ''`, + ).Scan(&bad); err != nil { + return fmt.Errorf("counting rows with null repo_did: %w", err) + } + if bad > 0 { + log.FromContext(ctx).Warn( + "dropping repos with null repo_did; their on-disk dirs will be orphaned. re-crawl via tap to restore", + "count", bad, + ) + if _, err := tx.ExecContext(ctx, + `delete from repos where repo_did is null or repo_did = ''`, + ); err != nil { + return fmt.Errorf("deleting null repo_did rows: %w", err) + } + } + return execAll(ctx, tx, + `alter table repos alter column repo_did set not null`, + `alter table repos drop constraint if exists repos_pkey`, + `alter table repos add constraint repos_pkey primary key (repo_did)`, + ) +} + +func execAll(ctx context.Context, tx *sql.Tx, stmts ...string) error { + if len(stmts) == 0 { + return nil + } + if _, err := tx.ExecContext(ctx, stmts[0]); err != nil { + return err + } + return execAll(ctx, tx, stmts[1:]...) +} diff --git a/knotmirror/db/repos.go b/knotmirror/db/repos.go --- a/knotmirror/db/repos.go +++ b/knotmirror/db/repos.go @@ -11,22 +11,16 @@ "tangled.org/core/appview/pagination" "tangled.org/core/knotmirror/models" ) -func AddRepo(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, cid syntax.CID, name, knot string) error { - if _, err := e.ExecContext(ctx, - `insert into repos (did, rkey, cid, name, knot_domain) - values ($1, $2, $3, $4, $5)`, - did, rkey, cid, name, knot, - ); err != nil { - return fmt.Errorf("inserting repo: %w", err) +func UpsertRepo(ctx context.Context, e *sql.DB, repo *models.Repo) error { + if repo.RepoDid == "" { + return fmt.Errorf("upsert repo: repo_did is required") } - return nil -} - -func UpsertRepo(ctx context.Context, e *sql.DB, repo *models.Repo) error { if _, err := e.ExecContext(ctx, - `insert into repos (did, rkey, cid, name, knot_domain, git_rev, repo_sha, state, error_msg, retry_count, retry_after) - values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) - on conflict(did, rkey) do update set + `insert into repos (did, rkey, cid, name, knot_domain, repo_did, git_rev, repo_sha, state, error_msg, retry_count, retry_after) + values ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12) + on conflict(repo_did) do update set + did = excluded.did, + rkey = excluded.rkey, cid = excluded.cid, name = excluded.name, knot_domain = excluded.knot_domain, @@ -36,12 +30,12 @@ state = excluded.state, error_msg = excluded.error_msg, retry_count = excluded.retry_count, retry_after = excluded.retry_after`, - // where repos.cid != excluded.cid`, repo.Did, repo.Rkey, repo.Cid, repo.Name, repo.KnotDomain, + repo.RepoDid, repo.GitRev, repo.RepoSha, repo.State, @@ -54,13 +48,13 @@ } return nil } -func UpdateRepoState(ctx context.Context, e *sql.DB, did syntax.DID, rkey syntax.RecordKey, state models.RepoState) error { +func UpdateRepoState(ctx context.Context, e *sql.DB, repoDid syntax.DID, state models.RepoState) error { if _, err := e.ExecContext(ctx, `update repos set state = $1 - where did = $2 and rkey = $3`, + where repo_did = $2`, state, - did, rkey, + repoDid, ); err != nil { return fmt.Errorf("updating repo: %w", err) } @@ -78,31 +72,29 @@ } return nil } -func GetRepoByName(ctx context.Context, e *sql.DB, did syntax.DID, name string) (*models.Repo, error) { +const repoColumns = ` + did, + rkey, + cid, + name, + knot_domain, + repo_did, + git_rev, + repo_sha, + state, + error_msg, + retry_count, + retry_after` + +func scanRepo(row interface{ Scan(...any) error }) (*models.Repo, error) { var repo models.Repo - if err := e.QueryRowContext(ctx, - `select - did, - rkey, - cid, - name, - knot_domain, - git_rev, - repo_sha, - state, - error_msg, - retry_count, - retry_after - from repos - where did = $1 and name = $2`, - did, - name, - ).Scan( + if err := row.Scan( &repo.Did, &repo.Rkey, &repo.Cid, &repo.Name, &repo.KnotDomain, + &repo.RepoDid, &repo.GitRev, &repo.RepoSha, &repo.State, @@ -110,51 +102,43 @@ &repo.ErrorMsg, &repo.RetryCount, &repo.RetryAfter, ); err != nil { + return nil, err + } + return &repo, nil +} + +func GetRepoByRepoDid(ctx context.Context, e *sql.DB, repoDid syntax.DID) (*models.Repo, error) { + row := e.QueryRowContext(ctx, + `select`+repoColumns+` + from repos + where repo_did = $1`, + repoDid, + ) + repo, err := scanRepo(row) + if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, fmt.Errorf("querying repo: %w", err) } - return &repo, nil + return repo, nil } func GetRepoByAtUri(ctx context.Context, e *sql.DB, aturi syntax.ATURI) (*models.Repo, error) { - var repo models.Repo - if err := e.QueryRowContext(ctx, - `select - did, - rkey, - cid, - name, - knot_domain, - git_rev, - repo_sha, - state, - error_msg, - retry_count, - retry_after + row := e.QueryRowContext(ctx, + `select`+repoColumns+` from repos where at_uri = $1`, aturi, - ).Scan( - &repo.Did, - &repo.Rkey, - &repo.Cid, - &repo.Name, - &repo.KnotDomain, - &repo.GitRev, - &repo.RepoSha, - &repo.State, - &repo.ErrorMsg, - &repo.RetryCount, - &repo.RetryAfter, - ); err != nil { + ) + repo, err := scanRepo(row) + if err != nil { if errors.Is(err, sql.ErrNoRows) { return nil, nil } return nil, fmt.Errorf("querying repo: %w", err) } - return &repo, nil + return repo, nil } func ListRepos(ctx context.Context, e *sql.DB, page pagination.Page, did, knot, state string) ([]models.Repo, error) { @@ -188,18 +172,7 @@ } } query := ` - select - did, - rkey, - cid, - name, - knot_domain, - git_rev, - repo_sha, - state, - error_msg, - retry_count, - retry_after + select` + repoColumns + ` from repos ` + whereClause + pageClause rows, err := e.QueryContext(ctx, query, args...) @@ -210,23 +183,11 @@ defer rows.Close() var repos []models.Repo for rows.Next() { - var repo models.Repo - if err := rows.Scan( - &repo.Did, - &repo.Rkey, - &repo.Cid, - &repo.Name, - &repo.KnotDomain, - &repo.GitRev, - &repo.RepoSha, - &repo.State, - &repo.ErrorMsg, - &repo.RetryCount, - &repo.RetryAfter, - ); err != nil { + repo, err := scanRepo(rows) + if err != nil { return nil, fmt.Errorf("scanning row: %w", err) } - repos = append(repos, repo) + repos = append(repos, *repo) } if err := rows.Err(); err != nil { return nil, fmt.Errorf("scanning rows: %w ", err) diff --git a/knotmirror/git.go b/knotmirror/git.go --- a/knotmirror/git.go +++ b/knotmirror/git.go @@ -19,8 +19,6 @@ ) type GitMirrorManager interface { Exist(repo *models.Repo) (bool, error) - // RemoteSetUrl updates git repository 'origin' remote - RemoteSetUrl(ctx context.Context, repo *models.Repo) error // Clone clones the repository as a mirror Clone(ctx context.Context, repo *models.Repo) error // Fetch fetches the repository @@ -44,33 +42,16 @@ var _ GitMirrorManager = new(CliGitMirrorManager) func (c *CliGitMirrorManager) makeRepoPath(repo *models.Repo) string { - return filepath.Join(c.repoBasePath, repo.Did.String(), repo.Rkey.String()) + return filepath.Join(c.repoBasePath, repo.RepoDid.String()) } func (c *CliGitMirrorManager) Exist(repo *models.Repo) (bool, error) { return isDir(c.makeRepoPath(repo)) } -func (c *CliGitMirrorManager) RemoteSetUrl(ctx context.Context, repo *models.Repo) error { - path := c.makeRepoPath(repo) - url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL) - if err != nil { - return fmt.Errorf("constructing repo remote url: %w", err) - } - cmd := exec.CommandContext(ctx, "git", "-C", path, "remote", "set-url", "origin", url) - if out, err := cmd.CombinedOutput(); err != nil { - if ctx.Err() != nil { - return ctx.Err() - } - msg := string(out) - return fmt.Errorf("running 'git remote set-url origin %s': %w\n%s", url, err, msg) - } - return nil -} - func (c *CliGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) - url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL) + url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } @@ -94,12 +75,15 @@ } func (c *CliGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) - return c.fetch(ctx, path) + url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) + if err != nil { + return fmt.Errorf("constructing repo remote url: %w", err) + } + return c.fetch(ctx, path, url) } -func (c *CliGitMirrorManager) fetch(ctx context.Context, path string) error { - // TODO: use `repo.Knot` instead of depending on origin - cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", "origin") +func (c *CliGitMirrorManager) fetch(ctx context.Context, path, url string) error { + cmd := exec.CommandContext(ctx, "git", "-C", path, "fetch", "--prune", url, "+refs/*:refs/*") if out, err := cmd.CombinedOutput(); err != nil { if ctx.Err() != nil { return ctx.Err() @@ -111,7 +95,7 @@ } func (c *CliGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) - url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL) + url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } @@ -125,7 +109,7 @@ if err := c.clone(ctx, path, url); err != nil { return fmt.Errorf("cloning repo: %w", err) } } else { - if err := c.fetch(ctx, path); err != nil { + if err := c.fetch(ctx, path, url); err != nil { return fmt.Errorf("fetching repo: %w", err) } } @@ -191,20 +175,16 @@ var _ GitMirrorManager = new(GoGitMirrorManager) func (c *GoGitMirrorManager) makeRepoPath(repo *models.Repo) string { - return filepath.Join(c.repoBasePath, repo.Did.String(), repo.Rkey.String()) + return filepath.Join(c.repoBasePath, repo.RepoDid.String()) } func (c *GoGitMirrorManager) Exist(repo *models.Repo) (bool, error) { return isDir(c.makeRepoPath(repo)) } -func (c *GoGitMirrorManager) RemoteSetUrl(ctx context.Context, repo *models.Repo) error { - panic("unimplemented") -} - func (c *GoGitMirrorManager) Clone(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) - url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL) + url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } @@ -224,7 +204,7 @@ } func (c *GoGitMirrorManager) Fetch(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) - url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL) + url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } @@ -250,7 +230,7 @@ } func (c *GoGitMirrorManager) Sync(ctx context.Context, repo *models.Repo) error { path := c.makeRepoPath(repo) - url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), c.knotUseSSL) + url, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), c.knotUseSSL) if err != nil { return fmt.Errorf("constructing repo remote url: %w", err) } @@ -271,7 +251,7 @@ } return nil } -func makeRepoRemoteUrl(knot, didSlashRepo string, knotUseSSL bool) (string, error) { +func makeRepoRemoteUrl(knot, repoIdentifier string, knotUseSSL bool) (string, error) { if !strings.Contains(knot, "://") { if knotUseSSL { knot = "https://" + knot @@ -289,7 +269,7 @@ if u.Scheme != "http" && u.Scheme != "https" { return "", fmt.Errorf("unsupported scheme: %s", u.Scheme) } - u = u.JoinPath(didSlashRepo) + u = u.JoinPath(repoIdentifier) return u.String(), nil } diff --git a/knotmirror/knotstream/slurper.go b/knotmirror/knotstream/slurper.go --- a/knotmirror/knotstream/slurper.go +++ b/knotmirror/knotstream/slurper.go @@ -15,7 +15,6 @@ "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/util/ssrf" "github.com/carlmjohnson/versioninfo" "github.com/gorilla/websocket" - "tangled.org/core/api/tangled" "tangled.org/core/knotmirror/config" "tangled.org/core/knotmirror/db" "tangled.org/core/knotmirror/models" @@ -262,10 +261,16 @@ }) } } +type legacyGitRefUpdate struct { + OwnerDid *string `json:"ownerDid,omitempty"` + Repo *string `json:"repo,omitempty"` + LegacyRepoDid *string `json:"repoDid,omitempty"` +} + type LegacyGitEvent struct { Rkey string Nsid string - Event tangled.GitRefUpdate + Event legacyGitRefUpdate } func (s *KnotSlurper) ProcessEvent(ctx context.Context, task *Task) error { @@ -280,33 +285,40 @@ } return nil } +// lookupRepoForRefUpdate resolves the local repo row for an incoming refUpdate +// via the stable RepoDid join. Returns (nil, "", nil) when the event has no +// repoDid (unjoinable) and (nil, key, nil) on a clean miss. +func (s *KnotSlurper) lookupRepoForRefUpdate(ctx context.Context, evt *LegacyGitEvent) (*models.Repo, string, error) { + raw := evt.Event.Repo + if raw == nil || *raw == "" { + raw = evt.Event.LegacyRepoDid + } + if raw == nil || *raw == "" { + return nil, "", nil + } + repoDid := syntax.DID(*raw) + curr, err := db.GetRepoByRepoDid(ctx, s.db, repoDid) + return curr, repoDid.String(), err +} + func (s *KnotSlurper) ProcessLegacyGitRefUpdate(ctx context.Context, source string, evt *LegacyGitEvent) error { knotstreamEventsReceived.Inc() l := s.logger.With("src", source) - ownerDid := "" - if evt.Event.OwnerDid != nil { - ownerDid = *evt.Event.OwnerDid - } else { - // handle legacy event - if evt.Event.RepoDid != nil { - ownerDid = *evt.Event.RepoDid - } - } - curr, err := db.GetRepoByName(ctx, s.db, syntax.DID(ownerDid), evt.Event.RepoName) + curr, lookupKey, err := s.lookupRepoForRefUpdate(ctx, evt) if err != nil { - return fmt.Errorf("failed to get repo '%s': %w", ownerDid+"/"+evt.Event.RepoName, err) + return fmt.Errorf("failed to get repo '%s': %w", lookupKey, err) } if curr == nil { - // if repo doesn't exist in DB, just ignore the event. That repo is unknown. - // - // Normally did+name is already enough to perform git-fetch as that's - // what needed to fetch the repository. - // But we want to store that in did/rkey in knot-mirror. - // Therefore, we should ignore when the repository is unknown. - // Hopefully crawler will sync it later. - l.Warn("skipping event from unknown repo", "did/name", ownerDid+"/"+evt.Event.RepoName) + if lookupKey == "" { + l.Warn("skipping gitRefUpdate: event has no fields to join on", + "repo", evt.Event.Repo, "legacy_repo_did", evt.Event.LegacyRepoDid) + } else { + // if repo doesn't exist in DB, just ignore the event. That repo is unknown. + // Hopefully crawler/tap will sync it later. + l.Warn("skipping event from unknown repo", "key", lookupKey) + } knotstreamEventsSkipped.Inc() return nil } @@ -325,13 +337,8 @@ knotstreamEventsSkipped.Inc() return nil } - // if curr.State == models.RepoStateResyncing { - // firehoseEventsSkipped.Inc() - // return fp.events.addToResyncBuffer(ctx, commit) - // } - // can't skip anything, update repo state - if err := db.UpdateRepoState(ctx, s.db, curr.Did, curr.Rkey, models.RepoStateDesynchronized); err != nil { + if err := db.UpdateRepoState(ctx, s.db, curr.RepoDid, models.RepoStateDesynchronized); err != nil { return err } diff --git a/knotmirror/migrate/migrate.go b/knotmirror/migrate/migrate.go new file mode 100644 --- /dev/null +++ b/knotmirror/migrate/migrate.go @@ -0,0 +1,135 @@ +package migrate + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "os" + "path/filepath" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + "tangled.org/core/knotmirror/db" +) + +type Stats struct { + Renamed int + Skipped int + Orphaned int + OwnerDirsRm int + AlreadyExists int +} + +func (s Stats) String() string { + return fmt.Sprintf( + "renamed=%d skipped=%d orphaned=%d owner_dirs_removed=%d target_existed=%d", + s.Renamed, s.Skipped, s.Orphaned, s.OwnerDirsRm, s.AlreadyExists, + ) +} + +func RenameDisk(ctx context.Context, base string, database *sql.DB, logger *slog.Logger) (Stats, error) { + entries, err := os.ReadDir(base) + if err != nil { + return Stats{}, fmt.Errorf("reading base path: %w", err) + } + return reduceEntries(ctx, entries, 0, Stats{}, ownerStep(base, database, logger)) +} + +type stepFn func(context.Context, os.DirEntry, Stats) (Stats, error) + +func reduceEntries(ctx context.Context, entries []os.DirEntry, idx int, acc Stats, fn stepFn) (Stats, error) { + if idx >= len(entries) { + return acc, nil + } + if err := ctx.Err(); err != nil { + return acc, err + } + next, err := fn(ctx, entries[idx], acc) + if err != nil { + return next, err + } + return reduceEntries(ctx, entries, idx+1, next, fn) +} + +func ownerStep(base string, database *sql.DB, logger *slog.Logger) stepFn { + return func(ctx context.Context, entry os.DirEntry, acc Stats) (Stats, error) { + if !entry.IsDir() || !strings.HasPrefix(entry.Name(), "did:") { + return acc, nil + } + ownerPath := filepath.Join(base, entry.Name()) + if _, err := os.Stat(filepath.Join(ownerPath, "HEAD")); err == nil { + return acc, nil + } + subEntries, err := os.ReadDir(ownerPath) + if err != nil { + logger.Error("reading owner dir", "ownerPath", ownerPath, "err", err) + return acc, nil + } + next, err := reduceEntries(ctx, subEntries, 0, acc, rkeyStep(base, database, logger, syntax.DID(entry.Name()), ownerPath)) + if err != nil { + return next, err + } + remaining, err := os.ReadDir(ownerPath) + if err == nil && len(remaining) == 0 { + if rmErr := os.Remove(ownerPath); rmErr == nil { + next.OwnerDirsRm++ + logger.Info("removed empty owner dir", "ownerPath", ownerPath) + } else { + logger.Warn("failed to remove empty owner dir", "ownerPath", ownerPath, "err", rmErr) + } + } + return next, nil + } +} + +func rkeyStep(base string, database *sql.DB, logger *slog.Logger, ownerDid syntax.DID, ownerPath string) stepFn { + return func(ctx context.Context, sub os.DirEntry, acc Stats) (Stats, error) { + if !sub.IsDir() { + return acc, nil + } + rkey := sub.Name() + subPath := filepath.Join(ownerPath, rkey) + l := logger.With("did", ownerDid, "rkey", rkey, "subPath", subPath) + + if _, err := os.Stat(filepath.Join(subPath, "HEAD")); err != nil { + l.Warn("skipping non-repo subdir") + acc.Skipped++ + return acc, nil + } + + aturi := syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", ownerDid, tangled.RepoNSID, rkey)) + repo, err := db.GetRepoByAtUri(ctx, database, aturi) + if err != nil { + return acc, fmt.Errorf("looking up repo by aturi %s: %w", aturi, err) + } + if repo == nil { + l.Warn("orphan disk repo, no DB row; leaving in place") + acc.Orphaned++ + return acc, nil + } + if repo.RepoDid == "" { + l.Warn("DB row has empty repo_did; leaving in place") + acc.Orphaned++ + return acc, nil + } + + target := filepath.Join(base, repo.RepoDid.String()) + if _, err := os.Stat(target); err == nil { + l.Warn("target path already exists; leaving source in place", "target", target) + acc.AlreadyExists++ + return acc, nil + } else if !errors.Is(err, os.ErrNotExist) { + return acc, fmt.Errorf("stat target %s: %w", target, err) + } + + if err := os.Rename(subPath, target); err != nil { + return acc, fmt.Errorf("rename %s -> %s: %w", subPath, target, err) + } + acc.Renamed++ + l.Info("renamed", "target", target) + return acc, nil + } +} diff --git a/knotmirror/models/models.go b/knotmirror/models/models.go --- a/knotmirror/models/models.go +++ b/knotmirror/models/models.go @@ -14,6 +14,7 @@ Cid *syntax.CID // content of tangled.Repo Name string KnotDomain string + RepoDid syntax.DID GitRev syntax.TID // last processed git.refUpdate revision RepoSha string // sha256 sum of git refs (to avoid no-op git fetch) @@ -27,8 +28,8 @@ func (r *Repo) AtUri() syntax.ATURI { return syntax.ATURI(fmt.Sprintf("at://%s/%s/%s", r.Did, tangled.RepoNSID, r.Rkey)) } -func (r *Repo) DidSlashRepo() string { - return fmt.Sprintf("%s/%s", r.Did, r.Name) +func (r *Repo) RepoIdentifier() string { + return r.RepoDid.String() } type RepoState string diff --git a/knotmirror/resyncer.go b/knotmirror/resyncer.go --- a/knotmirror/resyncer.go +++ b/knotmirror/resyncer.go @@ -278,7 +278,7 @@ } // checkKnotReachability checks if Knot is reachable and is valid git remote server func (r *Resyncer) checkKnotReachability(ctx context.Context, repo *models.Repo) error { - repoUrl, err := makeRepoRemoteUrl(repo.KnotDomain, repo.DidSlashRepo(), r.cfg.KnotUseSSL) + repoUrl, err := makeRepoRemoteUrl(repo.KnotDomain, repo.RepoIdentifier(), r.cfg.KnotUseSSL) if err != nil { return err } diff --git a/knotmirror/tapclient.go b/knotmirror/tapclient.go --- a/knotmirror/tapclient.go +++ b/knotmirror/tapclient.go @@ -11,6 +11,7 @@ "net/url" "strings" "time" + "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/api/tangled" "tangled.org/core/knotmirror/config" "tangled.org/core/knotmirror/db" @@ -104,32 +105,23 @@ status = models.RepoStateSuspended errMsg = "suspending non-public knot" } + if record.RepoDid == nil || *record.RepoDid == "" { + t.logger.Warn("dropping repo record without repo_did", "did", evt.Did, "rkey", evt.Rkey) + return nil + } repo := &models.Repo{ Did: evt.Did, Rkey: evt.Rkey, Cid: evt.CID, - Name: record.Name, + Name: evt.Rkey.String(), KnotDomain: knotUrl, + RepoDid: syntax.DID(*record.RepoDid), State: status, ErrorMsg: errMsg, RetryAfter: 0, // clear retry info RetryCount: 0, } - if evt.Action == tapc.RecordUpdateAction { - exist, err := t.gitm.Exist(repo) - if err != nil { - return fmt.Errorf("checking git repo existence: %w", err) - } - if exist { - // update git repo remote url - if err := t.gitm.RemoteSetUrl(ctx, repo); err != nil { - return fmt.Errorf("updating git repo remote url: %w", err) - } - } - } - - t.logger.Debug("tap: upserting repo with knot", "knot", repo.KnotDomain) if err := db.UpsertRepo(ctx, t.db, repo); err != nil { return fmt.Errorf("upserting repo to db: %w", err) } diff --git a/knotmirror/xrpc/git_list_branches.go b/knotmirror/xrpc/git_list_branches.go --- a/knotmirror/xrpc/git_list_branches.go +++ b/knotmirror/xrpc/git_list_branches.go @@ -9,6 +9,7 @@ "strconv" "github.com/bluesky-social/indigo/atproto/atclient" "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/knotmirror/db" "tangled.org/core/knotserver/git" "tangled.org/core/types" ) @@ -82,14 +83,15 @@ }, nil } func (x *Xrpc) makeRepoPath(ctx context.Context, repo syntax.ATURI) (string, error) { - id, err := x.resolver.ResolveIdent(ctx, repo.Authority().String()) + r, err := db.GetRepoByAtUri(ctx, x.db, repo) if err != nil { - return "", err + return "", fmt.Errorf("looking up repo: %w", err) + } + if r == nil { + return "", fmt.Errorf("repo not found: %s", repo) + } + if r.RepoDid == "" { + return "", fmt.Errorf("repo missing repo_did: %s", repo) } - - return filepath.Join( - x.cfg.GitRepoBasePath, - id.DID.String(), - repo.RecordKey().String(), - ), nil + return filepath.Join(x.cfg.GitRepoBasePath, r.RepoDid.String()), nil } diff --git a/knotmirror/xrpc/proxy.go b/knotmirror/xrpc/proxy.go --- a/knotmirror/xrpc/proxy.go +++ b/knotmirror/xrpc/proxy.go @@ -13,6 +13,7 @@ "github.com/bluesky-social/indigo/atproto/syntax" indigoxrpc "github.com/bluesky-social/indigo/xrpc" "tangled.org/core/api/tangled" "tangled.org/core/knotmirror/db" + "tangled.org/core/knotmirror/models" ) var mirrorToKnotNSID = map[string]string{ @@ -40,8 +41,8 @@ "Proxy-Authenticate": true, } type knotInfo struct { - baseURL string - didSlashRepo string + baseURL string + repoIdentifier string } func (x *Xrpc) resolveKnot(ctx context.Context, repoAt syntax.ATURI) (*knotInfo, error) { @@ -60,7 +61,7 @@ knotURL = "http://" + knotURL } } } - return &knotInfo{baseURL: knotURL, didSlashRepo: repo.DidSlashRepo()}, nil + return &knotInfo{baseURL: knotURL, repoIdentifier: repo.RepoIdentifier()}, nil } owner, err := x.resolver.ResolveIdent(ctx, repoAt.Authority().String()) @@ -75,6 +76,9 @@ return nil, fmt.Errorf("fetching repo record from PDS: %w", err) } record := out.Value.Val.(*tangled.Repo) + if record.RepoDid == nil || *record.RepoDid == "" { + return nil, fmt.Errorf("repo record has no repo_did") + } knotURL := record.Knot if !strings.Contains(record.Knot, "://") { if host, _ := db.GetHost(ctx, x.db, record.Knot); host != nil { @@ -89,9 +93,27 @@ } } } + rkey := repoAt.RecordKey().String() + repoDid := syntax.DID(*record.RepoDid) + go func() { + bgCtx := context.Background() + pending := &models.Repo{ + Did: owner.DID, + Rkey: repoAt.RecordKey(), + Cid: (*syntax.CID)(out.Cid), + Name: rkey, + KnotDomain: knotURL, + RepoDid: repoDid, + State: models.RepoStatePending, + } + if upsertErr := db.UpsertRepo(bgCtx, x.db, pending); upsertErr != nil { + x.logger.Error("failed to upsert repo after proxy resolution", "err", upsertErr) + } + }() + return &knotInfo{ - baseURL: knotURL, - didSlashRepo: fmt.Sprintf("%s/%s", owner.DID, record.Name), + baseURL: knotURL, + repoIdentifier: repoDid.String(), }, nil } @@ -112,7 +134,7 @@ params := make(url.Values) for k, v := range r.URL.Query() { params[k] = v } - params.Set("repo", knot.didSlashRepo) + params.Set("repo", knot.repoIdentifier) target := fmt.Sprintf("%s/xrpc/%s?%s", knot.baseURL, knotNSID, params.Encode()) diff --git a/knotmirror/xrpc/sync_request_crawl.go b/knotmirror/xrpc/sync_request_crawl.go --- a/knotmirror/xrpc/sync_request_crawl.go +++ b/knotmirror/xrpc/sync_request_crawl.go @@ -71,12 +71,19 @@ knotUrl = "https://" + knotUrl } } + if record.RepoDid == nil || *record.RepoDid == "" { + l.Warn("dropping repo crawl request without repo_did", "did", owner.DID, "rkey", repoAt.RecordKey()) + writeErr(w, fmt.Errorf("repo record missing repo_did")) + return + } + repo := &models.Repo{ Did: owner.DID, Rkey: repoAt.RecordKey(), Cid: (*syntax.CID)(out.Cid), - Name: record.Name, + Name: repoAt.RecordKey().String(), KnotDomain: knotUrl, + RepoDid: syntax.DID(*record.RepoDid), State: models.RepoStatePending, ErrorMsg: "", RetryAfter: 0, diff --git a/knotserver/db/db.go b/knotserver/db/db.go --- a/knotserver/db/db.go +++ b/knotserver/db/db.go @@ -19,6 +19,11 @@ db *sql.DB logger *slog.Logger } +type Querier interface { + QueryRow(query string, args ...any) *sql.Row + Exec(query string, args ...any) (sql.Result, error) +} + func Setup(ctx context.Context, dbPath string) (*DB, error) { // https://github.com/mattn/go-sqlite3#connection-string opts := []string{ @@ -130,24 +135,89 @@ }); err != nil { return nil, err } + if err := orm.RunMigration(conn, logger, "add-repo-aliases", func(tx *sql.Tx) error { + _, mErr := tx.ExecContext(ctx, ` + create table if not exists repo_aliases ( + owner_did text not null, + rkey text not null, + repo_did text not null, + rev text not null, + primary key (owner_did, rkey) + ); + create index if not exists idx_repo_aliases_repo_did on repo_aliases(repo_did); + + insert or ignore into repo_aliases (owner_did, rkey, repo_did, rev) + select owner_did, repo_name, repo_did, '1_' || created_at + from repo_keys + where owner_did is not null and repo_name is not null and repo_did is not null; + `) + return mErr + }); err != nil { + return nil, err + } + + if err := orm.RunMigration(conn, logger, "drop-at-uri-from-repo-keys", func(tx *sql.Tx) error { + _, mErr := tx.ExecContext(ctx, ` + create table repo_keys_new ( + repo_did text primary key, + signing_key blob, + created_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + owner_did text, + repo_name text, + key_type text not null default 'k256' + ); + insert into repo_keys_new (repo_did, signing_key, created_at, owner_did, repo_name, key_type) + select repo_did, signing_key, created_at, owner_did, repo_name, key_type + from repo_keys; + drop table repo_keys; + alter table repo_keys_new rename to repo_keys; + create unique index if not exists idx_repo_keys_owner_repo + on repo_keys(owner_did, repo_name); + `) + return mErr + }); err != nil { + return nil, err + } + return &DB{ db: db, logger: logger, }, nil } -func (d *DB) StoreRepoKey(repoDid string, signingKey []byte, ownerDid, repoName, atUri string) error { - _, err := d.db.Exec( - `INSERT INTO repo_keys (repo_did, signing_key, owner_did, repo_name, at_uri, key_type) VALUES (?, ?, ?, ?, ?, 'k256')`, - repoDid, signingKey, ownerDid, repoName, atUri, - ) - return err +func (d *DB) StoreRepoKey(repoDid string, signingKey []byte, ownerDid, repoName string) error { + return d.storeRepoKeyRow(repoDid, signingKey, ownerDid, repoName, "k256") } -func (d *DB) StoreRepoDidWeb(repoDid, ownerDid, repoName, atUri string) error { - _, err := d.db.Exec( - `INSERT INTO repo_keys (repo_did, signing_key, owner_did, repo_name, at_uri, key_type) VALUES (?, NULL, ?, ?, ?, 'web')`, - repoDid, ownerDid, repoName, atUri, +func (d *DB) StoreRepoDidWeb(repoDid, ownerDid, repoName string) error { + return d.storeRepoKeyRow(repoDid, nil, ownerDid, repoName, "web") +} + +func (d *DB) storeRepoKeyRow(repoDid string, signingKey []byte, ownerDid, repoName, keyType string) (err error) { + tx, err := d.db.Begin() + if err != nil { + return err + } + defer func() { + if err != nil { + tx.Rollback() + return + } + err = tx.Commit() + }() + + if _, err = tx.Exec( + `INSERT INTO repo_keys (repo_did, signing_key, owner_did, repo_name, key_type) VALUES (?, ?, ?, ?, ?)`, + repoDid, signingKey, ownerDid, repoName, keyType, + ); err != nil { + return err + } + + _, err = tx.Exec( + `INSERT INTO repo_aliases (owner_did, rkey, repo_did, rev) + VALUES (?, ?, ?, '0_' || strftime('%Y-%m-%dT%H:%M:%SZ', 'now')) + ON CONFLICT(owner_did, rkey) DO NOTHING`, + ownerDid, repoName, repoDid, ) return err } @@ -163,30 +233,34 @@ err := d.db.QueryRow(`SELECT count(1) FROM repo_keys WHERE repo_did = ?`, repoDid).Scan(&count) return count > 0, err } -func (d *DB) GetRepoDid(ownerDid, repoName string) (string, error) { +func (d *DB) GetRepoDid(ownerDid, rkey string) (string, error) { var repoDid string err := d.db.QueryRow( - `SELECT repo_did FROM repo_keys WHERE owner_did = ? AND repo_name = ?`, - ownerDid, repoName, + `SELECT repo_did FROM repo_aliases WHERE owner_did = ? AND rkey = ?`, + ownerDid, rkey, ).Scan(&repoDid) return repoDid, err } -func (d *DB) GetRepoKeyOwner(repoDid string) (ownerDid string, repoName string, err error) { - var nullOwner, nullName sql.NullString - err = d.db.QueryRow( - `SELECT owner_did, repo_name FROM repo_keys WHERE repo_did = ?`, +func (d *DB) GetRepoKeyOwner(repoDid string) (string, string, error) { + return GetRepoKeyOwner(d.db, repoDid) +} + +func GetRepoKeyOwner(q Querier, repoDid string) (ownerDid string, repoName string, err error) { + err = q.QueryRow( + `SELECT owner_did, rkey FROM repo_aliases + WHERE repo_did = ? + ORDER BY rev DESC + LIMIT 1`, repoDid, - ).Scan(&nullOwner, &nullName) + ).Scan(&ownerDid, &repoName) if err != nil { return } - if !nullOwner.Valid || !nullName.Valid || nullOwner.String == "" || nullName.String == "" { - err = fmt.Errorf("repo_keys row for %s has empty or null owner_did or repo_name", repoDid) + if ownerDid == "" || repoName == "" { + err = fmt.Errorf("repo_aliases row for %s has empty owner_did or rkey", repoDid) return } - ownerDid = nullOwner.String - repoName = nullName.String return } diff --git a/knotserver/db/didassign.go b/knotserver/db/didassign.go --- a/knotserver/db/didassign.go +++ b/knotserver/db/didassign.go @@ -3,8 +3,7 @@ const RepoDIDAssignNSID = "sh.tangled.repo.didAssign" type RepoDIDAssign struct { - OwnerDid string `json:"ownerDid"` - RepoName string `json:"repoName"` - RepoDid string `json:"repoDid"` - OldRepoAt string `json:"oldRepoAt,omitempty"` + OwnerDid string `json:"ownerDid"` + RepoName string `json:"repoName"` + RepoDid string `json:"repoDid"` } diff --git a/knotserver/db/events.go b/knotserver/db/events.go --- a/knotserver/db/events.go +++ b/knotserver/db/events.go @@ -31,12 +31,11 @@ return err } -func (d *DB) EmitDIDAssign(n *notifier.Notifier, ownerDid, repoName, repoDid, oldRepoAt string) error { +func (d *DB) EmitDIDAssign(n *notifier.Notifier, ownerDid, repoName, repoDid string) error { payload := RepoDIDAssign{ - OwnerDid: ownerDid, - RepoName: repoName, - RepoDid: repoDid, - OldRepoAt: oldRepoAt, + OwnerDid: ownerDid, + RepoName: repoName, + RepoDid: repoDid, } eventJson, err := json.Marshal(payload) diff --git a/knotserver/db/repo_aliases.go b/knotserver/db/repo_aliases.go new file mode 100644 --- /dev/null +++ b/knotserver/db/repo_aliases.go @@ -0,0 +1,60 @@ +package db + +import ( + "database/sql" + "errors" +) + +type RepoAlias struct { + OwnerDid string + Rkey string + RepoDid string + Rev string +} + +func (d *DB) UpsertRepoAlias(a RepoAlias) error { + _, err := d.db.Exec( + `insert into repo_aliases (owner_did, rkey, repo_did, rev) + values (?, ?, ?, ?) + on conflict(owner_did, rkey) do update set + repo_did = excluded.repo_did, + rev = excluded.rev + where excluded.rev > repo_aliases.rev`, + a.OwnerDid, a.Rkey, a.RepoDid, a.Rev, + ) + return err +} + +func (d *DB) DeleteRepoAlias(ownerDid, rkey string) error { + _, err := d.db.Exec( + `delete from repo_aliases where owner_did = ? and rkey = ?`, + ownerDid, rkey, + ) + return err +} + +func (d *DB) ResolveAlias(ownerDid, rkey string) (*RepoAlias, error) { + var a RepoAlias + err := d.db.QueryRow( + `select owner_did, rkey, repo_did, rev from repo_aliases where owner_did = ? and rkey = ?`, + ownerDid, rkey, + ).Scan(&a.OwnerDid, &a.Rkey, &a.RepoDid, &a.Rev) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + return &a, nil +} + +func (d *DB) CurrentRkey(repoDid string) (ownerDid string, rkey string, err error) { + err = d.db.QueryRow( + `select owner_did, rkey from repo_aliases + where repo_did = ? + order by rev desc + limit 1`, + repoDid, + ).Scan(&ownerDid, &rkey) + return +} diff --git a/knotserver/git.go b/knotserver/git.go --- a/knotserver/git.go +++ b/knotserver/git.go @@ -26,9 +26,9 @@ } return repoPath, repoName, nil } - repoDid, err := h.db.GetRepoDid(did, name) - if err == nil { - repoPath, _, _, resolveErr := h.db.ResolveRepoDIDOnDisk(h.c.Repo.ScanPath, repoDid) + alias, err := h.db.ResolveAlias(did, name) + if err == nil && alias != nil { + repoPath, _, _, resolveErr := h.db.ResolveRepoDIDOnDisk(h.c.Repo.ScanPath, alias.RepoDid) if resolveErr == nil { return repoPath, name, nil } @@ -44,10 +44,16 @@ } return repoPath, name, nil } +func (h *Knot) repoNotFound(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") + w.WriteHeader(http.StatusNotFound) + fmt.Fprint(w, "repository not found\n") +} + func (h *Knot) InfoRefs(w http.ResponseWriter, r *http.Request) { repoPath, name, err := h.resolveRepoPath(r) if err != nil { - gitError(w, "repository not found", http.StatusNotFound) + h.repoNotFound(w, r) h.l.Error("git: failed to resolve repo path", "handler", "InfoRefs", "error", err) return } @@ -81,7 +87,7 @@ func (h *Knot) UploadArchive(w http.ResponseWriter, r *http.Request) { repo, _, err := h.resolveRepoPath(r) if err != nil { - gitError(w, "repository not found", http.StatusNotFound) + h.repoNotFound(w, r) h.l.Error("git: failed to resolve repo path", "handler", "UploadArchive", "error", err) return } @@ -126,7 +132,7 @@ func (h *Knot) UploadPack(w http.ResponseWriter, r *http.Request) { repo, _, err := h.resolveRepoPath(r) if err != nil { - gitError(w, "repository not found", http.StatusNotFound) + h.repoNotFound(w, r) h.l.Error("git: failed to resolve repo path", "handler", "UploadPack", "error", err) return } @@ -173,7 +179,7 @@ func (h *Knot) ReceivePack(w http.ResponseWriter, r *http.Request) { _, name, err := h.resolveRepoPath(r) if err != nil { - gitError(w, "repository not found", http.StatusNotFound) + h.repoNotFound(w, r) h.l.Error("git: failed to resolve repo path", "handler", "ReceivePack", "error", err) return } diff --git a/knotserver/git/language.go b/knotserver/git/language.go --- a/knotserver/git/language.go +++ b/knotserver/git/language.go @@ -2,6 +2,7 @@ package git import ( "context" + "io" "path" "strings" @@ -11,15 +12,36 @@ ) type LangBreakdown map[string]int64 +const ( + langContentLimit = 16 * 1024 // read up to 16 KB for language detection + langSizeLimit = 1 * 1024 * 1024 // skip content read for blobs over 1 MB +) + func (g *GitRepo) AnalyzeLanguages(ctx context.Context) (LangBreakdown, error) { sizes := make(map[string]int64) err := g.Walk(ctx, "", func(node object.TreeEntry, parent *object.Tree, root string) error { filepath := path.Join(root, node.Name) - content, err := g.FileContentN(filepath, 16*1024) // 16KB + if enry.IsVendor(filepath) || enry.IsDocumentation(filepath) || + enry.IsDotFile(filepath) || enry.IsConfiguration(filepath) { + return nil + } + + blob, err := object.GetBlob(g.r.Storer, node.Hash) if err != nil { return nil } + sz := blob.Size + + var content []byte + if sz <= langSizeLimit { + r, err := blob.Reader() + if err != nil { + return nil + } + content, _ = io.ReadAll(io.LimitReader(r, langContentLimit)) + r.Close() + } if enry.IsGenerated(filepath, content) || enry.IsBinary(content) || @@ -37,7 +59,6 @@ if langType != enry.Programming && langType != enry.Markup { return nil } - sz, _ := parent.Size(node.Name) sizes[language] += sz return nil diff --git a/knotserver/git/tree.go b/knotserver/git/tree.go --- a/knotserver/git/tree.go +++ b/knotserver/git/tree.go @@ -58,7 +58,7 @@ dir: parent, entries: entries, } - times, err := g.lastCommitDirIn(ctx, lastCommitDir, 2*time.Second) + times, err := g.lastCommitDirIn(ctx, lastCommitDir, 500*time.Millisecond) if err != nil { return nts } diff --git a/knotserver/git_test.go b/knotserver/git_test.go new file mode 100644 --- /dev/null +++ b/knotserver/git_test.go @@ -0,0 +1,65 @@ +package knotserver + +import ( + "context" + "path/filepath" + "testing" + + "tangled.org/core/knotserver/db" +) + +func newTestKnotDB(t *testing.T) *db.DB { + t.Helper() + path := filepath.Join(t.TempDir(), "test.db") + d, err := db.Setup(context.Background(), path) + if err != nil { + t.Fatalf("db.Setup: %v", err) + } + return d +} + +func TestAliasResolvesOriginalName(t *testing.T) { + d := newTestKnotDB(t) + if err := d.StoreRepoKey("did:plc:repo1", []byte("dummy"), "did:plc:akshay", "foo"); err != nil { + t.Fatalf("StoreRepoKey: %v", err) + } + got, err := d.GetRepoDid("did:plc:akshay", "foo") + if err != nil { + t.Fatalf("GetRepoDid: %v", err) + } + if got != "did:plc:repo1" { + t.Errorf("repoDid = %q, want did:plc:repo1", got) + } +} + +func TestAliasUpsertRespectsRevOrdering(t *testing.T) { + d := newTestKnotDB(t) + if err := d.StoreRepoKey("did:plc:repo1", []byte("dummy"), "did:plc:akshay", "foo"); err != nil { + t.Fatalf("StoreRepoKey: %v", err) + } + if err := d.UpsertRepoAlias(db.RepoAlias{ + OwnerDid: "did:plc:akshay", + Rkey: "bar", + RepoDid: "did:plc:repo1", + Rev: "3laaaaaaaaaab", + }); err != nil { + t.Fatalf("UpsertRepoAlias bar: %v", err) + } + + _, current, err := d.CurrentRkey("did:plc:repo1") + if err != nil { + t.Fatalf("CurrentRkey: %v", err) + } + if current != "bar" { + t.Errorf("current rkey = %q, want bar", current) + } + + fooDid, err := d.GetRepoDid("did:plc:akshay", "foo") + if err != nil || fooDid != "did:plc:repo1" { + t.Errorf("old rkey lookup: got (%q, %v), want did:plc:repo1", fooDid, err) + } + barDid, err := d.GetRepoDid("did:plc:akshay", "bar") + if err != nil || barDid != "did:plc:repo1" { + t.Errorf("new rkey lookup: got (%q, %v), want did:plc:repo1", barDid, err) + } +} diff --git a/knotserver/ingester.go b/knotserver/ingester.go --- a/knotserver/ingester.go +++ b/knotserver/ingester.go @@ -18,6 +18,7 @@ "tangled.org/core/api/tangled" "tangled.org/core/appview/models" "tangled.org/core/knotserver/db" "tangled.org/core/knotserver/git" + knotxrpc "tangled.org/core/knotserver/xrpc" "tangled.org/core/log" "tangled.org/core/rbac" "tangled.org/core/workflow" @@ -99,27 +100,31 @@ if record.Target == nil { return nil, fmt.Errorf("ignoring pull record: target repo is nil") } + l := log.FromContext(ctx).With("handler", "validatePullRecord") + l = l.With("target_repo", record.Target.Repo) + l = l.With("target_branch", record.Target.Branch) + if record.Source == nil { return nil, fmt.Errorf("ignoring pull record: not a branch-based pull request") } - if record.Source.Repo != nil || record.Source.RepoDid != nil { + if record.Source.Repo != nil { return nil, fmt.Errorf("ignoring pull record: fork based pull") } var repoPath, ownerDid, repoName, repoDid string switch { - case record.Target.RepoDid != nil && *record.Target.RepoDid != "": - repoDid = *record.Target.RepoDid + case strings.HasPrefix(record.Target.Repo, "did:"): + repoDid = record.Target.Repo var lookupErr error repoPath, ownerDid, repoName, lookupErr = h.db.ResolveRepoDIDOnDisk(h.c.Repo.ScanPath, repoDid) if lookupErr != nil { return nil, fmt.Errorf("unknown target repo DID %s: %w", repoDid, lookupErr) } - case record.Target.Repo != nil: + case strings.Contains(record.Target.Repo, "/"): // TODO: get rid of this PDS fetch once all repos have DIDs - repoAt, parseErr := syntax.ParseATURI(*record.Target.Repo) + repoAt, parseErr := syntax.ParseATURI(record.Target.Repo) if parseErr != nil { return nil, fmt.Errorf("failed to parse ATURI: %w", parseErr) } @@ -138,14 +143,17 @@ if getErr != nil { return nil, fmt.Errorf("failed to resolve repo: %w", getErr) } - repo := resp.Value.Val.(*tangled.Repo) + repo, ok := resp.Value.Val.(*tangled.Repo) + if !ok { + return nil, fmt.Errorf("record at %s is not a tangled.Repo", repoAt) + } if repo.Knot != h.c.Server.Hostname { return nil, fmt.Errorf("rejected pull record: not this knot, %s != %s", repo.Knot, h.c.Server.Hostname) } ownerDid = ident.DID.String() - repoName = repo.Name + repoName = repoAt.RecordKey().String() repoDid, didErr := h.db.GetRepoDid(ownerDid, repoName) if didErr != nil { @@ -159,7 +167,7 @@ return nil, fmt.Errorf("failed to resolve repo on disk: %w", lookupErr) } default: - return nil, fmt.Errorf("ignoring pull record: target has neither repo nor repoDid") + return nil, fmt.Errorf("ignoring pull record: target repo has unrecognized format: %s", record.Target.Repo) } gr, err := git.Open(repoPath, record.Source.Branch) @@ -374,19 +382,19 @@ } var rbacResource string switch { - case record.RepoDid != nil && *record.RepoDid != "": - ownerDid, _, lookupErr := h.db.GetRepoKeyOwner(*record.RepoDid) + case strings.HasPrefix(record.Repo, "did:"): + ownerDid, _, lookupErr := h.db.GetRepoKeyOwner(record.Repo) if lookupErr != nil { - return fmt.Errorf("unknown repo DID %s: %w", *record.RepoDid, lookupErr) + return fmt.Errorf("unknown repo DID %s: %w", record.Repo, lookupErr) } if ownerDid != did { - return fmt.Errorf("collaborator record author %s does not own repo %s", did, *record.RepoDid) + return fmt.Errorf("collaborator record author %s does not own repo %s", did, record.Repo) } - rbacResource = *record.RepoDid + rbacResource = record.Repo - case record.Repo != nil: + case strings.Contains(record.Repo, "/"): // TODO: get rid of this PDS fetch once all repos have DIDs - repoAt, parseErr := syntax.ParseATURI(*record.Repo) + repoAt, parseErr := syntax.ParseATURI(record.Repo) if parseErr != nil { return parseErr } @@ -405,15 +413,18 @@ if getErr != nil { return getErr } - repo := resp.Value.Val.(*tangled.Repo) - repoDid, didErr := h.db.GetRepoDid(owner.DID.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + return fmt.Errorf("record at %s is not a tangled.Repo", repoAt) + } + rkey := repoAt.RecordKey().String() + repoDid, didErr := h.db.GetRepoDid(owner.DID.String(), rkey) if didErr != nil { - return fmt.Errorf("failed to resolve repo DID for %s/%s: %w", owner.DID.String(), repo.Name, didErr) + return fmt.Errorf("failed to resolve repo DID for %s/%s: %w", owner.DID.String(), rkey, didErr) } rbacResource = repoDid default: - return fmt.Errorf("collaborator record has neither repo nor repoDid") + return fmt.Errorf("collaborator record has unrecognized repo format: %s", record.Repo) } ok, err := h.e.IsCollaboratorInviteAllowed(did, rbac.ThisServer, rbacResource) @@ -479,6 +490,70 @@ } return nil } +func (h *Knot) processRepo(ctx context.Context, event *jmodels.Event) error { + l := log.FromContext(ctx).With("handler", "processRepo", "did", event.Did, "rkey", event.Commit.RKey) + + rkey := strings.TrimSuffix(strings.TrimSpace(event.Commit.RKey), ".git") + if rkey == "" { + return nil + } + + if event.Commit.Operation == jmodels.CommitOperationDelete { + if err := h.db.DeleteRepoAlias(event.Did, rkey); err != nil { + l.Warn("failed to delete repo alias", "err", err) + } + return nil + } + + if event.Commit.Operation != jmodels.CommitOperationCreate && event.Commit.Operation != jmodels.CommitOperationUpdate { + return nil + } + + raw := json.RawMessage(event.Commit.Record) + var record tangled.Repo + if err := json.Unmarshal(raw, &record); err != nil { + return fmt.Errorf("failed to unmarshal repo record: %w", err) + } + + if record.Knot != h.c.Server.Hostname { + return nil + } + if record.RepoDid == nil || *record.RepoDid == "" { + l.Info("skipping repo event without repoDid") + return nil + } + repoDid := *record.RepoDid + + if err := knotxrpc.ValidateRepoName(rkey); err != nil { + l.Warn("skipping repo event with invalid rkey", "repoDid", repoDid, "rkey", rkey, "err", err) + return nil + } + + ownerDid, _, lookupErr := h.db.GetRepoKeyOwner(repoDid) + if lookupErr != nil { + l.Info("skipping repo event for unknown repoDid", "repoDid", repoDid) + return nil + } + if ownerDid != event.Did { + l.Warn("repo event author does not own repoDid", "repoDid", repoDid, "author", event.Did) + return nil + } + + alias := db.RepoAlias{ + OwnerDid: event.Did, + Rkey: rkey, + RepoDid: repoDid, + Rev: event.Commit.Rev, + } + if err := h.db.UpsertRepoAlias(alias); err != nil { + l.Warn("failed to upsert repo alias", "err", err) + return nil + } + + l.Info("recorded repo alias", "repoDid", repoDid, "rkey", rkey, "rev", event.Commit.Rev) + return nil +} + func (h *Knot) processMessages(ctx context.Context, event *jmodels.Event) error { var err error switch event.Kind { @@ -490,6 +565,8 @@ case tangled.PublicKeyNSID: err = h.processPublicKey(ctx, event) case tangled.KnotMemberNSID: err = h.processKnotMember(ctx, event) + case tangled.RepoNSID: + err = h.processRepo(ctx, event) case tangled.RepoPullNSID: err = h.processPull(ctx, event) case tangled.RepoCollaboratorNSID: diff --git a/knotserver/ingester_repo_test.go b/knotserver/ingester_repo_test.go new file mode 100644 --- /dev/null +++ b/knotserver/ingester_repo_test.go @@ -0,0 +1,223 @@ +package knotserver + +import ( + "context" + "encoding/json" + "log/slog" + "sync" + "testing" + + jsmodels "github.com/bluesky-social/jetstream/pkg/models" + "tangled.org/core/api/tangled" + "tangled.org/core/knotserver/config" + "tangled.org/core/knotserver/db" + "tangled.org/core/log" +) + +type logRecord struct { + Level slog.Level + Msg string + Attrs map[string]any +} + +type capturingHandler struct { + mu *sync.Mutex + records *[]logRecord + attrs []slog.Attr +} + +func newCapturingHandler() *capturingHandler { + return &capturingHandler{ + mu: &sync.Mutex{}, + records: &[]logRecord{}, + } +} + +func (h *capturingHandler) Enabled(_ context.Context, _ slog.Level) bool { return true } + +func (h *capturingHandler) Handle(_ context.Context, r slog.Record) error { + rec := logRecord{Level: r.Level, Msg: r.Message, Attrs: map[string]any{}} + for _, a := range h.attrs { + rec.Attrs[a.Key] = a.Value.Any() + } + r.Attrs(func(a slog.Attr) bool { + rec.Attrs[a.Key] = a.Value.Any() + return true + }) + h.mu.Lock() + *h.records = append(*h.records, rec) + h.mu.Unlock() + return nil +} + +func (h *capturingHandler) WithAttrs(attrs []slog.Attr) slog.Handler { + merged := make([]slog.Attr, 0, len(h.attrs)+len(attrs)) + merged = append(merged, h.attrs...) + merged = append(merged, attrs...) + return &capturingHandler{mu: h.mu, records: h.records, attrs: merged} +} + +func (h *capturingHandler) WithGroup(string) slog.Handler { + panic("capturingHandler: WithGroup not supported") +} + +func (h *capturingHandler) snapshot() []logRecord { + h.mu.Lock() + defer h.mu.Unlock() + out := make([]logRecord, len(*h.records)) + copy(out, *h.records) + return out +} + +func newProcessRepoFixture(t *testing.T) (*Knot, context.Context, *capturingHandler) { + t.Helper() + d := newTestKnotDB(t) + cap := newCapturingHandler() + l := slog.New(cap) + ctx := log.IntoContext(context.Background(), l) + + c := &config.Config{ + Server: config.Server{Hostname: "knot.example"}, + } + return &Knot{ + c: c, + db: d, + l: l, + }, ctx, cap +} + +func repoEvent(t *testing.T, authorDid, rkey, rev string, record tangled.Repo, op string) *jsmodels.Event { + t.Helper() + raw, err := json.Marshal(record) + if err != nil { + t.Fatalf("marshal record: %v", err) + } + return &jsmodels.Event{ + Did: authorDid, + Kind: jsmodels.EventKindCommit, + Commit: &jsmodels.Commit{ + Operation: op, + Collection: tangled.RepoNSID, + RKey: rkey, + Rev: rev, + Record: raw, + }, + } +} + +func ptr(s string) *string { return &s } + +func TestProcessRepo_CreateRegistersAlias(t *testing.T) { + h, ctx, _ := newProcessRepoFixture(t) + if err := h.db.StoreRepoKey("did:plc:repo1", []byte("k"), "did:plc:akshay", "foo"); err != nil { + t.Fatalf("StoreRepoKey: %v", err) + } + + ev := repoEvent(t, "did:plc:akshay", "bar", "3laaaaaaaaaab", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:repo1"), + }, jsmodels.CommitOperationCreate) + if err := h.processRepo(ctx, ev); err != nil { + t.Fatalf("processRepo: %v", err) + } + + _, current, err := h.db.CurrentRkey("did:plc:repo1") + if err != nil { + t.Fatalf("CurrentRkey: %v", err) + } + if current != "bar" { + t.Errorf("current rkey = %q, want bar (highest rev alias)", current) + } + + oldDid, err := h.db.GetRepoDid("did:plc:akshay", "foo") + if err != nil || oldDid != "did:plc:repo1" { + t.Errorf("old rkey foo should still resolve: got (%q, %v)", oldDid, err) + } +} + +func TestProcessRepo_DeleteRemovesAlias(t *testing.T) { + h, ctx, _ := newProcessRepoFixture(t) + if err := h.db.StoreRepoKey("did:plc:repo1", []byte("k"), "did:plc:akshay", "foo"); err != nil { + t.Fatalf("StoreRepoKey: %v", err) + } + if err := h.db.UpsertRepoAlias(db.RepoAlias{ + OwnerDid: "did:plc:akshay", Rkey: "bar", RepoDid: "did:plc:repo1", Rev: "3laaaaaaaaaab", + }); err != nil { + t.Fatalf("UpsertRepoAlias: %v", err) + } + + ev := repoEvent(t, "did:plc:akshay", "bar", "3laaaaaaaaaac", tangled.Repo{}, jsmodels.CommitOperationDelete) + if err := h.processRepo(ctx, ev); err != nil { + t.Fatalf("processRepo: %v", err) + } + + if _, err := h.db.GetRepoDid("did:plc:akshay", "bar"); err == nil { + t.Errorf("bar alias should have been deleted") + } + + _, current, _ := h.db.CurrentRkey("did:plc:repo1") + if current != "foo" { + t.Errorf("current rkey after delete = %q, want foo", current) + } +} + +func TestProcessRepo_MalformedJSONReturnsError(t *testing.T) { + h, ctx, _ := newProcessRepoFixture(t) + + ev := &jsmodels.Event{ + Did: "did:plc:akshay", + Kind: jsmodels.EventKindCommit, + Commit: &jsmodels.Commit{ + Operation: jsmodels.CommitOperationCreate, + Collection: tangled.RepoNSID, + RKey: "rkey1", + Record: []byte("{not valid json"), + }, + } + if err := h.processRepo(ctx, ev); err == nil { + t.Fatalf("processRepo returned nil, want unmarshal error") + } +} + +func TestProcessRepo_NotOwnedRejected(t *testing.T) { + h, ctx, _ := newProcessRepoFixture(t) + if err := h.db.StoreRepoKey("did:plc:repo1", []byte("k"), "did:plc:akshay", "foo"); err != nil { + t.Fatalf("StoreRepoKey: %v", err) + } + + ev := repoEvent(t, "did:plc:mallory", "pwned", "3laaaaaaaaaab", tangled.Repo{ + Knot: "knot.example", + RepoDid: ptr("did:plc:repo1"), + }, jsmodels.CommitOperationCreate) + if err := h.processRepo(ctx, ev); err != nil { + t.Fatalf("processRepo: %v", err) + } + + _, current, _ := h.db.CurrentRkey("did:plc:repo1") + if current != "foo" { + t.Errorf("current rkey = %q, want foo (mallory's event must be rejected)", current) + } + if _, err := h.db.GetRepoDid("did:plc:mallory", "pwned"); err == nil { + t.Errorf("mallory should not be able to register an alias on alice's repo") + } +} + +func TestProcessRepo_WrongKnotIgnored(t *testing.T) { + h, ctx, _ := newProcessRepoFixture(t) + if err := h.db.StoreRepoKey("did:plc:repo1", []byte("k"), "did:plc:akshay", "foo"); err != nil { + t.Fatalf("StoreRepoKey: %v", err) + } + + ev := repoEvent(t, "did:plc:akshay", "bar", "3laaaaaaaaaab", tangled.Repo{ + Knot: "other.example", + RepoDid: ptr("did:plc:repo1"), + }, jsmodels.CommitOperationCreate) + if err := h.processRepo(ctx, ev); err != nil { + t.Fatalf("processRepo: %v", err) + } + + _, current, _ := h.db.CurrentRkey("did:plc:repo1") + if current != "foo" { + t.Errorf("current rkey = %q, want foo (foreign-knot event must be ignored)", current) + } +} diff --git a/knotserver/internal.go b/knotserver/internal.go --- a/knotserver/internal.go +++ b/knotserver/internal.go @@ -140,14 +140,16 @@ rbacResource = repoDid } else { legacyPath, joinErr := securejoin.SecureJoin(h.c.Repo.ScanPath, filepath.Join(ownerDid.String(), repoName)) if joinErr != nil { + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") w.WriteHeader(http.StatusNotFound) - fmt.Fprintln(w, "repo not found") + fmt.Fprint(w, "repo not found\n") return } if _, statErr := os.Stat(legacyPath); statErr != nil { + l.Info("legacy repo path missing, checking rename history", "owner", ownerDid, "name", repoName) + w.Header().Set("Content-Type", "text/plain; charset=UTF-8") w.WriteHeader(http.StatusNotFound) - l.Error("repo not found on disk (legacy)", "owner", ownerDid, "name", repoName) - fmt.Fprintln(w, "repo not found") + fmt.Fprint(w, "repo not found\n") return } repoPath = legacyPath @@ -253,7 +255,7 @@ Messages: make([]string, 0), } for _, line := range lines { - err := h.insertRefUpdate(line, gitUserDid, ownerDid, repoName, repoDid) + err := h.insertRefUpdate(line, gitUserDid, ownerDid, repoDid) if err != nil { l.Error("failed to insert op", "err", err, "line", line, "did", gitUserDid, "repo", gitRelativeDir) } @@ -272,7 +274,7 @@ writeJSON(w, resp) } -func (h *InternalHandle) insertRefUpdate(line git.PostReceiveLine, gitUserDid, ownerDid, repoName, repoDid string) error { +func (h *InternalHandle) insertRefUpdate(line git.PostReceiveLine, gitUserDid, ownerDid, repoDid string) error { repoPath, _, _, resolveErr := h.db.ResolveRepoDIDOnDisk(h.c.Repo.ScanPath, repoDid) if resolveErr != nil { return fmt.Errorf("failed to resolve repo on disk: %w", resolveErr) @@ -296,8 +298,7 @@ NewSha: line.NewSha.String(), Ref: line.Ref, CommitterDid: gitUserDid, OwnerDid: &ownerDid, - RepoName: repoName, - RepoDid: &repoDid, + Repo: repoDid, Meta: &metaRecord, } diff --git a/knotserver/migrate.go b/knotserver/migrate.go --- a/knotserver/migrate.go +++ b/knotserver/migrate.go @@ -145,7 +145,7 @@ if err := os.Remove(ownerDir); err != nil && !errors.Is(err, os.ErrNotExist) { l.Warn("could not remove empty owner dir", "path", ownerDir, "error", err) } - if err := d.EmitDIDAssign(n, repo.ownerDid, repo.repoName, repoDid, ""); err != nil { + if err := d.EmitDIDAssign(n, repo.ownerDid, repo.repoName, repoDid); err != nil { l.Error("emitting didAssign event failed (non-fatal)", "error", err) } @@ -170,7 +170,7 @@ if err := submitWithBackoff(ctx, prepared, l); err != nil { return "", fmt.Errorf("PLC submission: %w", err) } - if err := d.StoreRepoKey(prepared.RepoDid, prepared.SigningKeyRaw, repo.ownerDid, repo.repoName, ""); err != nil { + if err := d.StoreRepoKey(prepared.RepoDid, prepared.SigningKeyRaw, repo.ownerDid, repo.repoName); err != nil { return "", fmt.Errorf("storing repo key: %w", err) } diff --git a/knotserver/server.go b/knotserver/server.go --- a/knotserver/server.go +++ b/knotserver/server.go @@ -81,6 +81,7 @@ jc, err := jetstream.NewJetstreamClient(c.Server.JetstreamEndpoint, "knotserver", []string{ tangled.PublicKeyNSID, tangled.KnotMemberNSID, + tangled.RepoNSID, tangled.RepoPullNSID, tangled.RepoCollaboratorNSID, }, nil, log.SubLogger(logger, "jetstream"), db, true, c.Server.LogDids) diff --git a/knotserver/xrpc/create_repo.go b/knotserver/xrpc/create_repo.go --- a/knotserver/xrpc/create_repo.go +++ b/knotserver/xrpc/create_repo.go @@ -63,7 +63,7 @@ if data.DefaultBranch != nil && *data.DefaultBranch != "" { defaultBranch = *data.DefaultBranch } - if err := validateRepoName(repoName); err != nil { + if err := ValidateRepoName(repoName); err != nil { l.Error("creating repo", "error", err.Error()) fail(xrpcerr.GenericError(err)) return @@ -129,8 +129,7 @@ return } repoDid = prepared.RepoDid - atUri := fmt.Sprintf("at://%s/%s/%s", actorDid, tangled.RepoNSID, data.Rkey) - if err := h.Db.StoreRepoKey(repoDid, prepared.SigningKeyRaw, actorDid.String(), repoName, atUri); err != nil { + if err := h.Db.StoreRepoKey(repoDid, prepared.SigningKeyRaw, actorDid.String(), repoName); err != nil { if strings.Contains(err.Error(), "UNIQUE constraint failed") { writeError(w, xrpcerr.GenericError(fmt.Errorf("repository %s already being created", repoName)), http.StatusConflict) return @@ -188,8 +187,7 @@ } } if data.RepoDid != nil && strings.HasPrefix(*data.RepoDid, "did:web:") { - webAtUri := fmt.Sprintf("at://%s/%s/%s", actorDid, tangled.RepoNSID, data.Rkey) - if err := h.Db.StoreRepoDidWeb(repoDid, actorDid.String(), repoName, webAtUri); err != nil { + if err := h.Db.StoreRepoDidWeb(repoDid, actorDid.String(), repoName); err != nil { cleanupAll() if strings.Contains(err.Error(), "UNIQUE constraint failed") { writeError(w, xrpcerr.GenericError(fmt.Errorf("did:web %s is already in use", repoDid)), http.StatusConflict) @@ -266,7 +264,11 @@ } return nil } -func validateRepoName(name string) error { +var reservedRepoNames = map[string]struct{}{ + "self": {}, +} + +func ValidateRepoName(name string) error { // check for path traversal attempts if name == "." || name == ".." || strings.Contains(name, "/") || strings.Contains(name, "\\") { @@ -279,6 +281,13 @@ strings.HasPrefix(name, ".") || strings.HasSuffix(name, ".") { return fmt.Errorf("Repository name contains invalid path sequence") } + if len(name) == 0 { + return fmt.Errorf("Repository name cannot be empty") + } + if len(name) > 100 { + return fmt.Errorf("Repository name must be 100 characters or fewer") + } + // then continue with character validation for _, char := range name { if !((char >= 'a' && char <= 'z') || @@ -292,6 +301,10 @@ // additional check to prevent multiple sequential dots if strings.Contains(name, "..") { return fmt.Errorf("Repository name cannot contain sequential dots") + } + + if _, reserved := reservedRepoNames[strings.ToLower(name)]; reserved { + return fmt.Errorf("Repository name %q is reserved", name) } // if all checks pass diff --git a/knotserver/xrpc/delete_branch.go b/knotserver/xrpc/delete_branch.go --- a/knotserver/xrpc/delete_branch.go +++ b/knotserver/xrpc/delete_branch.go @@ -55,8 +55,11 @@ fail(xrpcerr.GenericError(err)) return } - repo := resp.Value.Val.(*tangled.Repo) - repoDid, err := x.Db.GetRepoDid(ident.DID.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + fail(xrpcerr.RepoNotFoundError) + return + } + repoDid, err := x.Db.GetRepoDid(ident.DID.String(), repoAt.RecordKey().String()) if err != nil { fail(xrpcerr.RepoNotFoundError) return diff --git a/knotserver/xrpc/hidden_ref.go b/knotserver/xrpc/hidden_ref.go --- a/knotserver/xrpc/hidden_ref.go +++ b/knotserver/xrpc/hidden_ref.go @@ -61,8 +61,11 @@ fail(xrpcerr.GenericError(err)) return } - repo := resp.Value.Val.(*tangled.Repo) - repoDid, err := x.Db.GetRepoDid(actorDid.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + fail(xrpcerr.RepoNotFoundError) + return + } + repoDid, err := x.Db.GetRepoDid(actorDid.String(), repoAt.RecordKey().String()) if err != nil { fail(xrpcerr.RepoNotFoundError) return diff --git a/knotserver/xrpc/merge.go b/knotserver/xrpc/merge.go --- a/knotserver/xrpc/merge.go +++ b/knotserver/xrpc/merge.go @@ -118,9 +118,8 @@ newSha := gr.Hash() go func() { refUpdate := tangled.GitRefUpdate{ - RepoDid: &repoDid, + Repo: repoDid, OwnerDid: &data.Did, - RepoName: data.Name, Ref: data.Branch, OldSha: oldSha.String(), NewSha: newSha.String(), diff --git a/knotserver/xrpc/repo_describe_repo.go b/knotserver/xrpc/repo_describe_repo.go new file mode 100644 --- /dev/null +++ b/knotserver/xrpc/repo_describe_repo.go @@ -0,0 +1,39 @@ +package xrpc + +import ( + "database/sql" + "errors" + "net/http" + + "github.com/bluesky-social/indigo/atproto/syntax" + "tangled.org/core/api/tangled" + xrpcerr "tangled.org/core/xrpc/errors" +) + +func (x *Xrpc) RepoDescribeRepo(w http.ResponseWriter, r *http.Request) { + raw := r.URL.Query().Get("repoDid") + repoDid, err := syntax.ParseDID(raw) + if err != nil { + writeError(w, xrpcerr.NewXrpcError( + xrpcerr.WithTag("InvalidRequest"), + xrpcerr.WithMessage("missing or invalid repoDid parameter"), + ), http.StatusBadRequest) + return + } + + ownerDid, rkey, err := x.Db.GetRepoKeyOwner(repoDid.String()) + if errors.Is(err, sql.ErrNoRows) { + writeError(w, xrpcerr.RepoNotFoundError, http.StatusNotFound) + return + } + if err != nil { + writeError(w, xrpcerr.GenericError(err), http.StatusInternalServerError) + return + } + + x.writeJson(w, tangled.RepoDescribeRepo_Output{ + RepoDid: repoDid.String(), + OwnerDid: ownerDid, + Rkey: rkey, + }) +} diff --git a/knotserver/xrpc/repo_describe_repo_test.go b/knotserver/xrpc/repo_describe_repo_test.go new file mode 100644 --- /dev/null +++ b/knotserver/xrpc/repo_describe_repo_test.go @@ -0,0 +1,94 @@ +package xrpc + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http" + "net/http/httptest" + "net/url" + "path/filepath" + "testing" + + "tangled.org/core/api/tangled" + "tangled.org/core/knotserver/config" + "tangled.org/core/knotserver/db" +) + +func newTestXrpc(t *testing.T) *Xrpc { + t.Helper() + d, err := db.Setup(context.Background(), filepath.Join(t.TempDir(), "test.db")) + if err != nil { + t.Fatalf("db.Setup: %v", err) + } + return &Xrpc{ + Db: d, + Config: &config.Config{Server: config.Server{Hostname: "knot.example", MaxResponseKB: 5120}}, + Logger: slog.New(slog.NewTextHandler(io.Discard, nil)), + } +} + +func TestRepoDescribeRepo_ReturnsOwner(t *testing.T) { + x := newTestXrpc(t) + if err := x.Db.StoreRepoKey("did:plc:repo1", []byte("dummy"), "did:plc:akshay", "myrepo"); err != nil { + t.Fatalf("StoreRepoKey: %v", err) + } + + req := httptest.NewRequest(http.MethodGet, "/xrpc/sh.tangled.repo.describeRepo?repoDid=did:plc:repo1", nil) + rec := httptest.NewRecorder() + x.RepoDescribeRepo(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body=%s", rec.Code, rec.Body.String()) + } + + var out tangled.RepoDescribeRepo_Output + if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { + t.Fatalf("decode: %v", err) + } + if out.RepoDid != "did:plc:repo1" { + t.Errorf("RepoDid = %q", out.RepoDid) + } + if out.OwnerDid != "did:plc:akshay" { + t.Errorf("OwnerDid = %q, want did:plc:akshay", out.OwnerDid) + } + if out.Rkey != "myrepo" { + t.Errorf("Rkey = %q, want myrepo", out.Rkey) + } +} + +func TestRepoDescribeRepo_UnknownRepoDidReturns404(t *testing.T) { + x := newTestXrpc(t) + + req := httptest.NewRequest(http.MethodGet, "/xrpc/sh.tangled.repo.describeRepo?repoDid=did:plc:unknown", nil) + rec := httptest.NewRecorder() + x.RepoDescribeRepo(rec, req) + + if rec.Code != http.StatusNotFound { + t.Errorf("status = %d, want 404", rec.Code) + } +} + +func TestRepoDescribeRepo_MissingParamReturns400(t *testing.T) { + x := newTestXrpc(t) + + req := httptest.NewRequest(http.MethodGet, "/xrpc/sh.tangled.repo.describeRepo", nil) + rec := httptest.NewRecorder() + x.RepoDescribeRepo(rec, req) + + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", rec.Code) + } +} + +func TestRepoDescribeRepo_MalformedDidParamReturns400(t *testing.T) { + x := newTestXrpc(t) + u := "/xrpc/sh.tangled.repo.describeRepo?repoDid=" + url.QueryEscape("notadid") + req := httptest.NewRequest(http.MethodGet, u, nil) + rec := httptest.NewRecorder() + x.RepoDescribeRepo(rec, req) + if rec.Code != http.StatusBadRequest { + t.Errorf("status = %d, want 400", rec.Code) + } +} diff --git a/knotserver/xrpc/set_default_branch.go b/knotserver/xrpc/set_default_branch.go --- a/knotserver/xrpc/set_default_branch.go +++ b/knotserver/xrpc/set_default_branch.go @@ -59,8 +59,11 @@ fail(xrpcerr.GenericError(err)) return } - repo := resp.Value.Val.(*tangled.Repo) - repoDid, err := x.Db.GetRepoDid(actorDid.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + fail(xrpcerr.RepoNotFoundError) + return + } + repoDid, err := x.Db.GetRepoDid(actorDid.String(), repoAt.RecordKey().String()) if err != nil { fail(xrpcerr.RepoNotFoundError) return @@ -92,9 +95,8 @@ } ownerDid := ident.DID.String() refUpdate := tangled.GitRefUpdate{ - RepoDid: repo.RepoDid, + Repo: repoDid, OwnerDid: &ownerDid, - RepoName: repo.Name, CommitterDid: actorDid.String(), } eventJson, err := json.Marshal(refUpdate) diff --git a/knotserver/xrpc/version.go b/knotserver/xrpc/version.go --- a/knotserver/xrpc/version.go +++ b/knotserver/xrpc/version.go @@ -19,17 +19,13 @@ http.Error(w, "failed to read build info", http.StatusInternalServerError) return } - var modVer string - var sha string - var modified bool - - for _, mod := range info.Deps { - if mod.Path == "tangled.org/tangled.org/knotserver/xrpc" { - modVer = mod.Version - break - } + modVer := info.Main.Version + if modVer == "" || modVer == "(devel)" { + modVer = "(devel)" } + var sha string + var modified bool for _, setting := range info.Settings { switch setting.Key { case "vcs.revision": @@ -37,10 +33,6 @@ sha = setting.Value case "vcs.modified": modified = setting.Value == "true" } - } - - if modVer == "" { - modVer = "unknown" } if sha == "" { diff --git a/knotserver/xrpc/xrpc.go b/knotserver/xrpc/xrpc.go --- a/knotserver/xrpc/xrpc.go +++ b/knotserver/xrpc/xrpc.go @@ -67,6 +67,7 @@ r.Get("/"+tangled.RepoBlobNSID, x.RepoBlob) r.Get("/"+tangled.RepoDiffNSID, x.RepoDiff) r.Get("/"+tangled.RepoCompareNSID, x.RepoCompare) r.Get("/"+tangled.RepoGetDefaultBranchNSID, x.RepoGetDefaultBranch) + r.Get("/"+tangled.RepoDescribeRepoNSID, x.RepoDescribeRepo) r.Get("/"+tangled.RepoBranchNSID, x.RepoBranch) r.Get("/"+tangled.RepoArchiveNSID, x.RepoArchive) r.Get("/"+tangled.RepoLanguagesNSID, x.RepoLanguages) diff --git a/lexicons/feed/star.json b/lexicons/feed/star.json --- a/lexicons/feed/star.json +++ b/lexicons/feed/star.json @@ -10,21 +10,39 @@ "key": "tid", "record": { "type": "object", "required": [ + "subject", "createdAt" ], "properties": { "subject": { - "type": "string", - "format": "at-uri" - }, - "subjectDid": { - "type": "string", - "format": "did" + "type": "union", + "refs": ["#repo", "#string"], + "closed": true }, "createdAt": { "type": "string", "format": "datetime" } + } + } + }, + "repo": { + "type": "object", + "required": ["did"], + "properties": { + "did": { + "type": "string", + "format": "did" + } + } + }, + "string": { + "type": "object", + "required": ["uri"], + "properties": { + "uri": { + "type": "string", + "format": "at-uri" } } } diff --git a/lexicons/git/refUpdate.json b/lexicons/git/refUpdate.json --- a/lexicons/git/refUpdate.json +++ b/lexicons/git/refUpdate.json @@ -11,7 +11,7 @@ "type": "object", "required": [ "ref", "committerDid", - "repoName", + "repo", "oldSha", "newSha", "meta" @@ -33,14 +33,10 @@ "type": "string", "description": "did of the owner of the repo", "format": "did" }, - "repoDid": { + "repo": { "type": "string", "description": "DID of the repo itself", "format": "did" - }, - "repoName": { - "type": "string", - "description": "name of the repo" }, "oldSha": { "type": "string", diff --git a/lexicons/issue/issue.json b/lexicons/issue/issue.json --- a/lexicons/issue/issue.json +++ b/lexicons/issue/issue.json @@ -9,13 +9,9 @@ "type": "record", "key": "tid", "record": { "type": "object", - "required": ["title", "createdAt"], + "required": ["repo", "title", "createdAt"], "properties": { "repo": { - "type": "string", - "format": "at-uri" - }, - "repoDid": { "type": "string", "format": "did" }, diff --git a/lexicons/pulls/pull.json b/lexicons/pulls/pull.json --- a/lexicons/pulls/pull.json +++ b/lexicons/pulls/pull.json @@ -65,14 +65,11 @@ }, "target": { "type": "object", "required": [ + "repo", "branch" ], "properties": { "repo": { - "type": "string", - "format": "at-uri" - }, - "repoDid": { "type": "string", "format": "did" }, @@ -91,10 +88,6 @@ "branch": { "type": "string" }, "repo": { - "type": "string", - "format": "at-uri" - }, - "repoDid": { "type": "string", "format": "did" } diff --git a/lexicons/repo/collaborator.json b/lexicons/repo/collaborator.json --- a/lexicons/repo/collaborator.json +++ b/lexicons/repo/collaborator.json @@ -11,6 +11,7 @@ "record": { "type": "object", "required": [ "subject", + "repo", "createdAt" ], "properties": { @@ -20,11 +21,7 @@ "format": "did" }, "repo": { "type": "string", - "description": "repo to add this user to", - "format": "at-uri" - }, - "repoDid": { - "type": "string", + "description": "repo DID to add this user to", "format": "did" }, "createdAt": { diff --git a/lexicons/repo/describeRepo.json b/lexicons/repo/describeRepo.json new file mode 100644 --- /dev/null +++ b/lexicons/repo/describeRepo.json @@ -0,0 +1,53 @@ +{ + "lexicon": 1, + "id": "sh.tangled.repo.describeRepo", + "defs": { + "main": { + "type": "query", + "description": "Fetch the knot's authoritative metadata for a git repo DID.", + "parameters": { + "type": "params", + "required": ["repoDid"], + "properties": { + "repoDid": { + "type": "string", + "format": "did", + "description": "DID of the git repo as minted by the knot" + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["repoDid", "ownerDid", "rkey"], + "properties": { + "repoDid": { + "type": "string", + "format": "did" + }, + "ownerDid": { + "type": "string", + "format": "did", + "description": "DID of the current owner according to the knot." + }, + "rkey": { + "type": "string", + "description": "Current rkey of the sh.tangled.repo record tracked by this knot" + } + } + } + }, + "errors": [ + { + "name": "RepoNotFound", + "description": "Repo DID is not registered on this knot" + }, + { + "name": "InvalidRequest", + "description": "Invalid request parameters" + } + ] + } + } +} diff --git a/lexicons/repo/repo.json b/lexicons/repo/repo.json --- a/lexicons/repo/repo.json +++ b/lexicons/repo/repo.json @@ -6,18 +6,17 @@ "needsType": true, "defs": { "main": { "type": "record", - "key": "tid", + "key": "any", "record": { "type": "object", "required": [ - "name", "knot", "createdAt" ], "properties": { "name": { "type": "string", - "description": "name of the repo" + "description": "Cosmetic name of the repo." }, "knot": { "type": "string", diff --git a/nix/modules/appview.nix b/nix/modules/appview.nix --- a/nix/modules/appview.nix +++ b/nix/modules/appview.nix @@ -33,6 +33,12 @@ default = "0.0.0.0:${toString cfg.port}"; description = "Listen address for the appview service"; }; + metricsListenAddr = mkOption { + type = types.str; + default = "0.0.0.0:9090"; + description = "Listen address for the Prometheus metrics endpoint"; + }; + dbPath = mkOption { type = types.str; default = "/var/lib/appview/appview.db"; @@ -286,6 +292,7 @@ environment = { TANGLED_DB_PATH = cfg.dbPath; TANGLED_LISTEN_ADDR = cfg.listenAddr; + TANGLED_METRICS_LISTEN_ADDR = cfg.metricsListenAddr; TANGLED_APPVIEW_HOST = cfg.appviewHost; TANGLED_APPVIEW_NAME = cfg.appviewName; TANGLED_DEV = diff --git a/nix/pkgs/knot-unwrapped.nix b/nix/pkgs/knot-unwrapped.nix --- a/nix/pkgs/knot-unwrapped.nix +++ b/nix/pkgs/knot-unwrapped.nix @@ -4,7 +4,7 @@ modules, sqlite-lib, src, }: let - version = "1.13.0-alpha"; + version = "1.14.0-alpha"; in buildGoApplication { pname = "knot"; diff --git a/ogre/src/components/shared/logo.tsx b/ogre/src/components/shared/logo.tsx --- a/ogre/src/components/shared/logo.tsx +++ b/ogre/src/components/shared/logo.tsx @@ -6,63 +6,16 @@ width: 256, height: 70, display: "contents", }}> - - - - - - - - - - - - + + + + + + + + + + ); } diff --git a/orm/orm.go b/orm/orm.go --- a/orm/orm.go +++ b/orm/orm.go @@ -3,11 +3,23 @@ import ( "context" "database/sql" + "errors" "fmt" "log/slog" "reflect" "strings" + + "github.com/mattn/go-sqlite3" ) + +func IsUniqueViolation(err error) bool { + var sqlErr sqlite3.Error + if !errors.As(err, &sqlErr) { + return false + } + return sqlErr.ExtendedCode == sqlite3.ErrConstraintUnique || + sqlErr.ExtendedCode == sqlite3.ErrConstraintPrimaryKey +} type migrationFn = func(*sql.Tx) error diff --git a/spindle/engines/nixery/setup_steps.go b/spindle/engines/nixery/setup_steps.go --- a/spindle/engines/nixery/setup_steps.go +++ b/spindle/engines/nixery/setup_steps.go @@ -8,7 +8,11 @@ func nixConfStep() Step { setupCmd := `mkdir -p /etc/nix echo 'extra-experimental-features = nix-command flakes' >> /etc/nix/nix.conf -echo 'build-users-group = ' >> /etc/nix/nix.conf` +echo 'build-users-group = ' >> /etc/nix/nix.conf +echo 'sandbox = false' >> /etc/nix/nix.conf +printf '#!/bin/sh\nrm -rf /homeless-shelter\n' > /etc/nix/post-build-hook.sh +chmod +x /etc/nix/post-build-hook.sh +echo 'post-build-hook = /etc/nix/post-build-hook.sh' >> /etc/nix/nix.conf` return Step{ command: setupCmd, name: "Configure Nix", diff --git a/spindle/ingester.go b/spindle/ingester.go --- a/spindle/ingester.go +++ b/spindle/ingester.go @@ -5,6 +5,7 @@ "context" "encoding/json" "errors" "fmt" + "strings" "time" "tangled.org/core/api/tangled" @@ -154,26 +155,27 @@ return err } domain := s.cfg.Server.Hostname + rkey := e.Commit.RKey // no spindle configured for this repo if record.Spindle == nil { - l.Info("no spindle configured", "name", record.Name) + l.Info("no spindle configured", "rkey", rkey) return nil } // this repo did not want this spindle if *record.Spindle != domain { - l.Info("different spindle configured", "name", record.Name, "spindle", *record.Spindle, "domain", domain) + l.Info("different spindle configured", "rkey", rkey, "spindle", *record.Spindle, "domain", domain) return nil } // add this repo to the watch list - if err := s.db.AddRepo(record.Knot, did, record.Name); err != nil { + if err := s.db.AddRepo(record.Knot, did, rkey); err != nil { l.Error("failed to add repo", "error", err) return fmt.Errorf("failed to add repo: %w", err) } - didSlashRepo, err := securejoin.SecureJoin(did, record.Name) + didSlashRepo, err := securejoin.SecureJoin(did, rkey) if err != nil { return err } @@ -228,10 +230,18 @@ var rbacResource string var ownerDid string switch { - case record.Repo != nil: - repoAt, parseErr := syntax.ParseATURI(*record.Repo) + case strings.HasPrefix(record.Repo, "did:"): + resolvedOwner, repoName, lookupErr := s.resolveRepoDid(ctx, e.Did, record.Repo) + if lookupErr != nil { + return fmt.Errorf("unknown repo DID %s: %w", record.Repo, lookupErr) + } + ownerDid = resolvedOwner + rbacResource, _ = securejoin.SecureJoin(ownerDid, repoName) + + case strings.Contains(record.Repo, "/"): + repoAt, parseErr := syntax.ParseATURI(record.Repo) if parseErr != nil { - l.Info("rejecting record, invalid repoAt", "repoAt", *record.Repo) + l.Info("rejecting record, invalid repoAt", "repoAt", record.Repo) return nil } @@ -249,12 +259,14 @@ if getErr != nil { return getErr } - repo := resp.Value.Val.(*tangled.Repo) - rbacResource, _ = securejoin.SecureJoin(owner.DID.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + return fmt.Errorf("record at %s is not a tangled.Repo", repoAt) + } + rbacResource, _ = securejoin.SecureJoin(owner.DID.String(), repoAt.RecordKey().String()) ownerDid = owner.DID.String() default: - l.Info("rejecting collaborator record without repo at-uri (spindle RBAC keyed by owner/name)") + l.Info("rejecting collaborator record with unrecognized repo format", "repo", record.Repo) return nil } @@ -270,6 +282,46 @@ return nil } return nil +} + +func (s *Spindle) resolveRepoDid(ctx context.Context, ownerDid string, repoDid string) (string, string, error) { + owner, resolveErr := s.res.ResolveIdent(ctx, ownerDid) + if resolveErr != nil || owner.Handle.IsInvalidHandle() { + return "", "", fmt.Errorf("failed to resolve owner %s: %w", ownerDid, resolveErr) + } + + xrpcc := xrpc.Client{ + Host: owner.PDSEndpoint(), + } + + cursor := "" + for { + resp, listErr := comatproto.RepoListRecords(ctx, &xrpcc, tangled.RepoNSID, cursor, 100, ownerDid, false) + if listErr != nil { + return "", "", fmt.Errorf("failed to list repo records for %s: %w", ownerDid, listErr) + } + + for _, r := range resp.Records { + if r == nil { + continue + } + repo, ok := r.Value.Val.(*tangled.Repo) + if !ok { + continue + } + if repo.RepoDid != nil && *repo.RepoDid == repoDid { + rkey := r.Uri[strings.LastIndex(r.Uri, "/")+1:] + return ownerDid, rkey, nil + } + } + + if resp.Cursor == nil || *resp.Cursor == "" { + break + } + cursor = *resp.Cursor + } + + return "", "", fmt.Errorf("repo DID %s not found in records for %s", repoDid, ownerDid) } func (s *Spindle) fetchAndAddCollaborators(ctx context.Context, owner *identity.Identity, didSlashRepo string) error { diff --git a/spindle/xrpc/add_secret.go b/spindle/xrpc/add_secret.go --- a/spindle/xrpc/add_secret.go +++ b/spindle/xrpc/add_secret.go @@ -61,8 +61,11 @@ fail(xrpcerr.GenericError(err)) return } - repo := resp.Value.Val.(*tangled.Repo) - didPath, err := securejoin.SecureJoin(ident.DID.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + fail(xrpcerr.RepoNotFoundError) + return + } + didPath, err := securejoin.SecureJoin(ident.DID.String(), repoAt.RecordKey().String()) if err != nil { fail(xrpcerr.GenericError(err)) return diff --git a/spindle/xrpc/list_secrets.go b/spindle/xrpc/list_secrets.go --- a/spindle/xrpc/list_secrets.go +++ b/spindle/xrpc/list_secrets.go @@ -56,8 +56,11 @@ fail(xrpcerr.GenericError(err)) return } - repo := resp.Value.Val.(*tangled.Repo) - didPath, err := securejoin.SecureJoin(ident.DID.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + fail(xrpcerr.RepoNotFoundError) + return + } + didPath, err := securejoin.SecureJoin(ident.DID.String(), repoAt.RecordKey().String()) if err != nil { fail(xrpcerr.GenericError(err)) return diff --git a/spindle/xrpc/pipeline_cancel_pipeline.go b/spindle/xrpc/pipeline_cancel_pipeline.go --- a/spindle/xrpc/pipeline_cancel_pipeline.go +++ b/spindle/xrpc/pipeline_cancel_pipeline.go @@ -66,8 +66,11 @@ fail(xrpcerr.GenericError(err)) return } - repo := resp.Value.Val.(*tangled.Repo) - didSlashRepo, err := securejoin.SecureJoin(ident.DID.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + fail(xrpcerr.RepoNotFoundError) + return + } + didSlashRepo, err := securejoin.SecureJoin(ident.DID.String(), repoAt.RecordKey().String()) if err != nil { fail(xrpcerr.GenericError(err)) return diff --git a/spindle/xrpc/remove_secret.go b/spindle/xrpc/remove_secret.go --- a/spindle/xrpc/remove_secret.go +++ b/spindle/xrpc/remove_secret.go @@ -55,8 +55,11 @@ fail(xrpcerr.GenericError(err)) return } - repo := resp.Value.Val.(*tangled.Repo) - didPath, err := securejoin.SecureJoin(ident.DID.String(), repo.Name) + if _, ok := resp.Value.Val.(*tangled.Repo); !ok { + fail(xrpcerr.RepoNotFoundError) + return + } + didPath, err := securejoin.SecureJoin(ident.DID.String(), repoAt.RecordKey().String()) if err != nil { fail(xrpcerr.GenericError(err)) return -- tangled.sh