diff --git a/packages/api/internal/backfill/backfill.go b/packages/api/internal/backfill/backfill.go --- a/packages/api/internal/backfill/backfill.go +++ b/packages/api/internal/backfill/backfill.go @@ -26,6 +26,7 @@ resolver handleResolver follows followFetcher profiles profileFetcher + repos repoFetcher lightrail lightrailRepoLister log *slog.Logger } @@ -36,6 +37,7 @@ NewXRPCHandleResolver(xrpcClient), NewXRPCFollowFetcher(xrpcClient), NewXRPCProfileFetcher(xrpcClient), + NewXRPCRepoFetcher(xrpcClient), NewHTTPLightrailClient(), log, ) @@ -43,7 +45,7 @@ func NewRunnerWithDeps( store discoveryStore, tap tapAdmin, resolver handleResolver, - follows followFetcher, profiles profileFetcher, lightrail lightrailRepoLister, + follows followFetcher, profiles profileFetcher, repos repoFetcher, lightrail lightrailRepoLister, log *slog.Logger, ) *Runner { if log == nil { @@ -54,7 +56,7 @@ } return &Runner{ store: store, tap: tap, resolver: resolver, follows: follows, - profiles: profiles, lightrail: lightrail, log: log, + profiles: profiles, repos: repos, lightrail: lightrail, log: log, } } @@ -167,8 +169,8 @@ slog.Int("submit_failures", submitFailures), ) - if err := r.indexProfiles(ctx, discovered, seedHandles, opts.Concurrency); err != nil { - return fmt.Errorf("index profiles: %w", err) + if err := r.bootstrapProfilesAndRepos(ctx, discovered, seedHandles, opts.Concurrency); err != nil { + return fmt.Errorf("bootstrap profiles and repos: %w", err) } return nil @@ -218,6 +220,11 @@ slog.Int("submitted", submitted), slog.Int("submit_failures", submitFailures), ) + + if err := r.bootstrapProfilesAndRepos(ctx, discovered, nil, opts.Concurrency); err != nil { + return fmt.Errorf("bootstrap profiles and repos: %w", err) + } + return nil } @@ -448,18 +455,20 @@ return normalized } -// indexProfiles fetches sh.tangled.actor.profile records via XRPC for each -// discovered user, persists the DID→handle mapping, and upserts a searchable -// profile document. -func (r *Runner) indexProfiles(ctx context.Context, users []DiscoveredUser, seedHandles map[string]string, concurrency int) error { +// bootstrapProfilesAndRepos fetches actor profiles and repo records via XRPC +// for each discovered user, persists DID→handle mappings, and upserts +// searchable bootstrap documents. +func (r *Runner) bootstrapProfilesAndRepos(ctx context.Context, users []DiscoveredUser, seedHandles map[string]string, concurrency int) error { if concurrency <= 0 { concurrency = 5 } type result struct { - did string - profile *ProfileRecord - err error + did string + profile *ProfileRecord + repos []RepoRecord + profileErr error + repoErr error } jobs := make(chan string) @@ -470,8 +479,14 @@ go func() { defer wg.Done() for did := range jobs { - pr, err := r.profiles.FetchProfile(ctx, did) - results <- result{did: did, profile: pr, err: err} + res := result{did: did} + if r.profiles != nil { + res.profile, res.profileErr = r.profiles.FetchProfile(ctx, did) + } + if r.repos != nil { + res.repos, res.repoErr = r.repos.ListRepos(ctx, did) + } + results <- res } }() } @@ -485,20 +500,30 @@ close(results) }() - indexed := 0 + profilesIndexed := 0 + reposIndexed := 0 identities := 0 failures := 0 for res := range results { - if res.err != nil { + if res.profileErr != nil { failures++ r.log.Warn("profile fetch failed", slog.String("did", res.did), - slog.String("error", res.err.Error()), + slog.String("error", res.profileErr.Error()), ) - continue + } + if res.repoErr != nil { + failures++ + r.log.Warn("repo list failed", + slog.String("did", res.did), + slog.String("error", res.repoErr.Error()), + ) } - handle := res.profile.Handle + handle := "" + if res.profile != nil { + handle = res.profile.Handle + } if h, ok := seedHandles[res.did]; ok && h != "" { handle = h } @@ -515,52 +540,39 @@ } } - if res.profile.Record == nil { - continue - } - - description, _ := res.profile.Record["description"].(string) - location, _ := res.profile.Record["location"].(string) - summary := description - if location != "" { - if summary != "" { - summary = summary + " · " + location + doc := bootstrapProfileDocument(res.did, res.profile, handle) + if doc != nil { + if err := r.store.UpsertDocument(ctx, doc); err != nil { + r.log.Warn("upsert profile document failed", + slog.String("did", res.did), + slog.String("error", err.Error()), + ) } else { - summary = location + profilesIndexed++ } } - if len(summary) > 200 { - summary = summary[:200] - } - doc := &store.Document{ - ID: fmt.Sprintf("%s|%s|self", res.did, profileCollection), - DID: res.did, - Collection: profileCollection, - RKey: "self", - ATURI: fmt.Sprintf("at://%s/%s/self", res.did, profileCollection), - CID: res.profile.CID, - RecordType: "profile", - Title: handle, - Body: description, - Summary: summary, - AuthorHandle: handle, - TagsJSON: "[]", + for _, repo := range res.repos { + doc := bootstrapRepoDocument(res.did, handle, repo) + if doc == nil { + continue + } + if err := r.store.UpsertDocument(ctx, doc); err != nil { + r.log.Warn("upsert repo document failed", + slog.String("did", res.did), + slog.String("rkey", repo.RKey), + slog.String("error", err.Error()), + ) + continue + } + reposIndexed++ } - - if err := r.store.UpsertDocument(ctx, doc); err != nil { - r.log.Warn("upsert profile document failed", - slog.String("did", res.did), - slog.String("error", err.Error()), - ) - continue - } - indexed++ } - r.log.Info("profile indexing complete", + r.log.Info("bootstrap indexing complete", slog.Int("identities_stored", identities), - slog.Int("profiles_indexed", indexed), + slog.Int("profiles_indexed", profilesIndexed), + slog.Int("repos_indexed", reposIndexed), slog.Int("failures", failures), ) return nil diff --git a/packages/api/internal/backfill/backfill_test.go b/packages/api/internal/backfill/backfill_test.go --- a/packages/api/internal/backfill/backfill_test.go +++ b/packages/api/internal/backfill/backfill_test.go @@ -95,6 +95,17 @@ return &ProfileRecord{}, nil } +type fakeRepoFetcher struct { + repos map[string][]RepoRecord +} + +func (f *fakeRepoFetcher) ListRepos(_ context.Context, did string) ([]RepoRecord, error) { + if repos, ok := f.repos[did]; ok { + return repos, nil + } + return nil, nil +} + type fakeLightrailRepoLister struct { dids []string err error @@ -130,6 +141,7 @@ r := NewRunnerWithDeps( st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, + &fakeRepoFetcher{}, &fakeLightrailRepoLister{}, log, ) @@ -167,6 +179,7 @@ r := NewRunnerWithDeps( st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, + &fakeRepoFetcher{}, &fakeLightrailRepoLister{}, log, ) @@ -203,6 +216,7 @@ r := NewRunnerWithDeps( st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, + &fakeRepoFetcher{}, &fakeLightrailRepoLister{}, log, ) @@ -239,6 +253,7 @@ r := NewRunnerWithDeps( st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, + &fakeRepoFetcher{}, &fakeLightrailRepoLister{}, log, ) @@ -296,6 +311,7 @@ r := NewRunnerWithDeps( st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, + &fakeRepoFetcher{}, &fakeLightrailRepoLister{}, log, ) @@ -346,7 +362,7 @@ }} log := slog.New(slog.NewTextHandler(io.Discard, nil)) r := NewRunnerWithDeps( - st, tap, resolver, follows, profiles, &fakeLightrailRepoLister{}, log, + st, tap, resolver, follows, profiles, &fakeRepoFetcher{}, &fakeLightrailRepoLister{}, log, ) dir := t.TempDir() @@ -395,7 +411,7 @@ lightrail := &fakeLightrailRepoLister{dids: []string{"did:plc:b", "did:plc:a"}} log := slog.New(slog.NewTextHandler(io.Discard, nil)) r := NewRunnerWithDeps( - st, tap, &fakeResolver{}, &fakeFollowFetcher{}, &fakeProfileFetcher{}, + st, tap, &fakeResolver{}, &fakeFollowFetcher{}, &fakeProfileFetcher{}, &fakeRepoFetcher{}, lightrail, log, ) @@ -432,7 +448,7 @@ } log := slog.New(slog.NewTextHandler(io.Discard, nil)) r := NewRunnerWithDeps( - st, tap, &fakeResolver{}, &fakeFollowFetcher{}, &fakeProfileFetcher{}, + st, tap, &fakeResolver{}, &fakeFollowFetcher{}, &fakeProfileFetcher{}, &fakeRepoFetcher{}, lightrail, log, ) @@ -448,5 +464,122 @@ } if len(tap.added[0]) != 2 { t.Fatalf("expected deduped DIDs, got %#v", tap.added) + } +} + +func TestRunner_LightrailIndexesProfiles(t *testing.T) { + st := &fakeStore{collaborators: map[string][]string{}} + tap := &fakeTapAdmin{} + lightrail := &fakeLightrailRepoLister{ + dids: []string{"did:plc:xg2vq45muivyy3xwatcehspu"}, + } + profiles := &fakeProfileFetcher{profiles: map[string]*ProfileRecord{ + "did:plc:xg2vq45muivyy3xwatcehspu": { + Record: map[string]any{ + "description": "Twisted maintainer", + "location": "Chicago", + }, + CID: "bafydesert123", + Handle: "desertthunder.dev", + }, + }} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + r := NewRunnerWithDeps( + st, tap, &fakeResolver{}, &fakeFollowFetcher{}, profiles, &fakeRepoFetcher{}, lightrail, log, + ) + + err := r.Run(context.Background(), Options{ + Source: SourceLightrail, + BatchSize: 10, + Concurrency: 1, + Collections: []string{"sh.tangled.repo"}, + }) + if err != nil { + t.Fatalf("run lightrail backfill: %v", err) + } + if st.identities["did:plc:xg2vq45muivyy3xwatcehspu"] != "desertthunder.dev" { + t.Fatalf("expected identity handle to be stored, got %#v", st.identities) + } + if len(st.documents) != 1 { + t.Fatalf("expected one profile document, got %d", len(st.documents)) + } + doc := st.documents[0] + if doc.RecordType != "profile" { + t.Fatalf("expected profile document, got %q", doc.RecordType) + } + if doc.AuthorHandle != "desertthunder.dev" { + t.Fatalf("expected author_handle desertthunder.dev, got %q", doc.AuthorHandle) + } + if doc.Title != "desertthunder.dev" { + t.Fatalf("expected title desertthunder.dev, got %q", doc.Title) + } +} + +func TestRunner_LightrailIndexesReposDirectly(t *testing.T) { + st := &fakeStore{collaborators: map[string][]string{}} + tap := &fakeTapAdmin{} + lightrail := &fakeLightrailRepoLister{ + dids: []string{"did:plc:xg2vq45muivyy3xwatcehspu"}, + } + profiles := &fakeProfileFetcher{profiles: map[string]*ProfileRecord{ + "did:plc:xg2vq45muivyy3xwatcehspu": { + Handle: "desertthunder.dev", + Record: map[string]any{ + "description": "Twisted maintainer", + }, + CID: "bafydesert123", + }, + }} + repos := &fakeRepoFetcher{repos: map[string][]RepoRecord{ + "did:plc:xg2vq45muivyy3xwatcehspu": { + { + RKey: "3mho6hukiei22", + CID: "bafyreitwisted123", + Record: map[string]any{ + "name": "twisted", + "description": "A tangled mobile client", + "topics": []any{"go", "search"}, + "createdAt": "2026-03-01T00:00:00Z", + }, + }, + }, + }} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + r := NewRunnerWithDeps( + st, tap, &fakeResolver{}, &fakeFollowFetcher{}, profiles, repos, lightrail, log, + ) + + err := r.Run(context.Background(), Options{ + Source: SourceLightrail, + BatchSize: 10, + Concurrency: 1, + Collections: []string{"sh.tangled.repo"}, + }) + if err != nil { + t.Fatalf("run lightrail backfill: %v", err) + } + + if len(st.documents) != 2 { + t.Fatalf("expected profile and repo bootstrap documents, got %d", len(st.documents)) + } + + var foundRepo *store.Document + for _, doc := range st.documents { + if doc.RecordType == "repo" { + foundRepo = doc + break + } + } + if foundRepo == nil { + t.Fatal("expected repo bootstrap document") + } + if foundRepo.Title != "twisted" { + t.Fatalf("expected repo title twisted, got %q", foundRepo.Title) + } + if foundRepo.AuthorHandle != "desertthunder.dev" { + t.Fatalf("expected repo author_handle desertthunder.dev, got %q", foundRepo.AuthorHandle) + } + if foundRepo.WebURL == "" { + t.Fatal("expected repo web_url to be populated") } } diff --git a/packages/api/internal/backfill/profile.go b/packages/api/internal/backfill/profile.go --- a/packages/api/internal/backfill/profile.go +++ b/packages/api/internal/backfill/profile.go @@ -5,10 +5,13 @@ "errors" "fmt" + "tangled.org/desertthunder.dev/twister/internal/normalize" + "tangled.org/desertthunder.dev/twister/internal/store" "tangled.org/desertthunder.dev/twister/internal/xrpc" ) const profileCollection = "sh.tangled.actor.profile" +const repoCollection = "sh.tangled.repo" // ProfileRecord holds the fetched profile data and resolved handle. type ProfileRecord struct { @@ -19,6 +22,16 @@ type profileFetcher interface { FetchProfile(ctx context.Context, did string) (*ProfileRecord, error) +} + +type RepoRecord struct { + RKey string + CID string + Record map[string]any +} + +type repoFetcher interface { + ListRepos(ctx context.Context, did string) ([]RepoRecord, error) } // XRPCProfileFetcher fetches sh.tangled.actor.profile records via xrpc.Client @@ -51,4 +64,99 @@ CID: rec.CID, Handle: info.Handle, }, nil +} + +type XRPCRepoFetcher struct { + client *xrpc.Client +} + +func NewXRPCRepoFetcher(client *xrpc.Client) *XRPCRepoFetcher { + return &XRPCRepoFetcher{client: client} +} + +func (f *XRPCRepoFetcher) ListRepos(ctx context.Context, did string) ([]RepoRecord, error) { + info, err := f.client.ResolveIdentity(ctx, did) + if err != nil { + return nil, fmt.Errorf("resolve identity: %w", err) + } + + records, err := f.client.ListAllRecords(ctx, info.PDS, did, repoCollection) + if err != nil { + return nil, fmt.Errorf("list repos: %w", err) + } + + repos := make([]RepoRecord, 0, len(records)) + for _, rec := range records { + _, _, rkey, err := normalize.ParseATURI(rec.URI) + if err != nil { + continue + } + repos = append(repos, RepoRecord{ + RKey: rkey, + CID: rec.CID, + Record: rec.Value, + }) + } + return repos, nil +} + +func bootstrapProfileDocument(did string, profile *ProfileRecord, handle string) *store.Document { + if profile == nil || profile.Record == nil { + return nil + } + + description, _ := profile.Record["description"].(string) + location, _ := profile.Record["location"].(string) + summary := description + if location != "" { + if summary != "" { + summary = summary + " · " + location + } else { + summary = location + } + } + if len(summary) > 200 { + summary = summary[:200] + } + + return &store.Document{ + ID: fmt.Sprintf("%s|%s|self", did, profileCollection), + DID: did, + Collection: profileCollection, + RKey: "self", + ATURI: fmt.Sprintf("at://%s/%s/self", did, profileCollection), + CID: profile.CID, + RecordType: "profile", + Title: handle, + Body: description, + Summary: summary, + AuthorHandle: handle, + TagsJSON: "[]", + } +} + +func bootstrapRepoDocument(did, handle string, repo RepoRecord) *store.Document { + adapter := &normalize.RepoAdapter{} + if !adapter.Searchable(repo.Record) { + return nil + } + + event := normalize.TapRecordEvent{ + Type: "record", + Record: &normalize.TapRecord{ + DID: did, + Collection: repoCollection, + RKey: repo.RKey, + CID: repo.CID, + Record: repo.Record, + }, + } + + doc, err := adapter.Normalize(event) + if err != nil { + return nil + } + doc.AuthorHandle = handle + doc.WebURL = xrpc.BuildWebURL(handle, doc.RepoName, doc.RecordType, doc.RKey) + return doc }