diff --git a/packages/api/internal/backfill/backfill.go b/packages/api/internal/backfill/backfill.go index 7acae38..a2d911e 100644 --- a/packages/api/internal/backfill/backfill.go +++ b/packages/api/internal/backfill/backfill.go @@ -7,10 +7,14 @@ import ( "sort" "sync" "time" + + "tangled.org/desertthunder.dev/twister/internal/store" ) type discoveryStore interface { GetRepoCollaborators(ctx context.Context, repoOwnerDID string) ([]string, error) + UpsertIdentityHandle(ctx context.Context, did, handle string, isActive bool, status string) error + UpsertDocument(ctx context.Context, doc *store.Document) error } // Runner executes seed resolution, graph discovery, and Tap registration. @@ -19,21 +23,25 @@ type Runner struct { tap tapAdmin resolver handleResolver follows followFetcher + profiles profileFetcher log *slog.Logger } func NewRunner(store discoveryStore, tap tapAdmin, resolver handleResolver, log *slog.Logger) *Runner { - return NewRunnerWithDeps(store, tap, resolver, NewHTTPFollowFetcher(), log) + return NewRunnerWithDeps(store, tap, resolver, NewHTTPFollowFetcher(), NewHTTPProfileFetcher(), log) } -func NewRunnerWithDeps(store discoveryStore, tap tapAdmin, resolver handleResolver, follows followFetcher, log *slog.Logger) *Runner { +func NewRunnerWithDeps(store discoveryStore, tap tapAdmin, resolver handleResolver, follows followFetcher, profiles profileFetcher, log *slog.Logger) *Runner { if log == nil { log = slog.Default() } if follows == nil { follows = NewHTTPFollowFetcher() } - return &Runner{store: store, tap: tap, resolver: resolver, follows: follows, log: log} + if profiles == nil { + profiles = NewHTTPProfileFetcher() + } + return &Runner{store: store, tap: tap, resolver: resolver, follows: follows, profiles: profiles, log: log} } func (r *Runner) Run(ctx context.Context, opts Options) error { @@ -57,7 +65,7 @@ func (r *Runner) Run(ctx context.Context, opts Options) error { if err != nil { return err } - seeds, err := r.resolveSeeds(ctx, seedEntries) + seeds, seedHandles, err := r.resolveSeeds(ctx, seedEntries) if err != nil { return err } @@ -167,12 +175,20 @@ func (r *Runner) Run(ctx context.Context, opts Options) error { slog.Int("status_failures", statusFailures), slog.Int("submit_failures", submitFailures), ) + + if err := r.indexProfiles(ctx, discovered, seedHandles, opts.Concurrency); err != nil { + return fmt.Errorf("index profiles: %w", err) + } + return nil } -func (r *Runner) resolveSeeds(ctx context.Context, entries []seedEntry) ([]string, error) { +// resolveSeeds returns (dids, did→handle map, error). The handle map contains +// entries for seeds that were specified as handles rather than DIDs. +func (r *Runner) resolveSeeds(ctx context.Context, entries []seedEntry) ([]string, map[string]string, error) { seen := map[string]bool{} seeds := make([]string, 0, len(entries)) + handles := make(map[string]string) // did → handle for _, entry := range entries { if entry.isDID { seen[entry.raw] = true @@ -181,15 +197,16 @@ func (r *Runner) resolveSeeds(ctx context.Context, entries []seedEntry) ([]strin } did, err := r.resolver.Resolve(ctx, entry.raw) if err != nil { - return nil, fmt.Errorf("resolve handle at line %d (%s): %w", entry.lineNo, entry.raw, err) + return nil, nil, fmt.Errorf("resolve handle at line %d (%s): %w", entry.lineNo, entry.raw, err) } if seen[did] { continue } seen[did] = true seeds = append(seeds, did) + handles[did] = entry.raw } - return seeds, nil + return seeds, handles, nil } func (r *Runner) discover(ctx context.Context, seeds []string, maxHops int, concurrency int) ([]DiscoveredUser, error) { @@ -300,3 +317,123 @@ func (r *Runner) discover(ctx context.Context, seeds []string, maxHops int, conc return ordered, nil } + +// indexProfiles fetches sh.tangled.actor.profile records via XRPC for each +// discovered user, persists the DID→handle mapping, and upserts a searchable +// profile document. +func (r *Runner) indexProfiles(ctx context.Context, users []DiscoveredUser, seedHandles map[string]string, concurrency int) error { + if concurrency <= 0 { + concurrency = 5 + } + + type result struct { + did string + profile *ProfileRecord + err error + } + + jobs := make(chan string) + results := make(chan result, len(users)) + var wg sync.WaitGroup + for i := 0; i < concurrency; i++ { + wg.Add(1) + go func() { + defer wg.Done() + for did := range jobs { + pr, err := r.profiles.FetchProfile(ctx, did) + results <- result{did: did, profile: pr, err: err} + } + }() + } + + go func() { + for _, u := range users { + jobs <- u.DID + } + close(jobs) + wg.Wait() + close(results) + }() + + indexed := 0 + identities := 0 + failures := 0 + for res := range results { + if res.err != nil { + failures++ + r.log.Warn("profile fetch failed", + slog.String("did", res.did), + slog.String("error", res.err.Error()), + ) + continue + } + + handle := res.profile.Handle + // Prefer the seed handle if the user was specified by handle in seeds. + if h, ok := seedHandles[res.did]; ok && h != "" { + handle = h + } + + if handle != "" { + if err := r.store.UpsertIdentityHandle(ctx, res.did, handle, true, "active"); err != nil { + r.log.Warn("upsert identity handle failed", + slog.String("did", res.did), + slog.String("handle", handle), + slog.String("error", err.Error()), + ) + } else { + identities++ + } + } + + // Only create a document if we got a profile record back. + if res.profile.Record == nil { + continue + } + + description, _ := res.profile.Record["description"].(string) + location, _ := res.profile.Record["location"].(string) + summary := description + if location != "" { + if summary != "" { + summary = summary + " · " + location + } else { + summary = location + } + } + if len(summary) > 200 { + summary = summary[:200] + } + + doc := &store.Document{ + ID: fmt.Sprintf("%s|%s|self", res.did, profileCollection), + DID: res.did, + Collection: profileCollection, + RKey: "self", + ATURI: fmt.Sprintf("at://%s/%s/self", res.did, profileCollection), + CID: res.profile.CID, + RecordType: "profile", + Title: handle, + Body: description, + Summary: summary, + AuthorHandle: handle, + TagsJSON: "[]", + } + + if err := r.store.UpsertDocument(ctx, doc); err != nil { + r.log.Warn("upsert profile document failed", + slog.String("did", res.did), + slog.String("error", err.Error()), + ) + continue + } + indexed++ + } + + r.log.Info("profile indexing complete", + slog.Int("identities_stored", identities), + slog.Int("profiles_indexed", indexed), + slog.Int("failures", failures), + ) + return nil +} diff --git a/packages/api/internal/backfill/backfill_test.go b/packages/api/internal/backfill/backfill_test.go index dfc6b62..ab0d0b6 100644 --- a/packages/api/internal/backfill/backfill_test.go +++ b/packages/api/internal/backfill/backfill_test.go @@ -9,16 +9,33 @@ import ( "path/filepath" "strings" "testing" + + "tangled.org/desertthunder.dev/twister/internal/store" ) type fakeStore struct { collaborators map[string][]string + identities map[string]string + documents []*store.Document } func (f *fakeStore) GetRepoCollaborators(_ context.Context, did string) ([]string, error) { return f.collaborators[did], nil } +func (f *fakeStore) UpsertIdentityHandle(_ context.Context, did, handle string, _ bool, _ string) error { + if f.identities == nil { + f.identities = map[string]string{} + } + f.identities[did] = handle + return nil +} + +func (f *fakeStore) UpsertDocument(_ context.Context, doc *store.Document) error { + f.documents = append(f.documents, doc) + return nil +} + type fakeFollowFetcher struct { follows map[string][]string } @@ -67,6 +84,17 @@ func (r *fakeResolver) Resolve(_ context.Context, handle string) (string, error) return "", io.EOF } +type fakeProfileFetcher struct { + profiles map[string]*ProfileRecord +} + +func (f *fakeProfileFetcher) FetchProfile(_ context.Context, did string) (*ProfileRecord, error) { + if pr, ok := f.profiles[did]; ok { + return pr, nil + } + return &ProfileRecord{}, nil +} + func TestRunner_DiscoveryAndSubmit(t *testing.T) { st := &fakeStore{ collaborators: map[string][]string{ @@ -77,7 +105,7 @@ func TestRunner_DiscoveryAndSubmit(t *testing.T) { tap := &fakeTapAdmin{statuses: map[string]RepoStatus{"did:plc:f1": {Found: true, Tracked: true, Backfilled: true}}} resolver := &fakeResolver{mapping: map[string]string{"alice.tangled.sh": "did:plc:seed"}} log := slog.New(slog.NewTextHandler(io.Discard, nil)) - r := NewRunnerWithDeps(st, tap, resolver, follows, log) + r := NewRunnerWithDeps(st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, log) dir := t.TempDir() seedsPath := filepath.Join(dir, "seeds.txt") @@ -109,7 +137,7 @@ func TestRunner_DryRunSkipsMutations(t *testing.T) { tap := &fakeTapAdmin{statuses: map[string]RepoStatus{}} resolver := &fakeResolver{mapping: map[string]string{"alice.tangled.sh": "did:plc:seed"}} log := slog.New(slog.NewTextHandler(io.Discard, nil)) - r := NewRunnerWithDeps(st, tap, resolver, follows, log) + r := NewRunnerWithDeps(st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, log) dir := t.TempDir() seedsPath := filepath.Join(dir, "seeds.txt") @@ -140,7 +168,7 @@ func TestRunner_SkipsInProgressBackfills(t *testing.T) { }} resolver := &fakeResolver{mapping: map[string]string{"alice.tangled.sh": "did:plc:seed"}} log := slog.New(slog.NewTextHandler(io.Discard, nil)) - r := NewRunnerWithDeps(st, tap, resolver, follows, log) + r := NewRunnerWithDeps(st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, log) dir := t.TempDir() seedsPath := filepath.Join(dir, "seeds.txt") @@ -170,7 +198,7 @@ func TestRunner_ContinuesWhenRepoStatusFails(t *testing.T) { } resolver := &fakeResolver{mapping: map[string]string{"alice.tangled.sh": "did:plc:seed"}} log := slog.New(slog.NewTextHandler(io.Discard, nil)) - r := NewRunnerWithDeps(st, tap, resolver, follows, log) + r := NewRunnerWithDeps(st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, log) dir := t.TempDir() seedsPath := filepath.Join(dir, "seeds.txt") @@ -222,7 +250,7 @@ func TestRunner_FallsBackToSingleRepoSubmissionOnBatchFailure(t *testing.T) { } resolver := &fakeResolver{mapping: map[string]string{"alice.tangled.sh": "did:plc:seed"}} log := slog.New(slog.NewTextHandler(io.Discard, nil)) - r := NewRunnerWithDeps(st, tap, resolver, follows, log) + r := NewRunnerWithDeps(st, tap, resolver, follows, &fakeProfileFetcher{profiles: map[string]*ProfileRecord{}}, log) dir := t.TempDir() seedsPath := filepath.Join(dir, "seeds.txt") @@ -252,3 +280,59 @@ func TestRunner_FallsBackToSingleRepoSubmissionOnBatchFailure(t *testing.T) { } } } + +func TestRunner_IndexesProfilesAndHandles(t *testing.T) { + st := &fakeStore{collaborators: map[string][]string{}} + follows := &fakeFollowFetcher{follows: map[string][]string{}} + tap := &fakeTapAdmin{statuses: map[string]RepoStatus{}} + resolver := &fakeResolver{mapping: map[string]string{"alice.tangled.sh": "did:plc:seed"}} + profiles := &fakeProfileFetcher{profiles: map[string]*ProfileRecord{ + "did:plc:seed": { + Record: map[string]any{ + "description": "Building cool stuff", + "location": "NYC", + }, + CID: "bafyabc123", + Handle: "alice.tangled.sh", + }, + }} + log := slog.New(slog.NewTextHandler(io.Discard, nil)) + r := NewRunnerWithDeps(st, tap, resolver, follows, profiles, log) + + dir := t.TempDir() + seedsPath := filepath.Join(dir, "seeds.txt") + if err := os.WriteFile(seedsPath, []byte("alice.tangled.sh\n"), 0o644); err != nil { + t.Fatalf("write seeds: %v", err) + } + + err := r.Run(context.Background(), Options{SeedsPath: seedsPath, MaxHops: 0}) + if err != nil { + t.Fatalf("run backfill: %v", err) + } + + // Identity handle should be persisted. + if st.identities["did:plc:seed"] != "alice.tangled.sh" { + t.Fatalf("expected identity handle for seed DID, got %#v", st.identities) + } + + // Profile document should be created. + if len(st.documents) != 1 { + t.Fatalf("expected 1 profile document, got %d", len(st.documents)) + } + doc := st.documents[0] + if doc.Title != "alice.tangled.sh" { + t.Errorf("expected title to be handle, got %q", doc.Title) + } + if doc.AuthorHandle != "alice.tangled.sh" { + t.Errorf("expected author_handle to be handle, got %q", doc.AuthorHandle) + } + if doc.Body != "Building cool stuff" { + t.Errorf("expected body to be description, got %q", doc.Body) + } + if doc.RecordType != "profile" { + t.Errorf("expected record_type profile, got %q", doc.RecordType) + } + if !strings.Contains(doc.Summary, "NYC") { + t.Errorf("expected summary to contain location, got %q", doc.Summary) + } +} diff --git a/packages/api/internal/backfill/profile.go b/packages/api/internal/backfill/profile.go new file mode 100644 index 0000000..cc174b2 --- /dev/null +++ b/packages/api/internal/backfill/profile.go @@ -0,0 +1,146 @@ +package backfill + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/url" + "strings" + "time" +) + +const profileCollection = "sh.tangled.actor.profile" + +// ProfileRecord holds the fetched profile data and resolved handle. +type ProfileRecord struct { + Record map[string]any + CID string + Handle string +} + +type profileFetcher interface { + FetchProfile(ctx context.Context, did string) (*ProfileRecord, error) +} + +// HTTPProfileFetcher fetches sh.tangled.actor.profile records via XRPC +// and resolves handles from the DID document. +type HTTPProfileFetcher struct { + client *http.Client +} + +func NewHTTPProfileFetcher() *HTTPProfileFetcher { + return &HTTPProfileFetcher{ + client: &http.Client{Timeout: 15 * time.Second}, + } +} + +func (f *HTTPProfileFetcher) FetchProfile(ctx context.Context, did string) (*ProfileRecord, error) { + pds, handle, err := f.resolveDIDDoc(ctx, did) + if err != nil { + return nil, fmt.Errorf("resolve did doc: %w", err) + } + + u, err := url.Parse(strings.TrimSuffix(pds, "/") + "/xrpc/com.atproto.repo.getRecord") + if err != nil { + return nil, fmt.Errorf("build getRecord url: %w", err) + } + q := u.Query() + q.Set("repo", did) + q.Set("collection", profileCollection) + q.Set("rkey", "self") + u.RawQuery = q.Encode() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return nil, fmt.Errorf("build getRecord request: %w", err) + } + + resp, err := f.client.Do(req) + if err != nil { + return nil, fmt.Errorf("getRecord request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusNotFound { + // No profile record — return handle only so identity can still be stored. + return &ProfileRecord{Handle: handle}, nil + } + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("getRecord failed: status %d", resp.StatusCode) + } + + var payload struct { + CID string `json:"cid"` + Value map[string]any `json:"value"` + } + if err := json.NewDecoder(resp.Body).Decode(&payload); err != nil { + return nil, fmt.Errorf("decode getRecord response: %w", err) + } + + return &ProfileRecord{ + Record: payload.Value, + CID: payload.CID, + Handle: handle, + }, nil +} + +// resolveDIDDoc fetches the DID document and returns (pdsEndpoint, handle, error). +func (f *HTTPProfileFetcher) resolveDIDDoc(ctx context.Context, did string) (string, string, error) { + var didDocURL string + switch { + case strings.HasPrefix(did, "did:plc:"): + didDocURL = plcDirectoryBase + "/" + url.PathEscape(did) + case strings.HasPrefix(did, "did:web:"): + hostAndPath := strings.TrimPrefix(did, "did:web:") + hostAndPath = strings.ReplaceAll(hostAndPath, ":", "/") + didDocURL = "https://" + hostAndPath + "/.well-known/did.json" + default: + return "", "", fmt.Errorf("unsupported did type: %s", did) + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, didDocURL, nil) + if err != nil { + return "", "", fmt.Errorf("build did doc request: %w", err) + } + resp, err := f.client.Do(req) + if err != nil { + return "", "", fmt.Errorf("did doc request: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return "", "", fmt.Errorf("did doc lookup failed: status %d", resp.StatusCode) + } + + var didDoc struct { + AlsoKnownAs []string `json:"alsoKnownAs"` + Service []struct { + Type string `json:"type"` + ServiceEndpoint string `json:"serviceEndpoint"` + } `json:"service"` + } + if err := json.NewDecoder(resp.Body).Decode(&didDoc); err != nil { + return "", "", fmt.Errorf("decode did doc: %w", err) + } + + var pds string + for _, svc := range didDoc.Service { + if svc.Type == "AtprotoPersonalDataServer" && strings.TrimSpace(svc.ServiceEndpoint) != "" { + pds = strings.TrimSpace(svc.ServiceEndpoint) + break + } + } + if pds == "" { + return "", "", fmt.Errorf("no atproto pds endpoint in did document") + } + + var handle string + for _, aka := range didDoc.AlsoKnownAs { + if strings.HasPrefix(aka, "at://") { + handle = strings.TrimPrefix(aka, "at://") + break + } + } + + return pds, handle, nil +}