diff --git a/internal/firehose/index.go b/internal/firehose/index.go index 13fe61e..6da42eb 100644 --- a/internal/firehose/index.go +++ b/internal/firehose/index.go @@ -1194,6 +1194,17 @@ func (idx *FeedIndex) listRecordsByCollection(ctx context.Context, collection st return records, rows.Err() } +// CountReferencesToURI returns how many records have a sourceRef pointing to the given URI. +// This searches the JSON record field across all collections. +func (idx *FeedIndex) CountReferencesToURI(ctx context.Context, uri string) (int, error) { + var count int + err := idx.db.QueryRowContext(ctx, ` + SELECT COUNT(*) FROM records + WHERE json_extract(record, '$.sourceRef') = ? + `, uri).Scan(&count) + return count, err +} + // RecordCount returns the total number of indexed records func (idx *FeedIndex) RecordCount() int { var count int diff --git a/internal/suggestions/suggestions.go b/internal/suggestions/suggestions.go index df48106..4f963b4 100644 --- a/internal/suggestions/suggestions.go +++ b/internal/suggestions/suggestions.go @@ -22,8 +22,14 @@ type EntitySuggestion struct { // RecordSource provides read access to indexed records. type RecordSource interface { ListRecordsByCollectionOldest(ctx context.Context, collection string) ([]firehose.IndexedRecord, error) + CountReferencesToURI(ctx context.Context, uri string) (int, error) } +// PreferredDIDs is a set of DIDs whose records should be preferred when +// choosing the representative sourceRef for a suggestion. Records from +// preferred DIDs get a scoring bonus during deduplication. +var PreferredDIDs = map[string]struct{}{} + // entityFieldConfig defines which fields to extract and search for each entity type type entityFieldConfig struct { allFields []string @@ -238,7 +244,7 @@ func Search(ctx context.Context, source RecordSource, collection, query string, // dedupKey -> aggregated suggestion type candidate struct { suggestion EntitySuggestion - fieldCount int // number of non-empty fields (to pick best representative) + score int // composite score for picking best representative dids map[string]struct{} } candidates := make(map[string]*candidate) @@ -286,15 +292,15 @@ func Search(ctx context.Context, source RecordSource, collection, query string, // Deduplicate using entity-specific key key := config.dedupKey(fields) + score := scoreRecord(ctx, source, indexed.URI, indexed.DID, fields) + if existing, ok := candidates[key]; ok { existing.dids[indexed.DID] = struct{}{} - // Keep the record with more complete fields - nonEmpty := countNonEmpty(fields) - if nonEmpty > existing.fieldCount { + if score > existing.score { existing.suggestion.Name = name existing.suggestion.Fields = fields existing.suggestion.SourceURI = indexed.URI - existing.fieldCount = nonEmpty + existing.score = score } } else { candidates[key] = &candidate{ @@ -303,8 +309,8 @@ func Search(ctx context.Context, source RecordSource, collection, query string, SourceURI: indexed.URI, Fields: fields, }, - fieldCount: countNonEmpty(fields), - dids: map[string]struct{}{indexed.DID: {}}, + score: score, + dids: map[string]struct{}{indexed.DID: {}}, } } } @@ -336,6 +342,25 @@ func Search(ctx context.Context, source RecordSource, collection, query string, return results, nil } +// scoreRecord computes a composite score for choosing the best representative +// record within a dedup group. Higher score wins. Factors: +// - Field completeness (1 point per non-empty field) +// - Reference count (2 points per record that references this URI via sourceRef) +// - Preferred DID bonus (10 points if the record's author is in PreferredDIDs) +func scoreRecord(ctx context.Context, source RecordSource, uri, did string, fields map[string]string) int { + score := countNonEmpty(fields) + + if refCount, err := source.CountReferencesToURI(ctx, uri); err == nil { + score += refCount * 2 + } + + if _, ok := PreferredDIDs[did]; ok { + score += 10 + } + + return score +} + func countNonEmpty(fields map[string]string) int { n := 0 for _, v := range fields { diff --git a/tests/integration/suggestions_test.go b/tests/integration/suggestions_test.go index d50192d..705568e 100644 --- a/tests/integration/suggestions_test.go +++ b/tests/integration/suggestions_test.go @@ -8,6 +8,7 @@ import ( "testing" "arabica/internal/models" + "arabica/internal/suggestions" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -61,6 +62,102 @@ func fetchSuggestions(t *testing.T, h *Harness, client *http.Client, entity, que return results } +// roasterURI builds the AT-URI for a roaster owned by the given DID. +func roasterURI(did, rkey string) string { + return "at://" + did + "/social.arabica.alpha.roaster/" + rkey +} + +// TestHTTP_SuggestionScoring_PrefersReferenced verifies the sourceRef selection +// logic end-to-end: when a roaster has been "adopted" by other users (their +// records carry source_ref pointing back at the original), that original URI +// should win as the canonical sourceRef in the suggestions response. +// +// This is the original motivating scenario for the scoring work — exercises +// witness cache writes, the json_extract reference query, and the composite +// score function all together. +func TestHTTP_SuggestionScoring_PrefersReferenced(t *testing.T) { + h := StartHarness(t, nil) + + // Four users total: alice/bob/carol contribute, dave queries. + bob := h.CreateAccount("bob@test.com", "bob.test", "hunter2") + carol := h.CreateAccount("carol@test.com", "carol.test", "hunter2") + dave := h.CreateAccount("dave@test.com", "dave.test", "hunter2") + + aliceClient := h.Client // primary == alice + bobClient := h.NewClientForAccount(bob) + carolClient := h.NewClientForAccount(carol) + daveClient := h.NewClientForAccount(dave) + alice := h.PrimaryAccount + + // Alice creates the canonical "Counter Culture" roaster (no sourceRef). + aliceRoaster := postRoasterAs(t, h, aliceClient, "Counter Culture", "Durham, NC", "") + aliceURI := roasterURI(alice.DID, aliceRoaster.RKey) + + // Bob and Carol both adopt Alice's roaster — i.e. they create their own + // records with source_ref pointing at her URI. Each adoption gives Alice + // +2 points in the score function (refCount * 2). + postRoasterAs(t, h, bobClient, "Counter Culture Coffee", "Durham, NC", aliceURI) + postRoasterAs(t, h, carolClient, "Counter Culture", "Durham, NC", aliceURI) + + // Dave queries suggestions. The three contributing roasters dedupe into + // one candidate, and Alice's URI should win because two records reference it. + results := fetchSuggestions(t, h, daveClient, "roasters", "counter") + require.NotEmpty(t, results, "expected at least one suggestion") + + var cc *suggestionResult + for i := range results { + if strings.Contains(strings.ToLower(results[i].Name), "counter culture") { + cc = &results[i] + break + } + } + require.NotNil(t, cc, "expected a Counter Culture suggestion in results") + + assert.Equal(t, 3, cc.Count, "all three contributing users should be counted") + assert.Equal(t, aliceURI, cc.SourceURI, + "alice's roaster (referenced by 2 others) should win as the canonical sourceRef") +} + +// TestHTTP_SuggestionScoring_PreferredDIDOverride verifies that a DID added to +// suggestions.PreferredDIDs wins over a record with more references. +func TestHTTP_SuggestionScoring_PreferredDIDOverride(t *testing.T) { + h := StartHarness(t, nil) + + bob := h.CreateAccount("bob@test.com", "bob.test", "hunter2") + carol := h.CreateAccount("carol@test.com", "carol.test", "hunter2") + dave := h.CreateAccount("dave@test.com", "dave.test", "hunter2") + + aliceClient := h.Client + bobClient := h.NewClientForAccount(bob) + carolClient := h.NewClientForAccount(carol) + daveClient := h.NewClientForAccount(dave) + alice := h.PrimaryAccount + + // Alice creates the canonical roaster, Bob and Carol adopt it. + aliceRoaster := postRoasterAs(t, h, aliceClient, "Counter Culture", "Durham, NC", "") + aliceURI := roasterURI(alice.DID, aliceRoaster.RKey) + bobRoaster := postRoasterAs(t, h, bobClient, "Counter Culture Coffee", "Durham, NC", aliceURI) + postRoasterAs(t, h, carolClient, "Counter Culture", "Durham, NC", aliceURI) + + // Mark Bob as preferred. The +10 bonus should overcome Alice's +4 from refs. + suggestions.PreferredDIDs[bob.DID] = struct{}{} + t.Cleanup(func() { delete(suggestions.PreferredDIDs, bob.DID) }) + + results := fetchSuggestions(t, h, daveClient, "roasters", "counter") + var cc *suggestionResult + for i := range results { + if strings.Contains(strings.ToLower(results[i].Name), "counter culture") { + cc = &results[i] + break + } + } + require.NotNil(t, cc) + + bobURI := roasterURI(bob.DID, bobRoaster.RKey) + assert.Equal(t, bobURI, cc.SourceURI, + "bob's roaster should win because his DID is in PreferredDIDs") +} + // TestHTTP_SuggestionDedupe verifies that when multiple users post a roaster // with the same fuzzy-name, the suggestion endpoint dedupes them into a // single result and counts all contributing DIDs.