From 417147c03bd09b3f20797d38f74f0c9505b13bdf Mon Sep 17 00:00:00 2001 From: Bretton Date: Tue, 1 Sep 2026 14:33:02 -0700 Subject: [PATCH] feat(bridgedvotes): poll the bridge vote-aggregate side channel for native content Bridge-asserted vote tallies used to reach the AppView only as bridgedStats stamped onto records via Jetstream, a channel that cannot touch content a native user authored in their own PDS repo. Votes cast on Lemmy against native posts and comments were therefore aggregated by the bridge and displayed by nobody. This adds a background poller that sweeps the trusted bridges' social.coves.bridge.getVoteAggregates endpoint for content in bridged communities and folds the fediverse tallies into the existing bridged_upvote_count / bridged_downvote_count / bridged_stats_as_of columns under the same >= asOf guard the Jetstream path uses. The invariant the design is built around: a value from the database (communities.pds_url) is never a dial target. Stored URLs are match keys only; the dial target is always the matched operator-configured host from TRUSTED_BRIDGE_PDS_HOSTS, and that is now a type (TrustedHost) that only ParseTrustedHost can produce. Changes: - internal/core/bridgedvotes: new package. Client (SSRF-guarded default, redirect refusal, 1 MiB cap, request-set binding, first-wins dedupe, transient vs permanent classification with 408/425/429/5xx transient, missing-key and all-rejected responses as contract errors). Poller.Sweep groups candidates per trusted host with per-host budgets, marks permanently failed batches past the rotation, bounds transient streaks at three sweeps so a persistent 5xx cannot wedge a host, isolates selection failures per host, and returns a Report the job logs. Shared NormalizeHost backs both trust matching and routing; shared ParseAsOf rejects the zero time and stamps more than five minutes ahead of the AppView clock, in both ingestion channels. MaxBridgedCount is exported and the Jetstream adapter aliases it. - internal/db/postgres: BridgedVotesRepository with candidate selection across posts UNION ALL comments (comments qualify only via a non-deleted indexed root), guarded ApplyAggregate with score recompute in one UPDATE, ErrMissingAsOf for a zero stamp, transactional MarkPolled. - Migration 043: bridged_polled_at watermark columns and partial rotation indexes on posts and comments; corrects 031's column comments to the >= semantics actually implemented, with a symmetric Down. - internal/config: BRIDGED_VOTE_POLL_INTERVAL / _LOOKBACK / _SWEEP_CAP; Validate rejects trusted-host entries that are not scheme+host, a non-positive interval when a trust list is set, and negative tuning. - cmd/server: buildBridgedVotePoller under the same non-empty trust guard as BridgeTrust, using the guarded client with the dev private-host hatch; startBridgedVotePollJob logs a report per sweep and warns when stored community hosts match nothing in the trust list. - docker-compose.prod.yml / .env.prod.example: pass-through and docs for the new variables, including per-host cap semantics and the fail-fast contract for malformed trust entries. - Tests: T0 coverage for the client, poller, normalizer, TrustedHost, ParseAsOf, config parsing and validation, job guard, and server wiring; T1 coverage for the repository, migration rollback chains, and an end-to-end sweep through the real repository, client, and an httptest bridge. Verified: make test, integration tier for touched packages, test-audit and ssrf-audit clean. Deferred: EXPLAIN ANALYZE the two partial indexes on production-shaped data (the ORDER BY runs after the UNION, so they may not serve the query); ApplyAggregate still probes posts then comments rather than splitting on the URI collection. Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_018Jd4QsKtSPJRrMc83ayyqD --- .env.prod.example | 18 + cmd/server/jobs.go | 71 + cmd/server/jobs_test.go | 101 ++ cmd/server/main.go | 7 + cmd/server/wiring.go | 50 +- cmd/server/wiring_bridgedvotes_test.go | 54 + docker-compose.prod.yml | 8 + internal/atproto/jetstream/bridge_trust.go | 30 +- .../jetstream/community_origin_test.go | 7 + internal/atproto/jetstream/post_consumer.go | 9 +- .../jetstream/postv2_mechanisms_test.go | 3 + internal/config/config.go | 64 + internal/config/config_bridgedvotes_test.go | 120 ++ internal/config/config_test.go | 2 +- internal/config/testing.go | 1 + internal/core/bridgedvotes/bridgedvotes.go | 131 ++ internal/core/bridgedvotes/client.go | 261 ++++ internal/core/bridgedvotes/client_test.go | 527 ++++++++ internal/core/bridgedvotes/harness_test.go | 14 + internal/core/bridgedvotes/normalizer.go | 40 + internal/core/bridgedvotes/normalizer_test.go | 17 + internal/core/bridgedvotes/poller.go | 348 +++++ .../bridgedvotes/poller_integration_test.go | 264 ++++ internal/core/bridgedvotes/poller_test.go | 1171 +++++++++++++++++ internal/core/bridgedvotes/trustedhost.go | 68 + .../core/bridgedvotes/trustedhost_test.go | 84 ++ .../043_add_bridged_vote_poll_watermark.sql | 46 + .../db/postgres/admission_repo_schema_test.go | 2 + internal/db/postgres/bridged_votes_repo.go | 233 ++++ .../db/postgres/bridged_votes_repo_test.go | 651 +++++++++ ...uture_comment_created_at_migration_test.go | 2 + .../rematerialize_ledger_schema_test.go | 2 + .../vote_drift_recount_migration_test.go | 2 + 33 files changed, 4374 insertions(+), 34 deletions(-) create mode 100644 cmd/server/wiring_bridgedvotes_test.go create mode 100644 internal/config/config_bridgedvotes_test.go create mode 100644 internal/core/bridgedvotes/bridgedvotes.go create mode 100644 internal/core/bridgedvotes/client.go create mode 100644 internal/core/bridgedvotes/client_test.go create mode 100644 internal/core/bridgedvotes/harness_test.go create mode 100644 internal/core/bridgedvotes/normalizer.go create mode 100644 internal/core/bridgedvotes/normalizer_test.go create mode 100644 internal/core/bridgedvotes/poller.go create mode 100644 internal/core/bridgedvotes/poller_integration_test.go create mode 100644 internal/core/bridgedvotes/poller_test.go create mode 100644 internal/core/bridgedvotes/trustedhost.go create mode 100644 internal/core/bridgedvotes/trustedhost_test.go create mode 100644 internal/db/migrations/043_add_bridged_vote_poll_watermark.sql create mode 100644 internal/db/postgres/bridged_votes_repo.go create mode 100644 internal/db/postgres/bridged_votes_repo_test.go diff --git a/.env.prod.example b/.env.prod.example index c0e1e10..a5c8e12 100644 --- a/.env.prod.example +++ b/.env.prod.example @@ -138,6 +138,24 @@ CURSOR_SECRET=CHANGE_ME_CURSOR_SECRET # public scheme+host URL (comma-separated if multiple bridges are trusted). # TRUSTED_BRIDGE_PDS_HOSTS=https://tdpl.io +# Optional: bridged-vote poller tuning. The poller runs only when +# TRUSTED_BRIDGE_PDS_HOSTS is set: it sweeps the trusted bridges' +# getVoteAggregates side channel for content in bridged communities and folds +# the fediverse tallies into bridged vote counts (the channel that covers +# native-authored posts/comments, which record-stamped bridgedStats cannot +# reach). Each TRUSTED_BRIDGE_PDS_HOSTS entry must be scheme + host only +# (https://tdpl.io, optionally with a port); a path, credentials, query or a +# schemeless value fails the boot with a config error rather than silently +# never reaching that bridge. The interval must be positive (default 5m); +# there is no "disabled" value, leave TRUSTED_BRIDGE_PDS_HOSTS unset instead. +# Lookback (how far back by created_at content stays in rotation) and sweep +# cap default to the poller's own values (90 days / 2000) when unset or zero. +# The sweep cap is per matched bridge host: with several trusted bridges each +# gets max(cap / hosts, 100) subjects per sweep. +# BRIDGED_VOTE_POLL_INTERVAL=5m +# BRIDGED_VOTE_POLL_LOOKBACK=2160h +# BRIDGED_VOTE_POLL_SWEEP_CAP=2000 + # ============================================================================= # Jetstream Configuration (multi-feed, real-time event indexing) # ============================================================================= diff --git a/cmd/server/jobs.go b/cmd/server/jobs.go index 55abf43..cf39b65 100644 --- a/cmd/server/jobs.go +++ b/cmd/server/jobs.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "Coves/internal/core/bridgedvotes" "Coves/internal/core/posts" ) @@ -139,6 +140,76 @@ type acceptanceQueuePass interface { RunPass(ctx context.Context) (posts.PassReport, error) } +// bridgedVoteSweeper is the seam startBridgedVotePollJob drives: one poll cycle +// against the bridge's vote-aggregate side channel. *bridgedvotes.Poller +// satisfies it. +type bridgedVoteSweeper interface { + Sweep(ctx context.Context) (bridgedvotes.Report, error) +} + +// startBridgedVotePollJob runs the bridged-vote poller on an interval. Bridge +// outages are transient: a failed sweep is logged and the next tick retries. +// runGuarded deadline-bounds each sweep at the interval so a stalled bridge +// cannot permanently stop the rotation. +// +// The guard is defensive only: config.Validate rejects a non-positive interval +// and main.go checks the concrete poller for nil, so reaching the early return +// means one of those was bypassed, which is worth a line in the log. +func startBridgedVotePollJob(ctx context.Context, wg *sync.WaitGroup, poller bridgedVoteSweeper, interval time.Duration) { + if poller == nil || interval <= 0 { + slog.Warn("bridged vote poll job not started", + "poller_present", poller != nil, + "interval", interval, + ) + return + } + + runTicker(ctx, wg, "bridged-vote-poll", interval, func(ctx context.Context) { + report, err := poller.Sweep(ctx) + if err != nil { + // Cancellation alone is shutdown; joinSweepErrors guarantees a real + // fault joined with a canceled leaf does not classify as canceled. + if !errors.Is(err, context.Canceled) { + slog.Error("bridged vote poll sweep failed", + "error", err, + "matched_hosts", report.MatchedHosts, + "failed_hosts", report.FailedHosts, + "candidates", report.Candidates, + "applied", report.Applied, + "marked", report.Marked, + ) + } + return + } + + // Three quiet outcomes need telling apart. A sweep that found no + // bridged community at all is the normal state of a fresh instance + // and logs at debug. One that saw stored community hosts and matched + // none of them to the trust list is the misconfiguration this poller + // cannot otherwise surface: identity resolution stored one URL form, + // the operator configured another, and every sweep returns nil. + switch { + case report.MatchedHosts == 0 && report.StoredHosts > 0: + slog.Warn("bridged vote poll matched no community PDS URL to a trusted bridge host", + "trusted_hosts", report.TrustedHosts, + "stored_hosts", report.StoredHosts, + ) + case report.Candidates > 0 || report.PoisonMarked > 0: + slog.Info("bridged vote poll sweep completed", + "matched_hosts", report.MatchedHosts, + "candidates", report.Candidates, + "fetched", report.Fetched, + "applied", report.Applied, + "marked", report.Marked, + "poison_marked", report.PoisonMarked, + ) + default: + slog.Debug("bridged vote poll sweep found nothing to poll", + "matched_hosts", report.MatchedHosts) + } + }) +} + // startAcceptanceQueueJob walks the undecided admission backlog on an interval. // // It is the PULL half of admission (docs/PRD_AUTHOR_OWNED_POSTS.md §5.6). The diff --git a/cmd/server/jobs_test.go b/cmd/server/jobs_test.go index 37ea0ca..ae0e1e7 100644 --- a/cmd/server/jobs_test.go +++ b/cmd/server/jobs_test.go @@ -2,13 +2,17 @@ package main import ( "context" + "errors" "fmt" "sync" "sync/atomic" "testing" "time" + "Coves/internal/core/bridgedvotes" "Coves/tests/testkit" + + "github.com/stretchr/testify/require" ) // The reason recovery lives per cycle rather than around the whole goroutine: @@ -225,3 +229,100 @@ func TestRunGuarded_ContainsRuntimePanics(t *testing.T) { _ = b.n }) } + +type bridgedVoteSweeperFake struct { + calls atomic.Int64 + err error +} + +func (f *bridgedVoteSweeperFake) Sweep(context.Context) (bridgedvotes.Report, error) { + f.calls.Add(1) + return bridgedvotes.Report{}, f.err +} + +func TestStartBridgedVotePollJob_GuardsInvalidInputs(t *testing.T) { + tests := []struct { + name string + poller bridgedVoteSweeper + interval time.Duration + }{ + {name: "nil poller", interval: time.Second}, + {name: "zero interval", poller: &bridgedVoteSweeperFake{}}, + {name: "negative interval", poller: &bridgedVoteSweeperFake{}, interval: -time.Second}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + var wg sync.WaitGroup + startBridgedVotePollJob(context.Background(), &wg, test.poller, test.interval) + + done := waitForBridgedVoteJob(&wg) + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, time.Millisecond, "guarded job must not increment the WaitGroup") + if fake, ok := test.poller.(*bridgedVoteSweeperFake); ok { + require.Zero(t, fake.calls.Load()) + } + }) + } +} + +func TestStartBridgedVotePollJob_RunsRepeatedlyAndStopsOnCancel(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var wg sync.WaitGroup + fake := &bridgedVoteSweeperFake{} + + startBridgedVotePollJob(ctx, &wg, fake, 20*time.Millisecond) + require.Eventually(t, func() bool { + return fake.calls.Load() >= 2 + }, 2*time.Second, 5*time.Millisecond, "bridged vote Sweep must run repeatedly") + + cancel() + done := waitForBridgedVoteJob(&wg) + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, time.Millisecond, "bridged vote job must stop after cancellation") +} + +func TestStartBridgedVotePollJob_SweepErrorsDoNotKillJob(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var wg sync.WaitGroup + fake := &bridgedVoteSweeperFake{err: errors.New("bridge unavailable")} + + startBridgedVotePollJob(ctx, &wg, fake, 20*time.Millisecond) + require.Eventually(t, func() bool { + return fake.calls.Load() >= 2 + }, 2*time.Second, 5*time.Millisecond, "a Sweep error must not stop later ticker cycles") + + cancel() + done := waitForBridgedVoteJob(&wg) + require.Eventually(t, func() bool { + select { + case <-done: + return true + default: + return false + } + }, time.Second, time.Millisecond, "errored bridged vote job must still stop on cancellation") +} + +func waitForBridgedVoteJob(wg *sync.WaitGroup) <-chan struct{} { + done := make(chan struct{}) + go func() { + wg.Wait() + close(done) + }() + return done +} diff --git a/cmd/server/main.go b/cmd/server/main.go index 6d88924..fa59389 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -102,6 +102,13 @@ func run() error { startAcceptanceQueueJob(backgroundCtx, &backgroundWG, app.acceptanceQueue, cfg.Submissions.AcceptanceQueueInterval) } + // Like the acceptance driver above, this concrete pointer must be checked + // before conversion to bridgedVoteSweeper: a nil *Poller inside that + // interface would pass the job's nil guard and panic on its first sweep. + if app.bridgedVotePoller != nil { + startBridgedVotePollJob(backgroundCtx, &backgroundWG, + app.bridgedVotePoller, cfg.Instance.BridgedVotePollInterval) + } consumers, err := startConsumers(backgroundCtx, &backgroundWG, app) if err != nil { diff --git a/cmd/server/wiring.go b/cmd/server/wiring.go index bc82fcc..b108ff0 100644 --- a/cmd/server/wiring.go +++ b/cmd/server/wiring.go @@ -11,6 +11,7 @@ import ( "Coves/internal/core/aggregators" "Coves/internal/core/blobs" "Coves/internal/core/blueskypost" + "Coves/internal/core/bridgedvotes" "Coves/internal/core/comments" "Coves/internal/core/communities" "Coves/internal/core/communityFeeds" @@ -141,6 +142,9 @@ type application struct { revGate *jetstream.RevGate bridgeTrust *jetstream.BridgeTrust + // bridgedVotePoller is nil unless TRUSTED_BRIDGE_PDS_HOSTS is set. + bridgedVotePoller *bridgedvotes.Poller + // imageProxyHandler is nil when the image proxy is disabled. imageProxyHandler *imageproxyhandlers.Handler // stopImageProxyCleanup halts the disk cache eviction job. Never nil. @@ -186,6 +190,9 @@ func buildApplication(ctx context.Context, cfg *config.Config, db *sql.DB) (app return nil, err } app.buildJetstreamInfrastructure() + if err = app.buildBridgedVotePoller(); err != nil { + return nil, err + } return app, nil } @@ -870,19 +877,10 @@ func (a *application) buildJetstreamInfrastructure() { // Cursors let each consumer resume from its last processed event after a // restart instead of silently losing the gap; the dead letter queue // captures events that fail every in-line retry so the redriver can - // replay them once the underlying failure clears. + // replay them once the cause is fixed. a.jetstreamState = jetstream.NewPostgresStateStore(a.db) - - // The rev gate is the per-record ordering guard that makes it safe to run - // every consumer against multiple Jetstream feeds carrying the same repos - // (see rev_gate.go and migration 033). a.revGate = jetstream.NewRevGate(a.db) - // Provenance gate for bridge-asserted vote aggregates. Only repos hosted - // on a trusted bridge PDS may inflate their displayed counts via - // bridgedStats; every native repo is default-denied so it cannot - // self-assert them. Empty means bridgedStats are ignored everywhere, - // which is the right default for a deployment with no bridge. a.bridgeTrust = jetstream.NewBridgeTrust(a.cfg.Instance.TrustedBridgePDSHosts) if len(a.cfg.Instance.TrustedBridgePDSHosts) > 0 { slog.Info("bridgedStats provenance configured", @@ -891,3 +889,35 @@ func (a *application) buildJetstreamInfrastructure() { slog.Info("no trusted bridge PDS hosts configured; bridgedStats will be ignored") } } + +// buildBridgedVotePoller constructs the poller under the same non-empty guard +// as BridgeTrust. The trust list is both the authorization boundary for +// bridgedStats and the poller's only source of network destinations, so an +// unconfigured deployment must never turn community metadata in the database +// into outbound traffic. A bridge is third-party infrastructure, not the +// operator's own PDS, so it dials through the SSRF-guarded client like every +// other outbound fetch; the private-host hatch follows allowPrivateHosts. +func (a *application) buildBridgedVotePoller() error { + hosts := a.cfg.Instance.TrustedBridgePDSHosts + if len(hosts) == 0 { + return nil + } + + client := bridgedvotes.NewClient( + oauth.NewSSRFSafeHTTPClient(oauth.PrivateAddressOptions(a.allowPrivateHosts())...)) + poller, err := bridgedvotes.NewPoller( + postgresRepo.NewBridgedVotesRepository(a.db), client, hosts, bridgedvotes.Options{ + Lookback: a.cfg.Instance.BridgedVotePollLookback, + SweepCap: a.cfg.Instance.BridgedVotePollSweepCap, + }) + if err != nil { + // NewPoller's message already names the stage and the offending host. + return err + } + a.bridgedVotePoller = poller + slog.Info("bridged vote poller configured", + "trusted_bridge_hosts", len(hosts), + "interval", a.cfg.Instance.BridgedVotePollInterval, + ) + return nil +} diff --git a/cmd/server/wiring_bridgedvotes_test.go b/cmd/server/wiring_bridgedvotes_test.go new file mode 100644 index 0000000..d76456c --- /dev/null +++ b/cmd/server/wiring_bridgedvotes_test.go @@ -0,0 +1,54 @@ +package main + +import ( + "testing" + "time" + + "Coves/internal/config" + + "github.com/stretchr/testify/require" +) + +// No CI tier sets TRUSTED_BRIDGE_PDS_HOSTS, so without this test the poller's +// wiring branch is never executed anywhere: a nil poller, a typo in the host +// list, or a constructor that panics on a nil DB would all ship unnoticed. +// NewBridgedVotesRepository(nil) wraps a nil DB without touching it and +// NewPoller never dials, so no infrastructure is needed. +func TestBuildBridgedVotePoller(t *testing.T) { + t.Run("unset trust list leaves the poller nil", func(t *testing.T) { + a := &application{cfg: &config.Config{}} + require.NoError(t, a.buildBridgedVotePoller()) + require.Nil(t, a.bridgedVotePoller, "an unconfigured deployment must never turn community metadata into outbound traffic") + }) + + t.Run("configured trust list builds the poller", func(t *testing.T) { + cfg := &config.Config{} + cfg.Instance.TrustedBridgePDSHosts = []string{"https://bridge.example"} + cfg.Instance.BridgedVotePollInterval = 5 * time.Minute + a := &application{cfg: cfg} + require.NoError(t, a.buildBridgedVotePoller()) + require.NotNil(t, a.bridgedVotePoller) + }) + + t.Run("invalid trust entry is a wiring error naming the entry", func(t *testing.T) { + cfg := &config.Config{} + cfg.Instance.TrustedBridgePDSHosts = []string{"https://bridge.example/pds"} + a := &application{cfg: cfg} + err := a.buildBridgedVotePoller() + require.Error(t, err) + require.Contains(t, err.Error(), "https://bridge.example/pds") + require.Equal(t, 1, countOccurrences(err.Error(), "creating bridged vote poller"), + "the constructor's stage prefix must not be repeated by the caller") + require.Nil(t, a.bridgedVotePoller) + }) +} + +func countOccurrences(s, substr string) int { + count := 0 + for i := 0; i+len(substr) <= len(s); i++ { + if s[i:i+len(substr)] == substr { + count++ + } + } + return count +} diff --git a/docker-compose.prod.yml b/docker-compose.prod.yml index 2bbb38f..07120bf 100644 --- a/docker-compose.prod.yml +++ b/docker-compose.prod.yml @@ -133,6 +133,14 @@ services: # be accepted. Example: https://tdpl.io TRUSTED_BRIDGE_PDS_HOSTS: ${TRUSTED_BRIDGE_PDS_HOSTS:-} + # Bridged-vote poller tuning (active only when TRUSTED_BRIDGE_PDS_HOSTS is + # set). Passed through explicitly: a variable only present in .env is + # invisible to the container, so setting it there without these lines + # silently has no effect. + BRIDGED_VOTE_POLL_INTERVAL: ${BRIDGED_VOTE_POLL_INTERVAL:-} + BRIDGED_VOTE_POLL_LOOKBACK: ${BRIDGED_VOTE_POLL_LOOKBACK:-} + BRIDGED_VOTE_POLL_SWEEP_CAP: ${BRIDGED_VOTE_POLL_SWEEP_CAP:-} + # Image proxy configuration (on-the-fly resizing with disk cache). # The base URL is the hostname every Coves image URL is built from; it is # the media hostname, not the AppView apex, because only that hostname is diff --git a/internal/atproto/jetstream/bridge_trust.go b/internal/atproto/jetstream/bridge_trust.go index ec166f9..0d60d3c 100644 --- a/internal/atproto/jetstream/bridge_trust.go +++ b/internal/atproto/jetstream/bridge_trust.go @@ -1,18 +1,15 @@ package jetstream import ( + "Coves/internal/core/bridgedvotes" "log" - "net/url" - "strings" "time" ) -// maxBridgedCount mirrors the tidepool bridge's own MaxSeededCount ceiling -// (1,000,000). A bridgedStats aggregate asserting a count above this is treated as -// malformed/hostile and the entire aggregate is ignored — it is far larger than any -// plausible origin-platform post score and is the shape a score-inflation attack -// takes. Keep this in sync with the bridge if it ever raises MaxSeededCount. -const maxBridgedCount = 1_000_000 +// maxBridgedCount aliases the shared ceiling both ingestion channels enforce on a +// bridge-asserted count; see bridgedvotes.MaxBridgedCount for the rationale and +// the tidepool constant it chases. +const maxBridgedCount = bridgedvotes.MaxBridgedCount // BridgeTrust is the provenance gate for bridge-asserted vote aggregates // (bridgedStats). bridgedStats let a record declare origin-platform vote counts that @@ -73,20 +70,11 @@ func (b *BridgeTrust) TrustsPDS(pdsURL string) bool { return ok } -// normalizePDSHost reduces a PDS URL to a stable scheme+host comparison key so that -// "https://Bridge.Example/", "https://bridge.example" and "https://bridge.example:443" -// compare consistently. Values that do not parse as a URL with a host fall back to the -// trimmed, lowercased, trailing-slash-stripped string so a plain host in config still -// matches an identically-formatted stored value. +// normalizePDSHost delegates to bridgedvotes.NormalizeHost, the shared trust and +// polling comparison rule. It lowercases scheme+host, removes matching default +// HTTP(S) ports and tolerantly normalizes schemeless stored values. func normalizePDSHost(raw string) string { - s := strings.TrimSpace(raw) - if s == "" { - return "" - } - if u, err := url.Parse(s); err == nil && u.Host != "" { - return strings.ToLower(u.Scheme + "://" + u.Host) - } - return strings.ToLower(strings.TrimRight(s, "/")) + return bridgedvotes.NormalizeHost(raw) } // validatedBridgedStats applies input hygiene to a bridgedStats aggregate and parses diff --git a/internal/atproto/jetstream/community_origin_test.go b/internal/atproto/jetstream/community_origin_test.go index a9804b1..0b55d26 100644 --- a/internal/atproto/jetstream/community_origin_test.go +++ b/internal/atproto/jetstream/community_origin_test.go @@ -41,6 +41,13 @@ const ( func trustingBridge() *BridgeTrust { return NewBridgeTrust([]string{trustedBridgePDS}) } +func TestBridgeTrust_NormalizesDefaultHTTPSPort(t *testing.T) { + t.Parallel() + + trust := NewBridgeTrust([]string{"https://bridge.example"}) + require.True(t, trust.TrustsPDS("https://Bridge.Example:443/")) +} + func TestAdmitCommunityOrigin(t *testing.T) { t.Parallel() diff --git a/internal/atproto/jetstream/post_consumer.go b/internal/atproto/jetstream/post_consumer.go index e9d740f..75e79e4 100644 --- a/internal/atproto/jetstream/post_consumer.go +++ b/internal/atproto/jetstream/post_consumer.go @@ -2,6 +2,7 @@ package jetstream import ( "Coves/internal/atproto/identity" + "Coves/internal/core/bridgedvotes" "Coves/internal/core/communities" "Coves/internal/core/posts" "Coves/internal/core/richtext" @@ -417,9 +418,13 @@ func (c *PostEventConsumer) applyPostContentUpdate(ctx context.Context, in postC // parseBridgedAsOf parses a bridgedStats.asOf timestamp, logging (and returning the // error) on failure so callers can decide to skip applying the aggregate. func parseBridgedAsOf(asOf, uri string) (time.Time, error) { - t, err := time.Parse(time.RFC3339, asOf) + // bridgedvotes.ParseAsOf is the shared rule for both ingestion channels: it + // rejects the zero time and any stamp more than MaxAsOfSkew ahead of this + // clock, because a far-future asOf would win the >= guard once and then make + // every later honest aggregate lose it, from either channel, until repaired. + t, err := bridgedvotes.ParseAsOf(asOf, time.Now()) if err != nil { - log.Printf("Warning: failed to parse bridgedStats.asOf %q for %s: %v", asOf, uri, err) + log.Printf("Warning: rejecting bridgedStats.asOf %q for %s: %v", asOf, uri, err) return time.Time{}, err } return t, nil diff --git a/internal/atproto/jetstream/postv2_mechanisms_test.go b/internal/atproto/jetstream/postv2_mechanisms_test.go index 25c1554..998bc5a 100644 --- a/internal/atproto/jetstream/postv2_mechanisms_test.go +++ b/internal/atproto/jetstream/postv2_mechanisms_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "Coves/internal/core/bridgedvotes" "Coves/internal/core/posts" "Coves/internal/db/postgres" "Coves/tests/testkit" @@ -489,6 +490,8 @@ func TestPostV2Mechanisms_BridgedStats_InvalidAggregateIgnoredWhole(t *testing.T "negative count": bridgedStatsRecord(-1, 9, asOfEarly), "absurd count": bridgedStatsRecord(maxBridgedCount+1, 9, asOfEarly), "invalid asOf": bridgedStatsRecord(9, 4, "not-a-timestamp"), + "zero asOf": bridgedStatsRecord(9, 4, "0001-01-01T00:00:00Z"), + "future asOf": bridgedStatsRecord(9, 4, time.Now().Add(bridgedvotes.MaxAsOfSkew+time.Hour).UTC().Format(time.RFC3339)), } for name, stats := range cases { t.Run(name, func(t *testing.T) { diff --git a/internal/config/config.go b/internal/config/config.go index e3eebd9..f5405d0 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "Coves/internal/core/bridgedvotes" "Coves/internal/core/imageproxy" ) @@ -267,6 +268,23 @@ type InstanceConfig struct { // community consumer. Dev-only: it is what stops a community record from // claiming to be hosted by a domain it does not control. SkipDIDWebVerification bool + + // BridgedVotePollInterval is how often the bridged-vote poller sweeps the + // trusted bridges' vote-aggregate side channel. It must be positive: unlike + // its two siblings below, zero does not mean "poller default", and the + // poller has no disabled state of its own — an instance that should not + // poll leaves TRUSTED_BRIDGE_PDS_HOSTS unset. Validate enforces this so a + // typo fails the boot instead of leaving a job that logs "configured" and + // never runs. + BridgedVotePollInterval time.Duration + + // BridgedVotePollLookback bounds sweep candidates by created_at. Zero means + // the poller's own default applies (single source of truth for the value). + BridgedVotePollLookback time.Duration + + // BridgedVotePollSweepCap bounds subjects polled per sweep. Zero means the + // poller's own default applies. + BridgedVotePollSweepCap int } // PDSConfig holds the settings for this instance's own PDS account, used to @@ -788,6 +806,18 @@ func (c *Config) loadInstance() error { if err != nil { return err } + bridgedVotePollInterval, err := durationVar("BRIDGED_VOTE_POLL_INTERVAL", 5*time.Minute) + if err != nil { + return err + } + bridgedVotePollLookback, err := durationVar("BRIDGED_VOTE_POLL_LOOKBACK", 0) + if err != nil { + return err + } + bridgedVotePollSweepCap, err := intVar("BRIDGED_VOTE_POLL_SWEEP_CAP", 0) + if err != nil { + return err + } did := stringVar("INSTANCE_DID", "did:web:coves.social") @@ -807,6 +837,9 @@ func (c *Config) loadInstance() error { AllowedCommunityCreators: csvVar("COMMUNITY_CREATORS"), TrustedBridgePDSHosts: csvVar("TRUSTED_BRIDGE_PDS_HOSTS"), SkipDIDWebVerification: skipDIDWeb, + BridgedVotePollInterval: bridgedVotePollInterval, + BridgedVotePollLookback: bridgedVotePollLookback, + BridgedVotePollSweepCap: bridgedVotePollSweepCap, } return nil } @@ -1023,6 +1056,37 @@ func (c *Config) Validate() error { "ACCEPTANCE_QUEUE_INTERVAL cannot be negative (got %s); use 0 to disable the acceptance queue driver", c.Submissions.AcceptanceQueueInterval)) } + // The bridged-vote poller. Trusted hosts are validated whether or not the + // poller will run, because the same list is BridgeTrust's provenance gate + // for record-stamped bridgedStats: a value BridgeTrust would tolerate but + // the poller would refuse must fail here, where it reads as a config + // error, not inside wiring where it reads as a crash. + for _, host := range c.Instance.TrustedBridgePDSHosts { + if _, err := bridgedvotes.ParseTrustedHost(host); err != nil { + problems = append(problems, fmt.Sprintf( + "TRUSTED_BRIDGE_PDS_HOSTS: %v (scheme + host only, e.g. https://tdpl.io)", err)) + } + } + // The interval is held to its rule only when the poller will actually run. + // A hand-assembled Config with no trust list has no poller to misconfigure, + // and Load always supplies the 5-minute default, so reaching this with a + // trust list set means an operator wrote a value that is not an interval. + if len(c.Instance.TrustedBridgePDSHosts) > 0 && c.Instance.BridgedVotePollInterval <= 0 { + problems = append(problems, fmt.Sprintf( + "BRIDGED_VOTE_POLL_INTERVAL must be greater than 0 (got %s); "+ + "the poller has no disabled state — leave TRUSTED_BRIDGE_PDS_HOSTS unset to run without it", + c.Instance.BridgedVotePollInterval)) + } + if c.Instance.BridgedVotePollLookback < 0 { + problems = append(problems, fmt.Sprintf( + "BRIDGED_VOTE_POLL_LOOKBACK cannot be negative (got %s); use 0 for the poller's default", + c.Instance.BridgedVotePollLookback)) + } + if c.Instance.BridgedVotePollSweepCap < 0 { + problems = append(problems, fmt.Sprintf( + "BRIDGED_VOTE_POLL_SWEEP_CAP cannot be negative (got %d); use 0 for the poller's default", + c.Instance.BridgedVotePollSweepCap)) + } // REDRIVE_INTERVAL is checked in loadJetstream instead of here; see the note // there for why the environment's value and a hand-assembled Config cannot be // held to the same rule. diff --git a/internal/config/config_bridgedvotes_test.go b/internal/config/config_bridgedvotes_test.go new file mode 100644 index 0000000..813653b --- /dev/null +++ b/internal/config/config_bridgedvotes_test.go @@ -0,0 +1,120 @@ +package config + +import ( + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestBridgedVotePollConfigDefaults(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, 5*time.Minute, cfg.Instance.BridgedVotePollInterval) + require.Zero(t, cfg.Instance.BridgedVotePollLookback, + "zero delegates the lookback default to the poller") + require.Zero(t, cfg.Instance.BridgedVotePollSweepCap, + "zero delegates the sweep-cap default to the poller") +} + +func TestBridgedVotePollConfigParsesExplicitValues(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("BRIDGED_VOTE_POLL_INTERVAL", "17s") + t.Setenv("BRIDGED_VOTE_POLL_LOOKBACK", "72h") + t.Setenv("BRIDGED_VOTE_POLL_SWEEP_CAP", "321") + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, 17*time.Second, cfg.Instance.BridgedVotePollInterval) + require.Equal(t, 72*time.Hour, cfg.Instance.BridgedVotePollLookback) + require.Equal(t, 321, cfg.Instance.BridgedVotePollSweepCap) +} + +func TestBridgedVotePollConfigRejectsInvalidInterval(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("BRIDGED_VOTE_POLL_INTERVAL", "not-a-duration") + + _, err := Load() + require.Error(t, err) + require.Contains(t, err.Error(), "BRIDGED_VOTE_POLL_INTERVAL") +} + +func TestBridgedVotePollConfigRejectsNonPositiveInterval(t *testing.T) { + for _, value := range []string{"0s", "-5m"} { + t.Run(value, func(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("TRUSTED_BRIDGE_PDS_HOSTS", "https://tdpl.io") + t.Setenv("BRIDGED_VOTE_POLL_INTERVAL", value) + + _, err := Load() + require.Error(t, err, "zero is not a disable switch for this job; a typo must fail the boot") + require.Contains(t, err.Error(), "BRIDGED_VOTE_POLL_INTERVAL") + }) + } +} + +func TestBridgedVotePollConfigRejectsNegativeTuning(t *testing.T) { + t.Run("lookback", func(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("BRIDGED_VOTE_POLL_LOOKBACK", "-1h") + + _, err := Load() + require.Error(t, err) + require.Contains(t, err.Error(), "BRIDGED_VOTE_POLL_LOOKBACK") + }) + t.Run("sweep cap", func(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("BRIDGED_VOTE_POLL_SWEEP_CAP", "-1") + + _, err := Load() + require.Error(t, err) + require.Contains(t, err.Error(), "BRIDGED_VOTE_POLL_SWEEP_CAP") + }) +} + +func TestTrustedBridgePDSHostsValidatedAtLoad(t *testing.T) { + t.Run("scheme and host accepted", func(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("TRUSTED_BRIDGE_PDS_HOSTS", "https://tdpl.io, https://bridge.example:8443/") + + cfg, err := Load() + require.NoError(t, err) + require.Equal(t, []string{"https://tdpl.io", "https://bridge.example:8443/"}, cfg.Instance.TrustedBridgePDSHosts) + }) + + invalid := map[string]string{ + "schemeless (BridgeTrust used to tolerate it)": "tdpl.io", + "path": "https://tdpl.io/pds", + "credentials": "https://user:secret@tdpl.io", + "one bad entry among good ones": "https://tdpl.io,ftp://bridge.example", + } + for name, value := range invalid { + t.Run(name, func(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("TRUSTED_BRIDGE_PDS_HOSTS", value) + + _, err := Load() + require.Error(t, err, "the same list gates bridgedStats provenance, so a value one consumer would refuse must fail as config") + require.Contains(t, err.Error(), "TRUSTED_BRIDGE_PDS_HOSTS") + }) + } +} + +func TestBridgedVotePollIntervalIsNotValidatedWithoutTrustedHosts(t *testing.T) { + clearEnv(t) + t.Setenv("IS_DEV_ENV", "true") + t.Setenv("BRIDGED_VOTE_POLL_INTERVAL", "0s") + + _, err := Load() + require.NoError(t, err, "with no trust list there is no poller to misconfigure") +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 693d201..2198079 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -650,7 +650,7 @@ func TestLoad_CSVListsAreTrimmed(t *testing.T) { clearEnv(t) prodEnv(t) t.Setenv("COMMUNITY_CREATORS", " did:plc:one , did:plc:two ,, ") - t.Setenv("TRUSTED_BRIDGE_PDS_HOSTS", "bridge.example.com") + t.Setenv("TRUSTED_BRIDGE_PDS_HOSTS", "https://bridge.example.com") cfg, err := Load() if err != nil { diff --git a/internal/config/testing.go b/internal/config/testing.go index 5dc3e51..71f2326 100644 --- a/internal/config/testing.go +++ b/internal/config/testing.go @@ -18,6 +18,7 @@ var loadedEnvVars = []string{ "OAUTH_CLIENT_PRIVATE_KEY", "OAUTH_CLIENT_KEY_ID", "INSTANCE_DID", "INSTANCE_DOMAIN", "COMMUNITY_CREATORS", "TRUSTED_BRIDGE_PDS_HOSTS", "SKIP_DID_WEB_VERIFICATION", + "BRIDGED_VOTE_POLL_INTERVAL", "BRIDGED_VOTE_POLL_LOOKBACK", "BRIDGED_VOTE_POLL_SWEEP_CAP", "PDS_URL", "PDS_INSTANCE_HANDLE", "PDS_INSTANCE_PASSWORD", "PDS_ADMIN_PASSWORD", "JETSTREAM_FEEDS", "REDRIVE_INTERVAL", "CURSOR_SECRET", diff --git a/internal/core/bridgedvotes/bridgedvotes.go b/internal/core/bridgedvotes/bridgedvotes.go new file mode 100644 index 0000000..d13e039 --- /dev/null +++ b/internal/core/bridgedvotes/bridgedvotes.go @@ -0,0 +1,131 @@ +// Package bridgedvotes polls the Tidepool bridge's vote-aggregate side channel +// (social.coves.bridge.getVoteAggregates) for content in bridged communities and +// folds the returned fediverse tallies into the bridged_* columns on posts and +// comments. It exists because the record-stamp channel (bridgedStats via +// Jetstream) cannot reach native-authored content: the bridge cannot write into +// a native user's own PDS repo, so without this poller votes cast on Lemmy +// against native posts/comments are aggregated by the bridge and then displayed +// by nobody. +package bridgedvotes + +import ( + "context" + "errors" + "fmt" + "time" +) + +const ( + // MaxBridgedCount is the largest per-direction count either ingestion + // channel accepts from a bridge. Anything above it is malformed or hostile: + // it is far larger than any plausible origin-platform score and is the shape + // a score-inflation attack takes. Keep it in sync with tidepool's + // MaxSeededCount; the Jetstream adapter aliases this constant rather than + // carrying its own copy. + MaxBridgedCount = 1_000_000 + + // MaxAsOfSkew bounds how far ahead of this AppView's clock a bridge's asOf + // stamp may run. A stamp past that is a clock fault or a hostile value, and + // accepting it would install a bridged_stats_as_of that every later honest + // aggregate loses the >= guard against, in both ingestion channels, until a + // repair migration ran. Five minutes tolerates ordinary NTP drift. + MaxAsOfSkew = 5 * time.Minute +) + +// ErrMissingAsOf is returned by a Store asked to apply an aggregate whose AsOf +// is the zero time. Counts and their sampling instant are one atomic trio, so +// an unstamped tally is a caller bug rather than a benign no-op. +var ErrMissingAsOf = errors.New("bridged vote aggregate has no asOf") + +// Aggregate is one subject's fediverse-only vote tally as served by the bridge. +type Aggregate struct { + URI string + Upvotes int + Downvotes int + AsOf time.Time +} + +// Candidate is one pollable subject: its at-uri and the community's stored PDS +// URL it was matched under. StoredPDSURL is a MATCH KEY, never a dial target. +// The dial target is always a TrustedHost, which a stored string cannot become +// without passing ParseTrustedHost, the same validation operator config does. +type Candidate struct { + URI string + StoredPDSURL string +} + +// Store is the persistence seam the poller sweeps through. +type Store interface { + // SelectCandidates returns pollable subjects (posts and comments) in + // communities whose pds_url is one of storedHosts, non-deleted, created + // within lookback, ordered bridged_polled_at ASC NULLS FIRST, capped at limit. + // Comments qualify only through an indexed, non-deleted root post — that + // join is their sole community source, so a deleted or unindexed root + // excludes its comments. + SelectCandidates(ctx context.Context, storedHosts []string, lookback time.Duration, limit int) ([]Candidate, error) + // DistinctCommunityPDSURLs returns every distinct non-empty communities.pds_url. + DistinctCommunityPDSURLs(ctx context.Context) ([]string, error) + // ApplyAggregate folds one aggregate into its row under the asOf >= guard, + // recomputing score in the same statement. Deleted/absent rows are a no-op + // success; a zero AsOf is ErrMissingAsOf. + ApplyAggregate(ctx context.Context, agg Aggregate) error + // MarkPolled advances bridged_polled_at for every named uri (posts and comments). + MarkPolled(ctx context.Context, uris []string) error +} + +// Options tunes a Poller. +type Options struct { + // Lookback bounds candidate selection by created_at. Non-positive takes the default. + Lookback time.Duration + // SweepCap bounds subjects selected per sweep for a single matched host. + // With several matched hosts each receives max(SweepCap/hosts, 100), so the + // per-sweep total can exceed SweepCap by up to 100 per host when the cap is + // small. Non-positive takes the default. + SweepCap int +} + +// Report counts what one Sweep did. It exists so the job can log a working +// poller, an idle one, and a misconfigured one differently: a sweep that +// returns nil and nothing else looks identical whether it folded a thousand +// tallies or matched no community at all. +type Report struct { + // TrustedHosts is how many operator-configured bridge hosts the poller holds. + TrustedHosts int + // StoredHosts is how many distinct community PDS URLs the store reported. + StoredHosts int + // MatchedHosts is how many trusted hosts at least one stored URL matched. + MatchedHosts int + // Candidates is how many subjects were selected for polling. + Candidates int + // Fetched is how many aggregates the bridges returned and the client accepted. + Fetched int + // Applied is how many accepted aggregates the store was asked to fold. + Applied int + // Marked is how many subjects advanced their poll watermark. + Marked int + // PoisonMarked is how many of Marked advanced because their batch failed + // permanently or exhausted its transient-failure allowance, not because + // it was fetched. + PoisonMarked int + // FailedHosts is how many matched hosts ended the sweep with a fetch or + // selection failure. + FailedHosts int +} + +// ParseAsOf parses a bridge's RFC 3339 asOf stamp and applies the hygiene both +// ingestion channels share: the zero time is rejected (time.Parse accepts +// "0001-01-01T00:00:00Z", and a zero stamp would defeat the >= guard) and so +// is a stamp more than MaxAsOfSkew ahead of now. +func ParseAsOf(raw string, now time.Time) (time.Time, error) { + t, err := time.Parse(time.RFC3339, raw) + if err != nil { + return time.Time{}, err + } + if t.IsZero() { + return time.Time{}, errors.New("asOf is the zero time") + } + if t.After(now.Add(MaxAsOfSkew)) { + return time.Time{}, fmt.Errorf("asOf %s is more than %s ahead of the AppView clock", t.UTC().Format(time.RFC3339), MaxAsOfSkew) + } + return t, nil +} diff --git a/internal/core/bridgedvotes/client.go b/internal/core/bridgedvotes/client.go new file mode 100644 index 0000000..9e14269 --- /dev/null +++ b/internal/core/bridgedvotes/client.go @@ -0,0 +1,261 @@ +package bridgedvotes + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "net/http" + "time" + + covesoauth "Coves/internal/atproto/oauth" +) + +const ( + maxAggregateBatch = 100 + // One hundred aggregates occupy tens of kilobytes. A misbehaving host must + // not stream unbounded JSON into a long-lived background job. + maxResponseBytes = 1 << 20 + // clientTimeout is the per-request budget. A full default sweep is twenty + // batches, so this must leave the 5-minute cycle deadline in + // startBridgedVotePollJob real headroom: 20 × 10 s = 200 s. + clientTimeout = 10 * time.Second + // keepAliveDrainBytes bounds the read that returns a connection to the + // keep-alive pool after an error: enough for an ordinary error body, far + // too little for an adversarial host to stream forever. + keepAliveDrainBytes = 4096 + aggregatesPath = "/xrpc/social.coves.bridge.getVoteAggregates" +) + +// Client fetches vote aggregates from a bridge host. +type Client struct { + httpClient *http.Client +} + +// NewClient builds a Client. A nil httpClient uses the SSRF-guarded default: +// a trusted bridge is third-party infrastructure, not the operator's own PDS, +// so its DNS answers are vetted like every other outbound fetch in this +// codebase. Callers that need the dev-only private-host hatch build the +// guarded client themselves with oauth.PrivateAddressOptions and pass it in. +func NewClient(httpClient *http.Client) *Client { + if httpClient == nil { + httpClient = covesoauth.NewSSRFSafeHTTPClient() + } + + // Clone caller-owned clients rather than mutating shared policy. Redirects + // are forbidden because a compromised trusted bridge must not be able to + // 302 the poller into an internal service, escaping the operator-configured + // dial-target invariant; the resulting 3xx is handled as a permanent non-200 + // contract failure. The timeout is set unconditionally so the sweep's time + // budget does not depend on whichever client was handed in. + client := *httpClient + client.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + client.Timeout = clientTimeout + return &Client{httpClient: &client} +} + +type transientError struct { + err error +} + +func (e *transientError) Error() string { + return e.err.Error() +} + +func (e *transientError) Unwrap() error { + return e.err +} + +// IsTransient reports whether err is a fetch fault the next sweep may not see +// again (rate limit, server error, transport failure) as opposed to a contract +// violation. "Transient" here means "do not poison-mark the batch past the +// rotation", not "retry now": the poller never retries within a sweep, which is +// also why the PDS client's choice not to retry 429 is no contradiction. +func IsTransient(err error) bool { + var transient *transientError + return errors.As(err, &transient) +} + +func isTransientStatus(status int) bool { + switch status { + case http.StatusRequestTimeout, http.StatusTooEarly, http.StatusTooManyRequests: + return true + } + return status >= http.StatusInternalServerError && status < 600 +} + +// drainForReuse reads a bounded tail of an errored body so the connection can +// return to the keep-alive pool. The count and error are irrelevant: every +// caller is already returning a primary error. +func drainForReuse(body io.Reader) { + _, _ = io.CopyN(io.Discard, body, keepAliveDrainBytes) +} + +type rawAggregate struct { + URI string `json:"uri"` + Upvotes int `json:"upvotes"` + Downvotes int `json:"downvotes"` + UpdatedAt string `json:"updatedAt"` +} + +// GetVoteAggregates fetches aggregates for up to 100 uris from host. The host +// parameter's type is the vetting: a TrustedHost exists only by way of +// ParseTrustedHost, so no stored community URL can reach this dial. +func (c *Client) GetVoteAggregates(ctx context.Context, host TrustedHost, uris []string) ([]Aggregate, error) { + if host.IsZero() { + return nil, errors.New("get vote aggregates: host is not a parsed trusted host") + } + if len(uris) > maxAggregateBatch { + return nil, fmt.Errorf("get vote aggregates from %q: batch contains %d URIs; maximum is %d", host, len(uris), maxAggregateBatch) + } + if len(uris) == 0 { + return nil, nil + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, host.String()+aggregatesPath, nil) + if err != nil { + return nil, fmt.Errorf("build vote aggregate request for %q: %w", host, err) + } + query := req.URL.Query() + for _, uri := range uris { + query.Add("uris", uri) + } + req.URL.RawQuery = query.Encode() + + resp, err := c.httpClient.Do(req) + if err != nil { + if errors.Is(err, covesoauth.ErrResponseTooLarge) { + // The guarded transport refused a declared Content-Length above its + // cap. That is the bridge violating the contract, not the network. + return nil, fmt.Errorf("get vote aggregates from %q: %w", host, err) + } + return nil, fmt.Errorf("get vote aggregates from %q: %w", host, &transientError{err: err}) + } + defer func() { + drainForReuse(resp.Body) + _ = resp.Body.Close() + }() + + if resp.StatusCode != http.StatusOK { + statusErr := fmt.Errorf("HTTP status %d", resp.StatusCode) + if isTransientStatus(resp.StatusCode) { + return nil, fmt.Errorf("get vote aggregates from %q: %w", host, &transientError{err: statusErr}) + } + return nil, fmt.Errorf("get vote aggregates from %q: %w", host, statusErr) + } + + body, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + if err != nil { + if errors.Is(err, covesoauth.ErrResponseTooLarge) { + return nil, fmt.Errorf("decode vote aggregates from %q: %w", host, err) + } + // A truncated body or HTTP/2 reset is a transport fault worth retrying; + // treating it as a permanent contract failure would poison-mark healthy + // subjects past the rotation without ever receiving a complete response. + return nil, fmt.Errorf("decode vote aggregates from %q: %w", host, &transientError{err: err}) + } + if len(body) > maxResponseBytes { + return nil, fmt.Errorf("decode vote aggregates from %q: response exceeds %d byte size cap", host, maxResponseBytes) + } + + // The pointer distinguishes an absent "aggregates" key from an empty list. + // An empty list is a bridge that knows nothing about these subjects; a + // missing key is a bridge speaking a different contract, and accepting it + // would silently mark every subject polled while landing no data. + var payload struct { + Aggregates *[]rawAggregate `json:"aggregates"` + } + if err := json.Unmarshal(body, &payload); err != nil { + return nil, fmt.Errorf("decode vote aggregates from %q: %w", host, err) + } + if payload.Aggregates == nil { + return nil, fmt.Errorf("decode vote aggregates from %q: response has no aggregates field", host) + } + + return c.acceptAggregates(host, uris, *payload.Aggregates) +} + +// acceptAggregates applies the response contract entry by entry. A trusted +// bridge may answer only about subjects named in the request — without that +// binding one compromised bridge could rewrite bridged counts for any content +// platform-wide — and first-wins deduplication bounds DB writes to at most one +// per requested subject per response. Counts and their stamp are one atomic +// trio, mirroring validatedBridgedStats in the Jetstream adapter, so one +// malformed member drops the entire entry. +func (c *Client) acceptAggregates(host TrustedHost, uris []string, entries []rawAggregate) ([]Aggregate, error) { + requested := make(map[string]struct{}, len(uris)) + for _, uri := range uris { + requested[uri] = struct{}{} + } + accepted := make(map[string]struct{}, len(entries)) + aggregates := make([]Aggregate, 0, len(entries)) + now := time.Now() + + var droppedInvalid, droppedUnrequested, droppedDuplicate int + var sampleURI, sampleReason, sampleUpdatedAt string + sample := func(entry rawAggregate, reason string) { + if sampleReason != "" { + return + } + sampleURI, sampleReason, sampleUpdatedAt = entry.URI, reason, entry.UpdatedAt + } + + for _, entry := range entries { + if _, ok := requested[entry.URI]; !ok { + droppedUnrequested++ + sample(entry, "unrequested") + continue + } + if _, ok := accepted[entry.URI]; ok { + droppedDuplicate++ + sample(entry, "duplicate") + continue + } + asOf, err := ParseAsOf(entry.UpdatedAt, now) + if err != nil { + droppedInvalid++ + sample(entry, "invalid updatedAt: "+err.Error()) + continue + } + if entry.Upvotes < 0 || entry.Downvotes < 0 || + entry.Upvotes > MaxBridgedCount || entry.Downvotes > MaxBridgedCount { + droppedInvalid++ + sample(entry, "count out of range") + continue + } + accepted[entry.URI] = struct{}{} + aggregates = append(aggregates, Aggregate{ + URI: entry.URI, + Upvotes: entry.Upvotes, + Downvotes: entry.Downvotes, + AsOf: asOf, + }) + } + + dropped := droppedInvalid + droppedUnrequested + droppedDuplicate + if len(entries) > 0 && len(aggregates) == 0 { + // A response that answers and gets every entry wrong is the same + // contract break as a malformed body, and takes the same poison-mark + // path. Handling it as a Warn-only partial success would advance + // watermarks over healthy subjects with no data landing and no error. + return nil, fmt.Errorf("decode vote aggregates from %q: all %d entries rejected (invalid=%d unrequested=%d duplicate=%d; first: %s %s updatedAt=%q)", + host, len(entries), droppedInvalid, droppedUnrequested, droppedDuplicate, sampleURI, sampleReason, sampleUpdatedAt) + } + if dropped > 0 { + slog.Warn("bridged vote response entries dropped", + "host", host.String(), + "invalid", droppedInvalid, + "unrequested", droppedUnrequested, + "duplicates", droppedDuplicate, + "sample_uri", sampleURI, + "sample_reason", sampleReason, + "sample_updated_at", sampleUpdatedAt, + ) + } + + return aggregates, nil +} diff --git a/internal/core/bridgedvotes/client_test.go b/internal/core/bridgedvotes/client_test.go new file mode 100644 index 0000000..10e82d8 --- /dev/null +++ b/internal/core/bridgedvotes/client_test.go @@ -0,0 +1,527 @@ +package bridgedvotes_test + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "Coves/internal/core/bridgedvotes" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const clientVoteAggregatesPath = "/xrpc/social.coves.bridge.getVoteAggregates" + +func TestClientGetVoteAggregatesRequestAndHappyPath(t *testing.T) { + t.Parallel() + + inputURIs := []string{"at://a", "at://b"} + updatedAt := "2026-08-31T02:04:01.080Z" + expectedAsOf, err := time.Parse(time.RFC3339Nano, updatedAt) + require.NoError(t, err) + + type recordedRequest struct { + method string + path string + uris []string + } + var ( + mu sync.Mutex + recorded []recordedRequest + ) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + mu.Lock() + recorded = append(recorded, recordedRequest{ + method: r.Method, + path: r.URL.Path, + uris: decodeClientURIs(r.URL.Query()["uris"]), + }) + mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "aggregates": []map[string]any{{ + "uri": "at://a", "upvotes": 2, "downvotes": 1, "updatedAt": updatedAt, + }}, + }); err != nil { + t.Errorf("encode aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates(context.Background(), trustedHost(t, server.URL), inputURIs) + require.NoError(t, err) + + mu.Lock() + requests := append([]recordedRequest(nil), recorded...) + mu.Unlock() + require.Len(t, requests, 1, "the client must issue one XRPC request") + require.Equal(t, http.MethodGet, requests[0].method) + require.Equal(t, clientVoteAggregatesPath, requests[0].path) + require.ElementsMatch(t, inputURIs, requests[0].uris) + require.Equal(t, []bridgedvotes.Aggregate{{ + URI: "at://a", Upvotes: 2, Downvotes: 1, AsOf: expectedAsOf, + }}, got, "well-formed but omitted URIs are absent without making the request fail") +} + +func TestClientGetVoteAggregatesDropsInvalidCounts(t *testing.T) { + t.Parallel() + + const updatedAt = "2026-08-31T02:04:01.080Z" + expectedAsOf, err := time.Parse(time.RFC3339Nano, updatedAt) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "aggregates": []map[string]any{ + {"uri": "at://valid", "upvotes": 2, "downvotes": 1, "updatedAt": updatedAt}, + {"uri": "at://negative-up", "upvotes": -1, "downvotes": 0, "updatedAt": updatedAt}, + {"uri": "at://negative-down", "upvotes": 0, "downvotes": -1, "updatedAt": updatedAt}, + {"uri": "at://excess-up", "upvotes": 1_000_001, "downvotes": 0, "updatedAt": updatedAt}, + {"uri": "at://excess-down", "upvotes": 0, "downvotes": 1_000_001, "updatedAt": updatedAt}, + }, + }); err != nil { + t.Errorf("encode aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), + []string{"at://valid", "at://negative-up", "at://negative-down", "at://excess-up", "at://excess-down"}, + ) + require.NoError(t, err) + require.Equal(t, []bridgedvotes.Aggregate{{ + URI: "at://valid", Upvotes: 2, Downvotes: 1, AsOf: expectedAsOf, + }}, got) +} + +func TestClientGetVoteAggregatesDropsInvalidAsOf(t *testing.T) { + t.Parallel() + + const validUpdatedAt = "2026-08-31T02:04:01.080Z" + expectedAsOf, err := time.Parse(time.RFC3339Nano, validUpdatedAt) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "aggregates": []map[string]any{ + {"uri": "at://invalid-as-of", "upvotes": 4, "downvotes": 1, "updatedAt": "not-a-time"}, + {"uri": "at://valid", "upvotes": 2, "downvotes": 1, "updatedAt": validUpdatedAt}, + }, + }); err != nil { + t.Errorf("encode aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://invalid-as-of", "at://valid"}, + ) + require.NoError(t, err) + require.Equal(t, []bridgedvotes.Aggregate{{ + URI: "at://valid", Upvotes: 2, Downvotes: 1, AsOf: expectedAsOf, + }}, got) +} + +func TestClientGetVoteAggregatesBatchCap(t *testing.T) { + t.Parallel() + + t.Run("101 URIs rejected without request", func(t *testing.T) { + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + t.Error("batch larger than 100 must be rejected before an HTTP request") + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), clientURIs(101), + ) + require.Error(t, err) + require.Zero(t, requests.Load()) + }) + + t.Run("exactly 100 URIs allowed", func(t *testing.T) { + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{"aggregates": []any{}}); err != nil { + t.Errorf("encode empty aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), clientURIs(100), + ) + require.NoError(t, err) + require.Empty(t, got) + require.EqualValues(t, 1, requests.Load()) + }) +} + +func TestClientGetVoteAggregatesErrorClassification(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + status int + wantTransient bool + }{ + {name: "rate limited", status: http.StatusTooManyRequests, wantTransient: true}, + {name: "service unavailable", status: http.StatusServiceUnavailable, wantTransient: true}, + {name: "request timeout", status: http.StatusRequestTimeout, wantTransient: true}, + {name: "too early", status: http.StatusTooEarly, wantTransient: true}, + {name: "bad request", status: http.StatusBadRequest, wantTransient: false}, + {name: "not found", status: http.StatusNotFound, wantTransient: false}, + {name: "forbidden", status: http.StatusForbidden, wantTransient: false}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, http.StatusText(test.status), test.status) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://a"}, + ) + require.Error(t, err) + require.Equal(t, test.wantTransient, bridgedvotes.IsTransient(err)) + }) + } + + t.Run("transport failure", func(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + client := bridgedvotes.NewClient(server.Client()) + host := trustedHost(t, server.URL) + server.Close() + + _, err := client.GetVoteAggregates(context.Background(), host, []string{"at://a"}) + require.Error(t, err) + require.True(t, bridgedvotes.IsTransient(err)) + }) +} + +func TestClientGetVoteAggregatesEmptyURIsDoesNotRequest(t *testing.T) { + t.Parallel() + + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + t.Error("empty URI input must not make an HTTP request") + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates(context.Background(), trustedHost(t, server.URL), nil) + require.NoError(t, err) + require.Empty(t, got) + require.Zero(t, requests.Load()) +} + +func TestClientGetVoteAggregatesExcludesUnrequestedResponseEntries(t *testing.T) { + t.Parallel() + + const updatedAt = "2026-08-31T02:04:01.080Z" + asOf, err := time.Parse(time.RFC3339Nano, updatedAt) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "aggregates": []map[string]any{ + {"uri": "at://requested-a", "upvotes": 2, "downvotes": 1, "updatedAt": updatedAt}, + {"uri": "at://unrequested", "upvotes": 99, "downvotes": 41, "updatedAt": updatedAt}, + {"uri": "at://requested-b", "upvotes": 4, "downvotes": 0, "updatedAt": updatedAt}, + }, + }); err != nil { + t.Errorf("encode aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested-a", "at://requested-b"}, + ) + require.NoError(t, err) + require.Equal(t, []bridgedvotes.Aggregate{ + {URI: "at://requested-a", Upvotes: 2, Downvotes: 1, AsOf: asOf}, + {URI: "at://requested-b", Upvotes: 4, Downvotes: 0, AsOf: asOf}, + }, got, "the bridge may answer only for subjects named in this request") +} + +func TestClientGetVoteAggregatesRejectsResponseLargerThanOneMiB(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"aggregates":[],"padding":"` + strings.Repeat("x", 1<<20) + `"}`)) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested"}, + ) + require.Error(t, err, "responses larger than 1 MiB must be rejected") + require.False(t, bridgedvotes.IsTransient(err), "a response-size contract violation is permanent") +} + +func TestClientGetVoteAggregatesDeduplicatesResponseURIsFirstWins(t *testing.T) { + t.Parallel() + + const updatedAt = "2026-08-31T02:04:01.080Z" + asOf, err := time.Parse(time.RFC3339Nano, updatedAt) + require.NoError(t, err) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "aggregates": []map[string]any{ + {"uri": "at://requested", "upvotes": 2, "downvotes": 1, "updatedAt": updatedAt}, + {"uri": "at://requested", "upvotes": 9, "downvotes": 4, "updatedAt": updatedAt}, + {"uri": "at://requested", "upvotes": 100, "downvotes": 50, "updatedAt": updatedAt}, + }, + }); err != nil { + t.Errorf("encode aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested"}, + ) + require.NoError(t, err) + require.Equal(t, []bridgedvotes.Aggregate{{ + URI: "at://requested", Upvotes: 2, Downvotes: 1, AsOf: asOf, + }}, got, "one requested URI may cause at most one DB write, with the first response occurrence winning") +} + +func TestClientGetVoteAggregatesDoesNotFollowRedirects(t *testing.T) { + t.Parallel() + + var redirectedRequests atomic.Int64 + redirectTarget := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + redirectedRequests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"aggregates":[]}`)) + })) + t.Cleanup(redirectTarget.Close) + redirectingServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Redirect(w, r, redirectTarget.URL+clientVoteAggregatesPath, http.StatusFound) + })) + t.Cleanup(redirectingServer.Close) + + _, err := bridgedvotes.NewClient(redirectingServer.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, redirectingServer.URL), []string{"at://requested"}, + ) + assert.Error(t, err, "redirect responses are permanent contract violations") + if err != nil { + assert.False(t, bridgedvotes.IsTransient(err)) + } + assert.Zero(t, redirectedRequests.Load(), "the client must not send trusted-host requests to a redirect target") +} + +func TestClientGetVoteAggregatesClassifiesTruncatedBodyAsTransient(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + w.Header().Set("Content-Length", "1024") + _, _ = w.Write([]byte(`{"aggregates":[{"uri":"at://requested"`)) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested"}, + ) + require.Error(t, err) + require.True(t, bridgedvotes.IsTransient(err), + "a response truncated mid-body is a retryable transport fault") +} + +func decodeClientURIs(values []string) []string { + var uris []string + for _, value := range values { + for _, uri := range strings.Split(value, ",") { + if uri = strings.TrimSpace(uri); uri != "" { + uris = append(uris, uri) + } + } + } + return uris +} + +func clientURIs(count int) []string { + uris := make([]string, count) + for i := range uris { + uris[i] = fmt.Sprintf("at://subject-%03d", i) + } + return uris +} + +func TestClientGetVoteAggregatesMalformedJSONWithOKStatusIsPermanent(t *testing.T) { + t.Parallel() + + // A proxy answering an XRPC path with an HTML error page and HTTP 200 is the + // exact failure that wedged the Tidepool side; the poller's poison-batch + // path depends on this classification. + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("Bad gateway")) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested"}, + ) + require.Error(t, err) + require.False(t, bridgedvotes.IsTransient(err), "a body that is not the contract is a permanent failure") +} + +func TestClientGetVoteAggregatesMissingAggregatesKeyIsPermanent(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"results":[]}`)) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested"}, + ) + require.Error(t, err, "a renamed envelope must not decode as an empty success that marks every subject polled") + require.False(t, bridgedvotes.IsTransient(err)) +} + +func TestClientGetVoteAggregatesEveryEntryRejectedIsPermanent(t *testing.T) { + t.Parallel() + + const updatedAt = "2026-08-31T02:04:01.080Z" + tests := []struct { + name string + entries []map[string]any + }{ + {name: "only unrequested subjects", entries: []map[string]any{ + {"uri": "at://other", "upvotes": 1, "downvotes": 0, "updatedAt": updatedAt}, + }}, + {name: "unparseable updatedAt format", entries: []map[string]any{ + {"uri": "at://requested", "upvotes": 1, "downvotes": 0, "updatedAt": "1756605841"}, + }}, + {name: "counts out of range", entries: []map[string]any{ + {"uri": "at://requested", "upvotes": -1, "downvotes": 0, "updatedAt": updatedAt}, + }}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{"aggregates": test.entries}); err != nil { + t.Errorf("encode aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested"}, + ) + require.Error(t, err, "a response that gets every entry wrong is a contract break, not a partial success") + require.False(t, bridgedvotes.IsTransient(err)) + }) + } +} + +func TestClientGetVoteAggregatesDropsFutureAndZeroAsOf(t *testing.T) { + t.Parallel() + + const validUpdatedAt = "2026-08-31T02:04:01.080Z" + expectedAsOf, err := time.Parse(time.RFC3339Nano, validUpdatedAt) + require.NoError(t, err) + farFuture := time.Now().Add(bridgedvotes.MaxAsOfSkew + time.Hour).UTC().Format(time.RFC3339) + withinSkew := time.Now().Add(bridgedvotes.MaxAsOfSkew / 2).UTC().Format(time.RFC3339) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{ + "aggregates": []map[string]any{ + {"uri": "at://future", "upvotes": 4, "downvotes": 1, "updatedAt": farFuture}, + {"uri": "at://zero", "upvotes": 4, "downvotes": 1, "updatedAt": "0001-01-01T00:00:00Z"}, + {"uri": "at://skew", "upvotes": 3, "downvotes": 0, "updatedAt": withinSkew}, + {"uri": "at://valid", "upvotes": 2, "downvotes": 1, "updatedAt": validUpdatedAt}, + }, + }); err != nil { + t.Errorf("encode aggregate response: %v", err) + } + })) + t.Cleanup(server.Close) + + got, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://future", "at://zero", "at://skew", "at://valid"}, + ) + require.NoError(t, err) + require.Len(t, got, 2) + require.Equal(t, "at://skew", got[0].URI, "a stamp inside the skew allowance is ordinary clock drift") + require.Equal(t, bridgedvotes.Aggregate{ + URI: "at://valid", Upvotes: 2, Downvotes: 1, AsOf: expectedAsOf, + }, got[1], "a far-future stamp would win the >= guard once and then reject every honest aggregate") +} + +func TestClientGetVoteAggregatesRejectsZeroTrustedHost(t *testing.T) { + t.Parallel() + + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + http.Error(w, "unexpected request", http.StatusInternalServerError) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(server.Client()).GetVoteAggregates( + context.Background(), bridgedvotes.TrustedHost{}, []string{"at://requested"}, + ) + require.Error(t, err) + require.Zero(t, requests.Load()) +} + +func TestNewClientNilDefaultIsSSRFGuarded(t *testing.T) { + t.Parallel() + + // The default client must be the guarded one used by every other outbound + // fetch: a loopback bridge is exactly what the guard refuses without the + // dev-only private-host hatch, and the refusal must never reach the server. + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"aggregates":[]}`)) + })) + t.Cleanup(server.Close) + + _, err := bridgedvotes.NewClient(nil).GetVoteAggregates( + context.Background(), trustedHost(t, server.URL), []string{"at://requested"}, + ) + require.Error(t, err, "the nil-client default must refuse a private address") + require.Zero(t, requests.Load(), "the guard must refuse before dialing") +} + +func trustedHost(t *testing.T, raw string) bridgedvotes.TrustedHost { + t.Helper() + host, err := bridgedvotes.ParseTrustedHost(raw) + require.NoError(t, err) + return host +} diff --git a/internal/core/bridgedvotes/harness_test.go b/internal/core/bridgedvotes/harness_test.go new file mode 100644 index 0000000..0c71a58 --- /dev/null +++ b/internal/core/bridgedvotes/harness_test.go @@ -0,0 +1,14 @@ +//go:build integration + +package bridgedvotes_test + +import ( + "os" + "testing" + + "Coves/tests/testkit" +) + +func TestMain(m *testing.M) { + os.Exit(testkit.Main(m, testkit.RequirePostgres)) +} diff --git a/internal/core/bridgedvotes/normalizer.go b/internal/core/bridgedvotes/normalizer.go new file mode 100644 index 0000000..66a0b77 --- /dev/null +++ b/internal/core/bridgedvotes/normalizer.go @@ -0,0 +1,40 @@ +package bridgedvotes + +import ( + "net" + "net/url" + "strings" +) + +// NormalizeHost is THE shared normalizer for bridge trust and poll routing; +// jetstream.normalizePDSHost delegates here so provenance checks and poller +// selection cannot drift into different answers. The fallback canonicalizes +// schemeless inputs for BridgeTrust's legacy equal-form comparison; it cannot +// match them to schemeful config, and NewPoller rejects them as dial targets. +// In normal operation identity resolution supplies schemeful communities.pds_url. +func NormalizeHost(raw string) string { + s := strings.TrimSpace(raw) + if s == "" { + return "" + } + + u, err := url.Parse(s) + if err != nil || u.Scheme == "" || u.Host == "" { + return strings.ToLower(strings.TrimRight(s, "/")) + } + + scheme := strings.ToLower(u.Scheme) + hostname := strings.ToLower(u.Hostname()) + port := u.Port() + if (scheme == "http" && port == "80") || (scheme == "https" && port == "443") { + port = "" + } + + host := hostname + if port != "" { + host = net.JoinHostPort(hostname, port) + } else if strings.Contains(hostname, ":") { + host = "[" + hostname + "]" + } + return scheme + "://" + host +} diff --git a/internal/core/bridgedvotes/normalizer_test.go b/internal/core/bridgedvotes/normalizer_test.go new file mode 100644 index 0000000..68d3996 --- /dev/null +++ b/internal/core/bridgedvotes/normalizer_test.go @@ -0,0 +1,17 @@ +package bridgedvotes_test + +import ( + "testing" + + "Coves/internal/core/bridgedvotes" + + "github.com/stretchr/testify/require" +) + +func TestNormalizeHostUsesUnifiedBridgeTrustSemantics(t *testing.T) { + t.Parallel() + + require.Equal(t, "https://bridge.example", bridgedvotes.NormalizeHost("HTTPS://Bridge.Example:443/")) + require.Equal(t, "bridge.example", bridgedvotes.NormalizeHost("bridge.example"), + "schemeless stored values retain BridgeTrust's tolerant normalized fallback") +} diff --git a/internal/core/bridgedvotes/poller.go b/internal/core/bridgedvotes/poller.go new file mode 100644 index 0000000..8bf45d4 --- /dev/null +++ b/internal/core/bridgedvotes/poller.go @@ -0,0 +1,348 @@ +package bridgedvotes + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" +) + +const ( + defaultLookback = 90 * 24 * time.Hour + + // Two thousand subjects are twenty full bridge requests. That bounds one + // sweep against the bridge's 10 rps/burst-30 limiter while leaving headroom + // for other AppView traffic sharing the same limiter. + defaultSweepCap = 2_000 + + // maxTransientStreak is how many consecutive sweeps one batch may fail + // transiently before it is marked past the rotation anyway. Transient + // failures deliberately leave a batch unmarked so an outage is retried, but + // selection is deterministic oldest-first, so a single subject the bridge + // 5xxs on would otherwise sit at the head of its host's rotation and block + // every other subject behind it forever. Three sweeps at the default + // interval is fifteen minutes: long enough to ride out a deploy, short + // enough that a poisoned head costs one batch of healthy rows a cycle + // rather than the whole host. + maxTransientStreak = 3 +) + +// Poller runs sweeps: select candidates, batch per bridge host, fetch, apply, +// advance watermarks. +type Poller struct { + store Store + client *Client + trustedHosts []TrustedHost + opts Options + + // transientStreaks counts consecutive transient failures per leading + // batch across sweeps. Sweeps run on one goroutine under runTicker; the + // mutex is for callers that do not. + mu sync.Mutex + transientStreaks map[string]int +} + +// NewPoller wires a Poller over the store, client, and the operator's trusted +// bridge hosts (TRUSTED_BRIDGE_PDS_HOSTS). Every host must satisfy +// ParseTrustedHost; config.Validate applies the same rule at boot, so a +// rejection here means the poller was built from an unvalidated Config. +func NewPoller(store Store, client *Client, trustedHosts []string, opts Options) (*Poller, error) { + if store == nil { + return nil, errors.New("creating bridged vote poller: Store is nil") + } + if client == nil { + return nil, errors.New("creating bridged vote poller: Client is nil") + } + parsed := make([]TrustedHost, 0, len(trustedHosts)) + for _, raw := range trustedHosts { + host, err := ParseTrustedHost(raw) + if err != nil { + // Rejecting one bad member avoids a poller that reports itself + // configured but silently never reaches that bridge. + return nil, fmt.Errorf("creating bridged vote poller: %w", err) + } + parsed = append(parsed, host) + } + if opts.Lookback <= 0 { + opts.Lookback = defaultLookback + } + if opts.SweepCap <= 0 { + opts.SweepCap = defaultSweepCap + } + + return &Poller{ + store: store, + client: client, + trustedHosts: parsed, + opts: opts, + transientStreaks: make(map[string]int), + }, nil +} + +// hostRouting is the sweep's view of which stored community URLs map to which +// trusted host. It is built once per sweep from the store's distinct URLs. +type hostRouting struct { + // order is the matched hosts in first-seen order; it fixes iteration order + // for selection, fetching and error reporting. + order []TrustedHost + // storedByHost lists the exact stored strings that matched each host; they + // go back to the store as its equality filter. + storedByHost map[TrustedHost][]string + // hostByStored maps each matched stored string to its host, for the + // defense-in-depth recheck of what the store returns. + hostByStored map[string]TrustedHost +} + +func (p *Poller) routeStoredHosts(storedHosts []string) hostRouting { + hostByNormalized := make(map[string]TrustedHost, len(p.trustedHosts)) + for _, host := range p.trustedHosts { + hostByNormalized[host.String()] = host + } + + routing := hostRouting{ + storedByHost: make(map[TrustedHost][]string, len(p.trustedHosts)), + hostByStored: make(map[string]TrustedHost, len(storedHosts)), + } + for _, stored := range storedHosts { + host, ok := hostByNormalized[NormalizeHost(stored)] + if !ok { + continue + } + if _, duplicate := routing.hostByStored[stored]; duplicate { + continue + } + routing.hostByStored[stored] = host + if _, seen := routing.storedByHost[host]; !seen { + routing.order = append(routing.order, host) + } + routing.storedByHost[host] = append(routing.storedByHost[host], stored) + } + return routing +} + +// Sweep runs one poll cycle. The Report is populated even when err is non-nil, +// so a partially failed sweep still tells the job what it managed to do. +func (p *Poller) Sweep(ctx context.Context) (Report, error) { + report := Report{TrustedHosts: len(p.trustedHosts)} + + // No configured bridge means no trust root. Return before even reading stored + // URLs so an unconfigured deployment cannot accidentally turn database values + // into network destinations. + if len(p.trustedHosts) == 0 { + return report, nil + } + + storedHosts, err := p.store.DistinctCommunityPDSURLs(ctx) + if err != nil { + return report, joinSweepErrors(fmt.Errorf("list community PDS URLs for bridged vote sweep: %w", err)) + } + report.StoredHosts = len(storedHosts) + + routing := p.routeStoredHosts(storedHosts) + report.MatchedHosts = len(routing.order) + if len(routing.order) == 0 { + return report, nil + } + + // A global select followed by grouping let one transiently failing bridge's + // deep never-polled backlog consume the entire cap forever: because those rows + // correctly remained unmarked for retry, healthy bridges never entered a sweep. + // Give every matched host its own budget while preserving the cap exactly for + // the single-host case. The 100-row floor is one full client batch and keeps + // small multi-host caps from recreating starvation with tiny allocations. + perHost := p.opts.SweepCap + if len(routing.order) > 1 { + perHost = max(p.opts.SweepCap/len(routing.order), maxAggregateBatch) + } + + var sweepErrors []error + failedHosts := make(map[TrustedHost]struct{}) + batchURIsByHost := make(map[TrustedHost][]string, len(routing.order)) + for _, host := range routing.order { + candidates, err := p.store.SelectCandidates(ctx, routing.storedByHost[host], p.opts.Lookback, perHost) + if err != nil { + // The store is shared, so a selection fault is rarely host-specific, + // but the fetch loop below isolates per host and selection should + // not be the one stage that lets a single failure empty the sweep. + sweepErrors = append(sweepErrors, fmt.Errorf("select bridged vote sweep candidates for %q: %w", host, err)) + failedHosts[host] = struct{}{} + continue + } + // Append in repository order: its watermark ordering is the fairness + // contract, and batching must not reshuffle candidates within a host. + for _, candidate := range candidates { + if matched, ok := routing.hostByStored[candidate.StoredPDSURL]; !ok || matched != host { + // Defense in depth: even if a Store returns more than this host's + // requested values, a database URL never becomes a dial target or + // crosses into another host's budget. The real repository filters + // by exact equality, so this firing means a Store bug. + slog.Warn("bridged vote candidate outside its host filter dropped", + "host", host.String(), + "stored_pds_url", candidate.StoredPDSURL, + "uri", candidate.URI, + ) + continue + } + batchURIsByHost[host] = append(batchURIsByHost[host], candidate.URI) + report.Candidates++ + } + } + + for _, host := range routing.order { + if _, failed := failedHosts[host]; failed { + continue + } + hostErrs, fatal := p.sweepHost(ctx, host, batchURIsByHost[host], &report) + if len(hostErrs) > 0 { + // Kept as separate leaves, not pre-joined: joinSweepErrors classifies + // per leaf, so a canceled mark must not drag the fetch failure it + // followed out of the job log with it. + sweepErrors = append(sweepErrors, hostErrs...) + failedHosts[host] = struct{}{} + } + if fatal { + break + } + } + report.FailedHosts = len(failedHosts) + + return report, joinSweepErrors(sweepErrors...) +} + +// sweepHost fetches, applies and marks one host's candidates in client-sized +// batches. It returns the host's errors, if any, and whether the last of them +// is a store fault the rest of the sweep cannot proceed past: a DB failure is +// not host-isolated, and batches committed and marked earlier remain honest +// completed work. +func (p *Poller) sweepHost(ctx context.Context, host TrustedHost, uris []string, report *Report) (errs []error, fatal bool) { + for start := 0; start < len(uris); start += maxAggregateBatch { + end := min(start+maxAggregateBatch, len(uris)) + batchURIs := uris[start:end] + + aggregates, fetchErr := p.client.GetVoteAggregates(ctx, host, batchURIs) + if fetchErr != nil { + fetchErr = fmt.Errorf("fetch bridged vote batch from %q: %w", host, fetchErr) + if markErr := p.handleFetchFailure(ctx, host, batchURIs, fetchErr, report); markErr != nil { + return []error{fetchErr, markErr}, true + } + // A host fault should not block another bridge, but continuing later + // batches on the same failed host would amplify load during an outage. + return []error{fetchErr}, false + } + p.clearTransientStreak(host, batchURIs) + report.Fetched += len(aggregates) + + for _, aggregate := range aggregates { + if err := p.store.ApplyAggregate(ctx, aggregate); err != nil { + return []error{fmt.Errorf("apply bridged vote aggregate for %q: %w", aggregate.URI, err)}, true + } + report.Applied++ + } + // Advance every attempted URI, including subjects omitted by the bridge, + // so absent aggregates cannot monopolize the oldest rotation slots. + if err := p.store.MarkPolled(ctx, batchURIs); err != nil { + return []error{fmt.Errorf("mark bridged vote batch polled for %q: %w", host, err)}, true + } + report.Marked += len(batchURIs) + } + return nil, false +} + +// handleFetchFailure decides whether a failed batch stays at its watermark for +// retry or is marked past the rotation. It returns a non-nil error only when +// the mark itself failed. +func (p *Poller) handleFetchFailure(ctx context.Context, host TrustedHost, batchURIs []string, fetchErr error, report *Report) error { + if ctx.Err() != nil { + // The sweep's own context ended: shutdown, or the cycle deadline in + // runGuarded. Neither says anything about the bridge, so the batch + // stays at its watermark and does not count toward a streak — a + // deadline that lands on the same batch three cycles running would + // otherwise poison-mark healthy rows for the AppView's slowness. + return nil + } + + reason := "permanent contract failure" + if IsTransient(fetchErr) { + streak := p.recordTransientStreak(host, batchURIs) + if streak < maxTransientStreak { + slog.Warn("bridged vote batch failed transiently; left at its watermark for retry", + "host", host.String(), + "batch_size", len(batchURIs), + "consecutive_sweeps", streak, + "error", fetchErr, + ) + return nil + } + reason = "transient failure persisted across sweeps" + } + + // A proxy serving HTML with HTTP 200 wedged the Tidepool side in exactly + // this position: a failure at the oldest watermark must advance or it + // monopolizes the rotation forever. + if markErr := p.store.MarkPolled(ctx, batchURIs); markErr != nil { + return fmt.Errorf("mark poison bridged vote batch polled for %q: %w", host, markErr) + } + p.clearTransientStreak(host, batchURIs) + report.Marked += len(batchURIs) + report.PoisonMarked += len(batchURIs) + slog.Warn("bridged vote batch skipped past rotation", + "host", host.String(), + "reason", reason, + "batch_size", len(batchURIs), + "first_uri", batchURIs[0], + "error", fetchErr, + ) + return nil +} + +// streakKey identifies a batch across sweeps. Selection is deterministic, so a +// batch that failed last sweep is the same leading run of URIs this sweep; the +// first and last member are enough to recognize it without hashing the set. +func streakKey(host TrustedHost, batchURIs []string) string { + return host.String() + "\x00" + batchURIs[0] + "\x00" + batchURIs[len(batchURIs)-1] +} + +func (p *Poller) recordTransientStreak(host TrustedHost, batchURIs []string) int { + p.mu.Lock() + defer p.mu.Unlock() + key := streakKey(host, batchURIs) + p.transientStreaks[key]++ + return p.transientStreaks[key] +} + +func (p *Poller) clearTransientStreak(host TrustedHost, batchURIs []string) { + if len(batchURIs) == 0 { + return + } + p.mu.Lock() + defer p.mu.Unlock() + delete(p.transientStreaks, streakKey(host, batchURIs)) +} + +// joinSweepErrors keeps shutdown cancellation quiet only when it is the sole +// failure. A canceled DB or fetch leaf joined with an earlier bridge fault would +// otherwise make errors.Is(result, context.Canceled) true and suppress the whole +// cycle in startBridgedVotePollJob, hiding the actionable failure. +func joinSweepErrors(errs ...error) error { + nonCancellation := make([]error, 0, len(errs)) + cancellation := make([]error, 0, 1) + for _, err := range errs { + if err == nil { + continue + } + if errors.Is(err, context.Canceled) { + cancellation = append(cancellation, err) + continue + } + nonCancellation = append(nonCancellation, err) + } + if len(nonCancellation) > 0 { + return errors.Join(nonCancellation...) + } + if len(cancellation) == 1 { + return cancellation[0] + } + return errors.Join(cancellation...) +} diff --git a/internal/core/bridgedvotes/poller_integration_test.go b/internal/core/bridgedvotes/poller_integration_test.go new file mode 100644 index 0000000..c2b0352 --- /dev/null +++ b/internal/core/bridgedvotes/poller_integration_test.go @@ -0,0 +1,264 @@ +//go:build integration + +package bridgedvotes_test + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "Coves/internal/core/bridgedvotes" + "Coves/internal/db/postgres" + "Coves/tests/testkit" + + "github.com/stretchr/testify/require" +) + +const voteAggregatesPath = "/xrpc/social.coves.bridge.getVoteAggregates" + +type servedAggregate struct { + URI string `json:"uri"` + Upvotes int `json:"upvotes"` + Downvotes int `json:"downvotes"` + UpdatedAt string `json:"updatedAt"` +} + +type aggregateServer struct { + mu sync.Mutex + aggregates map[string]servedAggregate + requested []string +} + +func (s *aggregateServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != voteAggregatesPath { + http.NotFound(w, r) + return + } + + var uris []string + for _, value := range r.URL.Query()["uris"] { + for _, uri := range strings.Split(value, ",") { + if uri = strings.TrimSpace(uri); uri != "" { + uris = append(uris, uri) + } + } + } + + s.mu.Lock() + s.requested = append(s.requested, uris...) + aggregates := make([]servedAggregate, 0, len(uris)) + for _, uri := range uris { + if aggregate, ok := s.aggregates[uri]; ok { + aggregates = append(aggregates, aggregate) + } + } + s.mu.Unlock() + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(struct { + Aggregates []servedAggregate `json:"aggregates"` + }{Aggregates: aggregates}); err != nil { + panic(fmt.Sprintf("encode aggregate response: %v", err)) + } +} + +func (s *aggregateServer) replace(aggregate servedAggregate) { + s.mu.Lock() + defer s.mu.Unlock() + s.aggregates[aggregate.URI] = aggregate +} + +func (s *aggregateServer) requestedURIs() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.requested...) +} + +type storedVoteStats struct { + nativeUp int + nativeDown int + bridgedUp int + bridgedDown int + score int + asOf sql.NullTime +} + +type expectedVoteStats struct { + nativeUp int + nativeDown int + bridgedUp int + bridgedDown int + score int + asOf *time.Time +} + +func TestPollerSweepFoldsBridgeAggregatesIntoNativeContent(t *testing.T) { + t.Parallel() + + const ( + postURI = "at://did:plc:nativeauthor/social.coves.community.postv2/native-post" + commentURI = "at://did:plc:nativecommenter/social.coves.community.comment/native-comment" + controlURI = "at://did:plc:controlauthor/social.coves.community.postv2/control-post" + t1Text = "2026-08-31T02:04:01.080Z" + t0Text = "2026-08-30T12:00:00.000Z" + ) + + t1 := mustParseTime(t, t1Text) + bridge := &aggregateServer{aggregates: map[string]servedAggregate{ + postURI: {URI: postURI, Upvotes: 5, Downvotes: 2, UpdatedAt: t1Text}, + commentURI: {URI: commentURI, Upvotes: 2, Downvotes: 0, UpdatedAt: t1Text}, + }} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + + var untrustedRequests atomic.Int64 + untrustedServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + untrustedRequests.Add(1) + http.Error(w, "untrusted community must not be polled", http.StatusInternalServerError) + })) + t.Cleanup(untrustedServer.Close) + + db := testkit.DB(t) + ctx := context.Background() + seedPollerFixtures(t, ctx, db, server.URL+"/", untrustedServer.URL, postURI, commentURI, controlURI) + + poller, err := bridgedvotes.NewPoller( + postgres.NewBridgedVotesRepository(db), + bridgedvotes.NewClient(server.Client()), + []string{server.URL}, + bridgedvotes.Options{}, + ) + require.NoError(t, err) + requireSweep(t, ctx, poller) + + postWant := expectedVoteStats{nativeUp: 3, nativeDown: 1, bridgedUp: 5, bridgedDown: 2, score: 5, asOf: &t1} + commentWant := expectedVoteStats{nativeUp: 4, nativeDown: 1, bridgedUp: 2, bridgedDown: 0, score: 5, asOf: &t1} + controlWant := expectedVoteStats{nativeUp: 7, nativeDown: 2, bridgedUp: 0, bridgedDown: 0, score: 5} + requireVoteStats(t, ctx, db, "posts", postURI, postWant) + requireVoteStats(t, ctx, db, "comments", commentURI, commentWant) + requireVoteStats(t, ctx, db, "posts", controlURI, controlWant) + require.Contains(t, bridge.requestedURIs(), postURI) + require.Contains(t, bridge.requestedURIs(), commentURI) + require.NotContains(t, bridge.requestedURIs(), controlURI) + require.Zero(t, untrustedRequests.Load(), "the untrusted community host must not receive a request") + + requireSweep(t, ctx, poller) + requireVoteStats(t, ctx, db, "posts", postURI, postWant) + requireVoteStats(t, ctx, db, "comments", commentURI, commentWant) + requireVoteStats(t, ctx, db, "posts", controlURI, controlWant) + + bridge.replace(servedAggregate{URI: postURI, Upvotes: 99, Downvotes: 41, UpdatedAt: t0Text}) + requireSweep(t, ctx, poller) + requireVoteStats(t, ctx, db, "posts", postURI, postWant) + require.NotContains(t, bridge.requestedURIs(), controlURI) + require.Zero(t, untrustedRequests.Load(), "the untrusted community host must remain untouched") +} + +func seedPollerFixtures( + t *testing.T, + ctx context.Context, + db *sql.DB, + bridgePDSURL string, + untrustedPDSURL string, + postURI string, + commentURI string, + controlURI string, +) { + t.Helper() + + const ( + bridgeCommunityDID = "did:plc:bridgedcommunity" + controlCommunityDID = "did:plc:controlcommunity" + ) + createdAt := time.Now().UTC() + + _, err := db.ExecContext(ctx, ` + INSERT INTO communities + (did, handle, name, owner_did, created_by_did, hosted_by_did, pds_url, federated_from, created_at) + VALUES + ($1, '!bridged@local.test', 'bridged', $1, $1, $1, $2, 'lemmy', $3), + ($4, '!control@local.test', 'control', $4, $4, $4, $5, NULL, $3) + `, bridgeCommunityDID, bridgePDSURL, createdAt, controlCommunityDID, untrustedPDSURL) + require.NoError(t, err, "seed communities") + + _, err = db.ExecContext(ctx, ` + INSERT INTO posts + (uri, cid, rkey, author_did, community_did, title, created_at, + upvote_count, downvote_count, score, bridged_upvote_count, bridged_downvote_count, bridged_stats_as_of) + VALUES + ($1, 'bafynativepost', 'native-post', 'did:plc:nativeauthor', $2, 'native post', $3, + 3, 1, 2, 0, 0, NULL), + ($4, 'bafycontrolpost', 'control-post', 'did:plc:controlauthor', $5, 'control post', $3, + 7, 2, 5, 0, 0, NULL) + `, postURI, bridgeCommunityDID, createdAt, controlURI, controlCommunityDID) + require.NoError(t, err, "seed posts") + + _, err = db.ExecContext(ctx, ` + INSERT INTO comments + (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, created_at, + upvote_count, downvote_count, score, bridged_upvote_count, bridged_downvote_count, bridged_stats_as_of) + VALUES + ($1, 'bafynativecomment', 'native-comment', 'did:plc:nativecommenter', $2, 'bafynativepost', + $2, 'bafynativepost', 'native comment', $3, 4, 1, 3, 0, 0, NULL) + `, commentURI, postURI, createdAt) + require.NoError(t, err, "seed comment") +} + +func requireVoteStats( + t *testing.T, + ctx context.Context, + db *sql.DB, + table string, + uri string, + want expectedVoteStats, +) { + t.Helper() + + query := `SELECT upvote_count, downvote_count, bridged_upvote_count, bridged_downvote_count, score, bridged_stats_as_of FROM posts WHERE uri = $1` + if table == "comments" { + query = `SELECT upvote_count, downvote_count, bridged_upvote_count, bridged_downvote_count, score, bridged_stats_as_of FROM comments WHERE uri = $1` + } + + var got storedVoteStats + require.NoError(t, db.QueryRowContext(ctx, query, uri).Scan( + &got.nativeUp, + &got.nativeDown, + &got.bridgedUp, + &got.bridgedDown, + &got.score, + &got.asOf, + ), "read %s vote stats for %s", table, uri) + require.Equal(t, want.nativeUp, got.nativeUp, "%s native upvotes", uri) + require.Equal(t, want.nativeDown, got.nativeDown, "%s native downvotes", uri) + require.Equal(t, want.bridgedUp, got.bridgedUp, "%s bridged upvotes", uri) + require.Equal(t, want.bridgedDown, got.bridgedDown, "%s bridged downvotes", uri) + require.Equal(t, want.score, got.score, "%s score", uri) + if want.asOf == nil { + require.False(t, got.asOf.Valid, "%s bridged stats as-of must remain NULL", uri) + return + } + require.True(t, got.asOf.Valid, "%s bridged stats as-of must be populated", uri) + require.True(t, got.asOf.Time.UTC().Equal(want.asOf.UTC()), + "%s bridged stats as-of: got %s, want %s", uri, got.asOf.Time.UTC(), want.asOf.UTC()) +} + +func mustParseTime(t *testing.T, value string) time.Time { + t.Helper() + parsed, err := time.Parse(time.RFC3339Nano, value) + require.NoError(t, err) + return parsed +} + +func requireSweep(t *testing.T, ctx context.Context, poller *bridgedvotes.Poller) { + t.Helper() + _, err := poller.Sweep(ctx) + require.NoError(t, err) +} diff --git a/internal/core/bridgedvotes/poller_test.go b/internal/core/bridgedvotes/poller_test.go new file mode 100644 index 0000000..ba5eded --- /dev/null +++ b/internal/core/bridgedvotes/poller_test.go @@ -0,0 +1,1171 @@ +package bridgedvotes_test + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "sort" + "sync" + "sync/atomic" + "testing" + "time" + + "Coves/internal/core/bridgedvotes" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const unrelatedSweepPDSURL = "https://unrelated.test" + +type sweepSelectCall struct { + storedHosts []string + lookback time.Duration + limit int +} + +type sweepStore struct { + mu sync.Mutex + + distinctURLs []string + candidates []bridgedvotes.Candidate + applyErr error + markErr error + distinctErr error + // selectErrByStoredHost fails SelectCandidates when the filter names that + // stored host, so one host's selection can fail while another's succeeds. + selectErrByStoredHost map[string]error + // Exercises the poller's defense when a buggy or racing Store returns rows + // outside the exact stored-host filter it was given. + ignoreHostFilter bool + rotateByWatermark bool + polledAt map[string]time.Time + markSequence int64 + + distinctCalls int + selectCalls []sweepSelectCall + applied []bridgedvotes.Aggregate + marked [][]string +} + +func (s *sweepStore) SelectCandidates(_ context.Context, storedHosts []string, lookback time.Duration, limit int) ([]bridgedvotes.Candidate, error) { + s.mu.Lock() + defer s.mu.Unlock() + + s.selectCalls = append(s.selectCalls, sweepSelectCall{ + storedHosts: append([]string(nil), storedHosts...), + lookback: lookback, + limit: limit, + }) + for _, host := range storedHosts { + if err := s.selectErrByStoredHost[host]; err != nil { + return nil, err + } + } + allowed := make(map[string]struct{}, len(storedHosts)) + for _, host := range storedHosts { + allowed[host] = struct{}{} + } + selected := make([]bridgedvotes.Candidate, 0, len(s.candidates)) + for _, candidate := range s.candidates { + if _, ok := allowed[candidate.StoredPDSURL]; s.ignoreHostFilter || ok { + selected = append(selected, candidate) + } + } + if s.rotateByWatermark { + sort.SliceStable(selected, func(i, j int) bool { + left, leftPolled := s.polledAt[selected[i].URI] + right, rightPolled := s.polledAt[selected[j].URI] + switch { + case leftPolled != rightPolled: + return !leftPolled + case !leftPolled: + return selected[i].URI < selected[j].URI + case left.Equal(right): + return selected[i].URI < selected[j].URI + default: + return left.Before(right) + } + }) + } + if limit >= 0 && len(selected) > limit { + selected = selected[:limit] + } + return selected, nil +} + +func (s *sweepStore) DistinctCommunityPDSURLs(context.Context) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + s.distinctCalls++ + if s.distinctErr != nil { + return nil, s.distinctErr + } + return append([]string(nil), s.distinctURLs...), nil +} + +func (s *sweepStore) ApplyAggregate(_ context.Context, aggregate bridgedvotes.Aggregate) error { + s.mu.Lock() + defer s.mu.Unlock() + s.applied = append(s.applied, aggregate) + return s.applyErr +} + +func (s *sweepStore) MarkPolled(_ context.Context, uris []string) error { + s.mu.Lock() + defer s.mu.Unlock() + s.marked = append(s.marked, append([]string(nil), uris...)) + if s.rotateByWatermark { + if s.polledAt == nil { + s.polledAt = make(map[string]time.Time) + } + s.markSequence++ + stamp := time.Unix(s.markSequence, 0).UTC() + for _, uri := range uris { + s.polledAt[uri] = stamp + } + } + return s.markErr +} + +func (s *sweepStore) selectCallsSnapshot() []sweepSelectCall { + s.mu.Lock() + defer s.mu.Unlock() + calls := make([]sweepSelectCall, len(s.selectCalls)) + for i, call := range s.selectCalls { + calls[i] = sweepSelectCall{ + storedHosts: append([]string(nil), call.storedHosts...), + lookback: call.lookback, + limit: call.limit, + } + } + return calls +} + +func (s *sweepStore) appliedSnapshot() []bridgedvotes.Aggregate { + s.mu.Lock() + defer s.mu.Unlock() + return append([]bridgedvotes.Aggregate(nil), s.applied...) +} + +func (s *sweepStore) markedSnapshot() [][]string { + s.mu.Lock() + defer s.mu.Unlock() + marked := make([][]string, len(s.marked)) + for i, uris := range s.marked { + marked[i] = append([]string(nil), uris...) + } + return marked +} + +func (s *sweepStore) callCounts() (distinct, selectCalls, applied, marked int) { + s.mu.Lock() + defer s.mu.Unlock() + return s.distinctCalls, len(s.selectCalls), len(s.applied), len(s.marked) +} + +type sweepServedAggregate struct { + URI string `json:"uri"` + Upvotes int `json:"upvotes"` + Downvotes int `json:"downvotes"` + UpdatedAt string `json:"updatedAt"` +} + +type sweepBridge struct { + mu sync.Mutex + + status int + aggregates map[string]sweepServedAggregate + batches [][]string +} + +func (b *sweepBridge) ServeHTTP(w http.ResponseWriter, r *http.Request) { + uris := decodeClientURIs(r.URL.Query()["uris"]) + b.mu.Lock() + b.batches = append(b.batches, append([]string(nil), uris...)) + status := b.status + aggregates := make([]sweepServedAggregate, 0, len(uris)) + for _, uri := range uris { + if aggregate, ok := b.aggregates[uri]; ok { + aggregates = append(aggregates, aggregate) + } + } + b.mu.Unlock() + + if status != 0 && status != http.StatusOK { + http.Error(w, http.StatusText(status), status) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(struct { + Aggregates []sweepServedAggregate `json:"aggregates"` + }{Aggregates: aggregates}); err != nil { + panic(fmt.Sprintf("encode sweep bridge response: %v", err)) + } +} + +func (b *sweepBridge) batchesSnapshot() [][]string { + b.mu.Lock() + defer b.mu.Unlock() + batches := make([][]string, len(b.batches)) + for i, batch := range b.batches { + batches[i] = append([]string(nil), batch...) + } + return batches +} + +func TestSweepNormalizesStoredHostsAndDialsConfiguredHost(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + const candidateURI = "at://did:plc:sweephost/social.coves.community.postv2/post" + store := &sweepStore{ + distinctURLs: []string{storedURL, unrelatedSweepPDSURL}, + candidates: []bridgedvotes.Candidate{ + {URI: candidateURI, StoredPDSURL: storedURL}, + }, + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + sweepOK(t, poller) + calls := store.selectCallsSnapshot() + require.Len(t, calls, 1) + require.Equal(t, []string{storedURL}, calls[0].storedHosts, + "SelectCandidates must receive the exact stored URL that normalized-matched trusted config") + require.Equal(t, [][]string{{candidateURI}}, bridge.batchesSnapshot(), + "the configured host, not the cosmetic stored URL, must be dialled") +} + +func TestSweepNormalizesHostCaseAndDefaultPort(t *testing.T) { + t.Parallel() + + const ( + storedURL = "https://BRIDGE.TEST:443/" // coves:allow-host-literal: inert normalization input with default port; no candidate is returned or dialled + configURL = "https://bridge.test" // coves:allow-host-literal: inert normalization input paired with storedURL; no candidate is returned or dialled + ) + store := &sweepStore{distinctURLs: []string{storedURL}} + poller := newSweepPoller(t, store, http.DefaultClient, []string{configURL}, bridgedvotes.Options{}) + + sweepOK(t, poller) + calls := store.selectCallsSnapshot() + require.Len(t, calls, 1) + require.Equal(t, []string{storedURL}, calls[0].storedHosts) +} + +func TestSweepBatchesEachHostAtOneHundredURIs(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: sweepCandidates(storedURL, 150), + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 150}) + + sweepOK(t, poller) + batches := bridge.batchesSnapshot() + require.Len(t, batches, 2) + sizes := make([]int, len(batches)) + for i, batch := range batches { + sizes[i] = len(batch) + require.LessOrEqual(t, len(batch), 100) + } + require.ElementsMatch(t, []int{100, 50}, sizes) +} + +func TestSweepAppliesServedAggregatesAndMarksEveryPolledURI(t *testing.T) { + t.Parallel() + + const updatedAt = "2026-08-31T02:04:01.080Z" + asOf, err := time.Parse(time.RFC3339Nano, updatedAt) + require.NoError(t, err) + bridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + "at://did:plc:sweepapply/social.coves.community.postv2/served-post": { + URI: "at://did:plc:sweepapply/social.coves.community.postv2/served-post", Upvotes: 5, Downvotes: 2, UpdatedAt: updatedAt, + }, + "at://did:plc:sweepapply/social.coves.community.comment/served-comment": { + URI: "at://did:plc:sweepapply/social.coves.community.comment/served-comment", Upvotes: 2, Downvotes: 0, UpdatedAt: updatedAt, + }, + }} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + polledURIs := []string{ + "at://did:plc:sweepapply/social.coves.community.postv2/served-post", + "at://did:plc:sweepapply/social.coves.community.postv2/omitted-post", + "at://did:plc:sweepapply/social.coves.community.comment/served-comment", + } + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: []bridgedvotes.Candidate{ + {URI: polledURIs[0], StoredPDSURL: storedURL}, + {URI: polledURIs[1], StoredPDSURL: storedURL}, + {URI: polledURIs[2], StoredPDSURL: storedURL}, + }, + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + sweepOK(t, poller) + require.ElementsMatch(t, []bridgedvotes.Aggregate{ + {URI: polledURIs[0], Upvotes: 5, Downvotes: 2, AsOf: asOf}, + {URI: polledURIs[2], Upvotes: 2, Downvotes: 0, AsOf: asOf}, + }, store.appliedSnapshot()) + require.ElementsMatch(t, polledURIs, flattenMarkedURIs(store.markedSnapshot()), + "served and omitted URIs must both advance after a successful fetch") +} + +func TestSweepFailedBatchDoesNotApplyOrMark(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{status: http.StatusServiceUnavailable} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: sweepCandidates(storedURL, 3), + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.True(t, bridgedvotes.IsTransient(err), "HTTP 503 must surface as a transient sweep failure") + require.Empty(t, store.appliedSnapshot(), "a failed fetch cannot apply aggregates") + require.Empty(t, store.markedSnapshot(), "a failed fetch cannot advance watermarks") +} + +func TestSweepNeverRoutesCandidatesFromUnmatchedStoredHost(t *testing.T) { + t.Parallel() + + const ( + matchedURI = "at://did:plc:sweeproute/social.coves.community.postv2/matched" + unmatchedURI = "at://did:plc:sweeproute/social.coves.community.postv2/unmatched" + ) + bridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + unmatchedURI: { + URI: unmatchedURI, Upvotes: 9, Downvotes: 1, UpdatedAt: "2026-08-31T02:04:01.080Z", + }, + }} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + store := &sweepStore{ + distinctURLs: []string{storedURL, unrelatedSweepPDSURL}, + ignoreHostFilter: true, + candidates: []bridgedvotes.Candidate{ + {URI: matchedURI, StoredPDSURL: storedURL}, + {URI: unmatchedURI, StoredPDSURL: unrelatedSweepPDSURL}, + }, + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + sweepOK(t, poller) + require.NotContains(t, flattenBatches(bridge.batchesSnapshot()), unmatchedURI) + require.NotContains(t, flattenMarkedURIs(store.markedSnapshot()), unmatchedURI) + appliedURIs := make([]string, 0, len(store.appliedSnapshot())) + for _, aggregate := range store.appliedSnapshot() { + appliedURIs = append(appliedURIs, aggregate.URI) + } + require.NotContains(t, appliedURIs, unmatchedURI) +} + +func TestSweepWithNoTrustedHostsIsNoOp(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + store := &sweepStore{ + distinctURLs: []string{server.URL + "/"}, + candidates: sweepCandidates(server.URL+"/", 1), + } + poller := newSweepPoller(t, store, server.Client(), nil, bridgedvotes.Options{}) + + sweepOK(t, poller) + distinctCalls, selectCalls, applied, marked := store.callCounts() + require.Zero(t, distinctCalls) + require.Zero(t, selectCalls) + require.Zero(t, applied) + require.Zero(t, marked) + require.Empty(t, bridge.batchesSnapshot()) +} + +func TestSweepZeroOptionsUseSaneSelectionDefaults(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + store := &sweepStore{distinctURLs: []string{server.URL + "/"}} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{}) + + sweepOK(t, poller) + calls := store.selectCallsSnapshot() + require.Len(t, calls, 1) + require.Equal(t, 90*24*time.Hour, calls[0].lookback, "the documented default lookback") + require.Equal(t, 2000, calls[0].limit, "the documented default sweep cap") +} + +func TestNewPollerRejectsNilDependencies(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + t.Cleanup(server.Close) + client := bridgedvotes.NewClient(server.Client()) + + t.Run("nil Store", func(t *testing.T) { + _, err := bridgedvotes.NewPoller(nil, client, []string{server.URL}, bridgedvotes.Options{}) + require.Error(t, err, "a poller without persistence cannot run safely") + }) + t.Run("nil Client", func(t *testing.T) { + _, err := bridgedvotes.NewPoller(&sweepStore{}, nil, []string{server.URL}, bridgedvotes.Options{}) + require.Error(t, err, "a poller without an HTTP client cannot fetch safely") + }) +} + +func TestNewPollerRejectsSchemelessTrustedHosts(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + t.Cleanup(server.Close) + tests := []struct { + name string + hosts []string + }{ + {name: "all invalid", hosts: []string{"tdpl.io"}}, + {name: "mixed valid and invalid", hosts: []string{server.URL, "tdpl.io"}}, + {name: "path would be prefixed onto the XRPC path", hosts: []string{"https://tdpl.io/pds"}}, + {name: "credentials would be sent and logged", hosts: []string{"https://user:secret@tdpl.io"}}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + _, err := bridgedvotes.NewPoller( + &sweepStore{}, bridgedvotes.NewClient(server.Client()), test.hosts, bridgedvotes.Options{}, + ) + require.Error(t, err) + require.Contains(t, err.Error(), "tdpl.io", "the configuration error must name the offending dial target") + }) + } +} + +func TestSweepNegativeOptionsUseSaneSelectionDefaults(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + store := &sweepStore{distinctURLs: []string{server.URL + "/"}} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{ + Lookback: -1, + SweepCap: -1, + }) + + sweepOK(t, poller) + calls := store.selectCallsSnapshot() + require.Len(t, calls, 1) + require.Equal(t, 90*24*time.Hour, calls[0].lookback, "the documented default lookback") + require.Equal(t, 2000, calls[0].limit, "the documented default sweep cap") +} + +func TestSweepPermanentFetchFailureMarksPoisonBatch(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{status: http.StatusBadRequest} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + candidates := sweepCandidates(storedURL, 3) + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: candidates, + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.Empty(t, store.appliedSnapshot(), "a rejected response cannot apply aggregates") + require.ElementsMatch(t, candidateURIs(candidates), flattenMarkedURIs(store.markedSnapshot()), + "a permanent poison batch must advance so it cannot wedge the rotation head") +} + +func TestSweepPriorHostFailureIsNotMaskedByLaterCancellation(t *testing.T) { + t.Parallel() + + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "first bridge failed", http.StatusInternalServerError) + })) + t.Cleanup(first.Close) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + second := httptest.NewServer(http.HandlerFunc(func(_ http.ResponseWriter, r *http.Request) { + cancel() + <-r.Context().Done() + })) + t.Cleanup(second.Close) + firstStored := first.URL + "/" + secondStored := second.URL + "/" + store := &sweepStore{ + distinctURLs: []string{firstStored, secondStored}, + candidates: []bridgedvotes.Candidate{ + {URI: "at://did:plc:sweepcancel/social.coves.community.postv2/first", StoredPDSURL: firstStored}, + {URI: "at://did:plc:sweepcancel/social.coves.community.postv2/second", StoredPDSURL: secondStored}, + }, + } + poller := newSweepPoller(t, store, first.Client(), []string{first.URL, second.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "500", "the first host failure must survive later cancellation") + require.False(t, errors.Is(err, context.Canceled), + "shutdown classification must not hide a real bridge failure from the job logger") +} + +func TestSweepApplyFailurePreservesEarlierFetchFailures(t *testing.T) { + t.Parallel() + + first := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "first bridge unavailable", http.StatusServiceUnavailable) + })) + t.Cleanup(first.Close) + const ( + healthyURI = "at://did:plc:sweepmultierror/social.coves.community.postv2/healthy" + updatedAt = "2026-08-31T02:04:01.080Z" + ) + secondBridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + healthyURI: {URI: healthyURI, Upvotes: 5, Downvotes: 2, UpdatedAt: updatedAt}, + }} + second := httptest.NewServer(secondBridge) + t.Cleanup(second.Close) + firstStored := first.URL + "/" + secondStored := second.URL + "/" + store := &sweepStore{ + distinctURLs: []string{firstStored, secondStored}, + candidates: []bridgedvotes.Candidate{ + {URI: "at://did:plc:sweepmultierror/social.coves.community.postv2/failing", StoredPDSURL: firstStored}, + {URI: healthyURI, StoredPDSURL: secondStored}, + }, + applyErr: errors.New("apply database unavailable"), + } + poller := newSweepPoller(t, store, first.Client(), []string{first.URL, second.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.Contains(t, err.Error(), "503", "the accumulated bridge failure must be retained") + require.Contains(t, err.Error(), "apply database unavailable", "the later Store failure must also be retained") +} + +func TestSweepIsolatesTransientFailureAndCompletesHealthyHost(t *testing.T) { + t.Parallel() + + failingBridge := &sweepBridge{status: http.StatusServiceUnavailable} + failingServer := httptest.NewServer(failingBridge) + t.Cleanup(failingServer.Close) + const ( + failingURI = "at://did:plc:sweepisolation/social.coves.community.postv2/failing" + healthyURI = "at://did:plc:sweepisolation/social.coves.community.postv2/healthy" + updatedAt = "2026-08-31T02:04:01.080Z" + ) + asOf, err := time.Parse(time.RFC3339Nano, updatedAt) + require.NoError(t, err) + healthyBridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + healthyURI: {URI: healthyURI, Upvotes: 6, Downvotes: 1, UpdatedAt: updatedAt}, + }} + healthyServer := httptest.NewServer(healthyBridge) + t.Cleanup(healthyServer.Close) + failingStored := failingServer.URL + "/" + healthyStored := healthyServer.URL + "/" + store := &sweepStore{ + distinctURLs: []string{failingStored, healthyStored}, + candidates: []bridgedvotes.Candidate{ + {URI: failingURI, StoredPDSURL: failingStored}, + {URI: healthyURI, StoredPDSURL: healthyStored}, + }, + } + poller := newSweepPoller(t, store, failingServer.Client(), + []string{failingServer.URL, healthyServer.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err = poller.Sweep(context.Background()) + require.Error(t, err) + require.False(t, errors.Is(err, context.Canceled)) + require.Equal(t, []bridgedvotes.Aggregate{{ + URI: healthyURI, Upvotes: 6, Downvotes: 1, AsOf: asOf, + }}, store.appliedSnapshot()) + marked := flattenMarkedURIs(store.markedSnapshot()) + require.Contains(t, marked, healthyURI) + require.NotContains(t, marked, failingURI, + "a transient failure must remain at its watermark for retry") +} + +func TestSweepApplyErrorDoesNotMarkBatch(t *testing.T) { + t.Parallel() + + const ( + uri = "at://did:plc:sweepapplyerror/social.coves.community.postv2/post" + updatedAt = "2026-08-31T02:04:01.080Z" + ) + bridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + uri: {URI: uri, Upvotes: 5, Downvotes: 2, UpdatedAt: updatedAt}, + }} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + applyErr := errors.New("apply aggregate failed") + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: []bridgedvotes.Candidate{{URI: uri, StoredPDSURL: storedURL}}, + applyErr: applyErr, + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.ErrorIs(t, err, applyErr) + require.Empty(t, store.markedSnapshot(), "a batch with an unapplied aggregate cannot advance") +} + +func TestSweepCapRotatesAcrossNeverPolledCandidatesOverSuccessiveSweeps(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + candidates := sweepCandidates(storedURL, 4) + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: candidates, + rotateByWatermark: true, + polledAt: make(map[string]time.Time), + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 2}) + + sweepOK(t, poller) + sweepOK(t, poller) + requested := flattenBatches(bridge.batchesSnapshot()) + require.Len(t, requested, 4) + require.ElementsMatch(t, candidateURIs(candidates), requested, + "two bounded sweeps must reach every never-polled candidate exactly once") +} + +func TestSweepPerHostSelectionPreventsFailingBacklogFromStarvingHealthyHost(t *testing.T) { + t.Parallel() + + failingBridge := &sweepBridge{status: http.StatusServiceUnavailable} + failingServer := httptest.NewServer(failingBridge) + t.Cleanup(failingServer.Close) + healthyBridge := &sweepBridge{aggregates: make(map[string]sweepServedAggregate)} + healthyServer := httptest.NewServer(healthyBridge) + t.Cleanup(healthyServer.Close) + failingStored := failingServer.URL + "/" + healthyStored := healthyServer.URL + "/" + failingCandidates := sweepCandidatesForHost(failingStored, "fairness-a", 20) + healthyCandidates := sweepCandidatesForHost(healthyStored, "fairness-b", 3) + const updatedAt = "2026-08-31T02:04:01.080Z" + for _, candidate := range healthyCandidates { + healthyBridge.aggregates[candidate.URI] = sweepServedAggregate{ + URI: candidate.URI, Upvotes: 3, Downvotes: 1, UpdatedAt: updatedAt, + } + } + store := &sweepStore{ + distinctURLs: []string{failingStored, healthyStored}, + candidates: append(failingCandidates, healthyCandidates...), + } + poller := newSweepPoller(t, store, failingServer.Client(), + []string{failingServer.URL, healthyServer.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.Error(t, err) + calls := store.selectCallsSnapshot() + assert.Len(t, calls, 2, "each matched config host needs its own candidate budget") + for _, call := range calls { + assert.Len(t, call.storedHosts, 1) + assert.Equal(t, 100, call.limit, "minimum per-host budget prevents starvation by small global caps") + } + healthyURIs := candidateURIs(healthyCandidates) + assert.ElementsMatch(t, healthyURIs, flattenBatches(healthyBridge.batchesSnapshot())) + assert.ElementsMatch(t, healthyURIs, aggregateURIs(store.appliedSnapshot())) + marked := flattenMarkedURIs(store.markedSnapshot()) + for _, uri := range healthyURIs { + assert.Contains(t, marked, uri) + } + for _, uri := range candidateURIs(failingCandidates) { + assert.NotContains(t, marked, uri, "transiently failed host candidates must remain eligible for retry") + } +} + +func TestSweepCanceledMarkErrorDoesNotMaskPriorHostFailure(t *testing.T) { + t.Parallel() + + transientServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "first host failed", http.StatusInternalServerError) + })) + t.Cleanup(transientServer.Close) + permanentServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "poison batch", http.StatusBadRequest) + })) + t.Cleanup(permanentServer.Close) + transientStored := transientServer.URL + "/" + permanentStored := permanentServer.URL + "/" + store := &sweepStore{ + distinctURLs: []string{transientStored, permanentStored}, + candidates: []bridgedvotes.Candidate{ + {URI: "at://did:plc:sweepdbcancel/social.coves.community.postv2/first", StoredPDSURL: transientStored}, + {URI: "at://did:plc:sweepdbcancel/social.coves.community.postv2/poison", StoredPDSURL: permanentStored}, + }, + markErr: fmt.Errorf("mark canceled: %w", context.Canceled), + } + poller := newSweepPoller(t, store, transientServer.Client(), + []string{transientServer.URL, permanentServer.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.Contains(t, err.Error(), "500", "the first host failure must remain visible") + require.NotEmpty(t, store.markedSnapshot(), "the permanent response must exercise the poison-batch DB path") + require.False(t, errors.Is(err, context.Canceled), + "a canceled DB leaf must not suppress an earlier actionable host failure") +} + +func newSweepPoller( + t *testing.T, + store bridgedvotes.Store, + httpClient *http.Client, + trustedHosts []string, + opts bridgedvotes.Options, +) *bridgedvotes.Poller { + t.Helper() + poller, err := bridgedvotes.NewPoller(store, bridgedvotes.NewClient(httpClient), trustedHosts, opts) + require.NoError(t, err) + return poller +} + +func sweepCandidates(pdsURL string, count int) []bridgedvotes.Candidate { + candidates := make([]bridgedvotes.Candidate, count) + for i := range candidates { + candidates[i] = bridgedvotes.Candidate{ + URI: fmt.Sprintf("at://did:plc:sweepbatch/social.coves.community.postv2/%03d", i), + StoredPDSURL: pdsURL, + } + } + return candidates +} + +func sweepCandidatesForHost(pdsURL, subject string, count int) []bridgedvotes.Candidate { + candidates := make([]bridgedvotes.Candidate, count) + for i := range candidates { + candidates[i] = bridgedvotes.Candidate{ + URI: fmt.Sprintf("at://did:plc:%s/social.coves.community.postv2/%03d", subject, i), + StoredPDSURL: pdsURL, + } + } + return candidates +} + +func candidateURIs(candidates []bridgedvotes.Candidate) []string { + uris := make([]string, len(candidates)) + for i, candidate := range candidates { + uris[i] = candidate.URI + } + return uris +} + +func aggregateURIs(aggregates []bridgedvotes.Aggregate) []string { + uris := make([]string, len(aggregates)) + for i, aggregate := range aggregates { + uris[i] = aggregate.URI + } + return uris +} + +func flattenMarkedURIs(batches [][]string) []string { + return flattenBatches(batches) +} + +func flattenBatches(batches [][]string) []string { + var flattened []string + for _, batch := range batches { + flattened = append(flattened, batch...) + } + return flattened +} + +func TestSweepMalformedJSONWithOKStatusMarksPoisonBatch(t *testing.T) { + t.Parallel() + + // The motivating failure: a proxy answering the XRPC path with an HTML page + // and HTTP 200. Left unmarked at the oldest watermark it would be selected + // first every sweep and block the host's rotation forever. + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + requests.Add(1) + w.Header().Set("Content-Type", "text/html") + _, _ = w.Write([]byte("502 Bad Gateway")) + })) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + candidates := sweepCandidates(storedURL, 3) + store := &sweepStore{distinctURLs: []string{storedURL}, candidates: candidates} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + report, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.False(t, bridgedvotes.IsTransient(err)) + require.EqualValues(t, 1, requests.Load()) + require.Empty(t, store.appliedSnapshot()) + require.ElementsMatch(t, candidateURIs(candidates), flattenMarkedURIs(store.markedSnapshot())) + require.Equal(t, 3, report.PoisonMarked) + require.Equal(t, 3, report.Marked) + require.Equal(t, 1, report.FailedHosts) +} + +func TestSweepSelectFailureIsIsolatedPerHost(t *testing.T) { + t.Parallel() + + const ( + healthyURI = "at://did:plc:sweepselect/social.coves.community.postv2/healthy" + updatedAt = "2026-08-31T02:04:01.080Z" + ) + brokenBridge := &sweepBridge{} + brokenServer := httptest.NewServer(brokenBridge) + t.Cleanup(brokenServer.Close) + healthyBridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + healthyURI: {URI: healthyURI, Upvotes: 1, Downvotes: 0, UpdatedAt: updatedAt}, + }} + healthyServer := httptest.NewServer(healthyBridge) + t.Cleanup(healthyServer.Close) + brokenStored := brokenServer.URL + "/" + healthyStored := healthyServer.URL + "/" + selectErr := errors.New("candidate query failed") + store := &sweepStore{ + distinctURLs: []string{brokenStored, healthyStored}, + candidates: []bridgedvotes.Candidate{{URI: healthyURI, StoredPDSURL: healthyStored}}, + selectErrByStoredHost: map[string]error{brokenStored: selectErr}, + } + poller := newSweepPoller(t, store, brokenServer.Client(), + []string{brokenServer.URL, healthyServer.URL}, bridgedvotes.Options{SweepCap: 10}) + + report, err := poller.Sweep(context.Background()) + require.ErrorIs(t, err, selectErr, "the selection failure must be reported") + require.Empty(t, brokenBridge.batchesSnapshot(), "a host whose selection failed is not dialled") + require.Equal(t, []string{healthyURI}, flattenBatches(healthyBridge.batchesSnapshot()), + "a selection failure on one host must not empty the sweep for the other") + require.Equal(t, []string{healthyURI}, aggregateURIs(store.appliedSnapshot())) + require.Equal(t, 1, report.FailedHosts) + require.Equal(t, 2, report.MatchedHosts) +} + +func TestSweepDistinctURLFailureReturnsBeforeDialing(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + distinctErr := errors.New("communities query failed") + store := &sweepStore{distinctErr: distinctErr, candidates: sweepCandidates(server.URL+"/", 2)} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.ErrorIs(t, err, distinctErr) + require.Empty(t, bridge.batchesSnapshot()) + _, selectCalls, _, marked := store.callCounts() + require.Zero(t, selectCalls) + require.Zero(t, marked) +} + +func TestSweepMarkFailureAfterSuccessfulFetchIsReturned(t *testing.T) { + t.Parallel() + + const ( + uri = "at://did:plc:sweepmarkfail/social.coves.community.postv2/post" + updatedAt = "2026-08-31T02:04:01.080Z" + ) + bridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + uri: {URI: uri, Upvotes: 5, Downvotes: 2, UpdatedAt: updatedAt}, + }} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + markErr := errors.New("watermark update failed") + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: []bridgedvotes.Candidate{{URI: uri, StoredPDSURL: storedURL}}, + markErr: markErr, + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + report, err := poller.Sweep(context.Background()) + require.ErrorIs(t, err, markErr) + require.False(t, errors.Is(err, context.Canceled)) + require.Len(t, store.appliedSnapshot(), 1, "the aggregate was applied before the mark failed") + require.Equal(t, 1, report.Applied) + require.Zero(t, report.Marked, "a failed mark must not be counted as advanced") +} + +func TestSweepTransientFailureStreakEventuallyMarksBatch(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{status: http.StatusInternalServerError} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + candidates := sweepCandidates(storedURL, 3) + store := &sweepStore{distinctURLs: []string{storedURL}, candidates: candidates} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + // Two transient sweeps leave the batch at its watermark for retry. + for sweep := 1; sweep <= 2; sweep++ { + report, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.True(t, bridgedvotes.IsTransient(err)) + require.Empty(t, store.markedSnapshot(), "sweep %d must not advance a transiently failed batch", sweep) + require.Zero(t, report.PoisonMarked) + } + + // The third consecutive failure of the same leading batch is the head-of-line + // wedge the transient classification would otherwise make permanent. + report, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.ElementsMatch(t, candidateURIs(candidates), flattenMarkedURIs(store.markedSnapshot()), + "a batch that fails transiently every sweep must eventually advance past the rotation") + require.Equal(t, 3, report.PoisonMarked) + + // The streak resets: a fresh run of failures on the same batch gets its + // full allowance again rather than poison-marking on the next sweep. + _, err = poller.Sweep(context.Background()) + require.Error(t, err) + require.Len(t, store.markedSnapshot(), 1, "the sweep after a poison mark starts a new streak") +} + +func TestSweepHealthyFetchResetsTransientStreak(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{status: http.StatusServiceUnavailable} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + candidates := sweepCandidates(storedURL, 2) + store := &sweepStore{distinctURLs: []string{storedURL}, candidates: candidates} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + for sweep := 1; sweep <= 2; sweep++ { + _, err := poller.Sweep(context.Background()) + require.Error(t, err) + } + bridge.mu.Lock() + bridge.status = http.StatusOK + bridge.mu.Unlock() + report, err := poller.Sweep(context.Background()) + require.NoError(t, err) + require.Equal(t, 2, report.Marked) + require.Zero(t, report.PoisonMarked, "a successful fetch is an ordinary mark, not a poison mark") + + bridge.mu.Lock() + bridge.status = http.StatusServiceUnavailable + bridge.mu.Unlock() + report, err = poller.Sweep(context.Background()) + require.Error(t, err) + require.Zero(t, report.PoisonMarked, "the earlier streak must have been cleared by the healthy sweep") +} + +func TestSweepStopsLaterBatchesOnSameHostAfterFailure(t *testing.T) { + t.Parallel() + + var requests atomic.Int64 + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + if requests.Add(1) == 2 { + http.Error(w, "second batch fails", http.StatusServiceUnavailable) + return + } + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"aggregates":[]}`)) + })) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + candidates := sweepCandidates(storedURL, 250) + store := &sweepStore{distinctURLs: []string{storedURL}, candidates: candidates} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 250}) + + report, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.EqualValues(t, 2, requests.Load(), "the third batch must not be sent to a host whose second batch failed") + require.ElementsMatch(t, candidateURIs(candidates[:100]), flattenMarkedURIs(store.markedSnapshot()), + "only the batch that completed advances") + require.Equal(t, 100, report.Marked) + require.Equal(t, 250, report.Candidates) +} + +func TestSweepPerHostBudgetDividesTheCap(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + hosts int + cap int + wantLimit int + }{ + {name: "two hosts share evenly", hosts: 2, cap: 400, wantLimit: 200}, + {name: "three hosts floor at one batch", hosts: 3, cap: 250, wantLimit: 100}, + {name: "single host keeps the whole cap", hosts: 1, cap: 50, wantLimit: 50}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + t.Parallel() + var configured, stored []string + var client *http.Client + for i := 0; i < test.hosts; i++ { + server := httptest.NewServer(&sweepBridge{}) + t.Cleanup(server.Close) + configured = append(configured, server.URL) + stored = append(stored, server.URL+"/") + client = server.Client() + } + store := &sweepStore{distinctURLs: stored} + poller := newSweepPoller(t, store, client, configured, bridgedvotes.Options{SweepCap: test.cap}) + + sweepOK(t, poller) + calls := store.selectCallsSnapshot() + require.Len(t, calls, test.hosts) + for _, call := range calls { + require.Equal(t, test.wantLimit, call.limit) + } + }) + } +} + +func TestSweepSoleCancellationStaysQuiet(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + store := &sweepStore{distinctURLs: []string{storedURL}, candidates: sweepCandidates(storedURL, 2)} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + _, err := poller.Sweep(ctx) + require.Error(t, err) + require.True(t, errors.Is(err, context.Canceled), + "shutdown as the only failure must classify as cancellation so the job logger stays quiet") + require.Empty(t, store.markedSnapshot(), "a canceled fetch is neither transient nor permanent; the batch stays put") +} + +func TestSweepConfiguredButUnmatchedHostsSelectsNothing(t *testing.T) { + t.Parallel() + + bridge := &sweepBridge{} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + store := &sweepStore{ + distinctURLs: []string{unrelatedSweepPDSURL}, + candidates: sweepCandidates(unrelatedSweepPDSURL, 2), + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + report := sweepOK(t, poller) + _, selectCalls, _, _ := store.callCounts() + require.Zero(t, selectCalls) + require.Empty(t, bridge.batchesSnapshot()) + require.Equal(t, 1, report.TrustedHosts) + require.Equal(t, 1, report.StoredHosts) + require.Zero(t, report.MatchedHosts, "the report is how the job tells a misconfigured trust list from an idle one") +} + +func TestSweepReportCountsCompletedWork(t *testing.T) { + t.Parallel() + + const updatedAt = "2026-08-31T02:04:01.080Z" + storedPrefix := "at://did:plc:sweepreport/social.coves.community.postv2/" + bridge := &sweepBridge{aggregates: map[string]sweepServedAggregate{ + storedPrefix + "000": {URI: storedPrefix + "000", Upvotes: 1, Downvotes: 0, UpdatedAt: updatedAt}, + storedPrefix + "002": {URI: storedPrefix + "002", Upvotes: 2, Downvotes: 1, UpdatedAt: updatedAt}, + }} + server := httptest.NewServer(bridge) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + store := &sweepStore{ + distinctURLs: []string{storedURL, unrelatedSweepPDSURL}, + candidates: sweepCandidatesForHost(storedURL, "sweepreport", 3), + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + report := sweepOK(t, poller) + require.Equal(t, bridgedvotes.Report{ + TrustedHosts: 1, + StoredHosts: 2, + MatchedHosts: 1, + Candidates: 3, + Fetched: 2, + Applied: 2, + Marked: 3, + }, report) +} + +func sweepOK(t *testing.T, poller *bridgedvotes.Poller) bridgedvotes.Report { + t.Helper() + report, err := poller.Sweep(context.Background()) + require.NoError(t, err) + return report +} + +func TestSweepExpiredCycleDeadlineDoesNotCountTowardStreak(t *testing.T) { + t.Parallel() + + // A bridge that answers only after the cycle deadline is the AppView + // running out of time, not the bridge failing the batch. Three such + // cycles must not poison-mark the batch. + release := make(chan struct{}) + t.Cleanup(func() { close(release) }) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + select { + case <-r.Context().Done(): + case <-release: + } + http.Error(w, "too late", http.StatusServiceUnavailable) + })) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + store := &sweepStore{distinctURLs: []string{storedURL}, candidates: sweepCandidates(storedURL, 2)} + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + for sweep := 1; sweep <= 4; sweep++ { + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Millisecond) + report, err := poller.Sweep(ctx) + cancel() + require.Error(t, err) + require.True(t, errors.Is(err, context.DeadlineExceeded), "sweep %d must surface the expired deadline", sweep) + require.Zero(t, report.PoisonMarked, "sweep %d: a cycle deadline is not a bridge failure", sweep) + require.Empty(t, store.markedSnapshot()) + } +} + +func TestSweepCanceledPoisonMarkKeepsFetchFailureVisible(t *testing.T) { + t.Parallel() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + http.Error(w, "poison batch", http.StatusBadRequest) + })) + t.Cleanup(server.Close) + storedURL := server.URL + "/" + store := &sweepStore{ + distinctURLs: []string{storedURL}, + candidates: sweepCandidates(storedURL, 1), + markErr: fmt.Errorf("mark canceled: %w", context.Canceled), + } + poller := newSweepPoller(t, store, server.Client(), []string{server.URL}, bridgedvotes.Options{SweepCap: 10}) + + _, err := poller.Sweep(context.Background()) + require.Error(t, err) + require.Contains(t, err.Error(), "400", + "the permanent fetch failure must not be discarded along with the canceled mark that followed it") + require.False(t, errors.Is(err, context.Canceled)) +} diff --git a/internal/core/bridgedvotes/trustedhost.go b/internal/core/bridgedvotes/trustedhost.go new file mode 100644 index 0000000..5dea2e0 --- /dev/null +++ b/internal/core/bridgedvotes/trustedhost.go @@ -0,0 +1,68 @@ +package bridgedvotes + +import ( + "errors" + "fmt" + "net/url" + "strconv" + "strings" +) + +// TrustedHost is an operator-configured bridge dial target. Only ParseTrustedHost +// builds a non-zero value, so a community's stored pds_url cannot reach +// Client.GetVoteAggregates without passing the validation TRUSTED_BRIDGE_PDS_HOSTS +// itself passes. That turns the poller's central invariant — a database value is +// never a dial target — from a code-review property into a type-level one. +type TrustedHost struct { + // dialURL is the canonical scheme://host[:port] form, normalized exactly as + // NormalizeHost would render it so trust matching and dialing share one key. + dialURL string +} + +// ParseTrustedHost validates one TRUSTED_BRIDGE_PDS_HOSTS entry. The contract is +// scheme + host (+ optional port) and nothing else: userinfo would be sent as +// basic auth and echoed into error logs, a path would be prefixed onto the XRPC +// path and turn every request into a 404 that poison-marks its batch, and a +// query or fragment has no meaning for a dial target. +func ParseTrustedHost(raw string) (TrustedHost, error) { + s := strings.TrimSpace(raw) + if s == "" { + return TrustedHost{}, errors.New("trusted bridge host is empty") + } + u, err := url.Parse(s) + if err != nil { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q is not a URL: %w", raw, err) + } + scheme := strings.ToLower(u.Scheme) + if scheme != "http" && scheme != "https" { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q must be an absolute http(s) URL", raw) + } + if u.Hostname() == "" { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q must name a host", raw) + } + if u.User != nil { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q must not carry credentials", raw) + } + if u.Path != "" && u.Path != "/" { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q must not carry a path; the XRPC path is appended by the poller", raw) + } + if u.RawQuery != "" || u.ForceQuery { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q must not carry a query", raw) + } + if u.Fragment != "" { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q must not carry a fragment", raw) + } + if port := u.Port(); port != "" { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 { + return TrustedHost{}, fmt.Errorf("trusted bridge host %q has an invalid port %q", raw, port) + } + } + return TrustedHost{dialURL: NormalizeHost(scheme + "://" + u.Host)}, nil +} + +// String returns the canonical dial URL, or "" for the zero value. +func (h TrustedHost) String() string { return h.dialURL } + +// IsZero reports whether h was never produced by ParseTrustedHost. +func (h TrustedHost) IsZero() bool { return h.dialURL == "" } diff --git a/internal/core/bridgedvotes/trustedhost_test.go b/internal/core/bridgedvotes/trustedhost_test.go new file mode 100644 index 0000000..7f25269 --- /dev/null +++ b/internal/core/bridgedvotes/trustedhost_test.go @@ -0,0 +1,84 @@ +package bridgedvotes_test + +import ( + "testing" + "time" + + "Coves/internal/core/bridgedvotes" + + "github.com/stretchr/testify/require" +) + +func TestParseTrustedHostAcceptsSchemeAndHostOnly(t *testing.T) { + t.Parallel() + + tests := []struct { + raw string + want string + }{ + {raw: "https://tdpl.io", want: "https://tdpl.io"}, + {raw: "https://tdpl.io/", want: "https://tdpl.io"}, + {raw: " HTTPS://TDPL.IO:443/ ", want: "https://tdpl.io"}, + {raw: "http://bridge.internal:8080", want: "http://bridge.internal:8080"}, + {raw: "https://[2001:db8::1]:8443", want: "https://[2001:db8::1]:8443"}, + } + for _, test := range tests { + host, err := bridgedvotes.ParseTrustedHost(test.raw) + require.NoError(t, err, test.raw) + require.Equal(t, test.want, host.String(), test.raw) + require.False(t, host.IsZero()) + require.Equal(t, bridgedvotes.NormalizeHost(test.raw), host.String(), + "the dial form must be the same key trust matching uses for %q", test.raw) + } +} + +func TestParseTrustedHostRejectsAnythingBeyondSchemeAndHost(t *testing.T) { + t.Parallel() + + tests := map[string]string{ + "empty": "", + "schemeless": "tdpl.io", + "non-http scheme": "ftp://tdpl.io", + "no host": "https://", + "credentials": "https://user:secret@tdpl.io", + "path": "https://tdpl.io/pds", + "query": "https://tdpl.io?x=1", + "fragment": "https://tdpl.io#top", + "port out of range": "https://tdpl.io:70000", + "nested userinfo trick": "https://user@evil.example@tdpl.io", + } + for name, raw := range tests { + t.Run(name, func(t *testing.T) { + t.Parallel() + host, err := bridgedvotes.ParseTrustedHost(raw) + require.Error(t, err) + require.True(t, host.IsZero()) + if raw != "" { + require.Contains(t, err.Error(), raw, "the error must name the offending entry") + } + }) + } +} + +func TestParseAsOfSharedHygiene(t *testing.T) { + t.Parallel() + + now := time.Date(2026, 9, 1, 12, 0, 0, 0, time.UTC) + + got, err := bridgedvotes.ParseAsOf("2026-08-31T02:04:01.080Z", now) + require.NoError(t, err) + require.Equal(t, time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC), got) + + got, err = bridgedvotes.ParseAsOf(now.Add(bridgedvotes.MaxAsOfSkew).Format(time.RFC3339), now) + require.NoError(t, err, "a stamp exactly at the skew allowance is accepted") + require.True(t, got.Equal(now.Add(bridgedvotes.MaxAsOfSkew))) + + _, err = bridgedvotes.ParseAsOf(now.Add(bridgedvotes.MaxAsOfSkew+time.Second).Format(time.RFC3339), now) + require.Error(t, err, "one second past the allowance is a clock fault or a hostile stamp") + + _, err = bridgedvotes.ParseAsOf("0001-01-01T00:00:00Z", now) + require.Error(t, err, "the zero time parses but would defeat the >= guard") + + _, err = bridgedvotes.ParseAsOf("not-a-time", now) + require.Error(t, err) +} diff --git a/internal/db/migrations/043_add_bridged_vote_poll_watermark.sql b/internal/db/migrations/043_add_bridged_vote_poll_watermark.sql new file mode 100644 index 0000000..0bf0e2e --- /dev/null +++ b/internal/db/migrations/043_add_bridged_vote_poll_watermark.sql @@ -0,0 +1,46 @@ +-- +goose Up +-- Add nullable poll watermarks for rotating the bridge aggregate side-channel +-- across eligible posts and comments. NULL means never polled and intentionally +-- sorts first, so newly indexed and pre-migration content receives a first pass +-- before already-visited rows cycle back through the bounded sweep. + +ALTER TABLE posts + ADD COLUMN bridged_polled_at TIMESTAMPTZ; + +ALTER TABLE comments + ADD COLUMN bridged_polled_at TIMESTAMPTZ; + +-- The bridged-vote poller repeatedly takes the oldest active rows from each table. +-- The created_at and uri suffix makes selection deterministic while a never-polled +-- cohort is larger than one sweep, rather than leaving its first pass to the plan. +CREATE INDEX idx_posts_bridged_poll + ON posts (bridged_polled_at ASC NULLS FIRST, created_at ASC, uri ASC) + WHERE deleted_at IS NULL; + +CREATE INDEX idx_comments_bridged_poll + ON comments (bridged_polled_at ASC NULLS FIRST, created_at ASC, uri ASC) + WHERE deleted_at IS NULL; + +COMMENT ON COLUMN posts.bridged_polled_at IS 'When the bridge aggregate side-channel last attempted this post; NULL means never polled'; +COMMENT ON COLUMN comments.bridged_polled_at IS 'When the bridge aggregate side-channel last attempted this comment; NULL means never polled'; + +-- Migration 031's catalog text said these guards accepted only strictly newer +-- samples. Both Jetstream ingestion and the poller deliberately use >= so an +-- equal-asOf replay is idempotent. The column comments are documentation only, +-- so they are corrected here rather than in a migration of their own; Down +-- restores 031's wording so a rollback lands exactly where 031 left the catalog. +COMMENT ON COLUMN posts.bridged_stats_as_of IS 'When the bridged counts were sampled; updates apply when the incoming asOf is newer-or-equal (>=)'; +COMMENT ON COLUMN comments.bridged_stats_as_of IS 'When the bridged counts were sampled; updates apply when the incoming asOf is newer-or-equal (>=)'; + +-- +goose Down +COMMENT ON COLUMN posts.bridged_stats_as_of IS 'When the bridged counts were sampled; updates apply only when strictly newer'; +COMMENT ON COLUMN comments.bridged_stats_as_of IS 'When the bridged counts were sampled; updates apply only when strictly newer'; + +DROP INDEX IF EXISTS idx_comments_bridged_poll; +DROP INDEX IF EXISTS idx_posts_bridged_poll; + +ALTER TABLE comments + DROP COLUMN IF EXISTS bridged_polled_at; + +ALTER TABLE posts + DROP COLUMN IF EXISTS bridged_polled_at; diff --git a/internal/db/postgres/admission_repo_schema_test.go b/internal/db/postgres/admission_repo_schema_test.go index 879f46d..daf2c00 100644 --- a/internal/db/postgres/admission_repo_schema_test.go +++ b/internal/db/postgres/admission_repo_schema_test.go @@ -326,6 +326,8 @@ func TestMigration034_DownRestoresTheAuthorForeignKeyUnvalidated(t *testing.T) { // sit on top of 034, so all eight have to come off first. Rolling back explicitly, // one asserted step at a time, is what keeps the assertions below pointed at // 034's Down rather than at whatever happens to be newest. + require.EqualValues(t, 43, testkit.MigrateDownOne(t, db, 43), + "043 (the bridged-vote poll watermark) sits on top and must be rolled back first") require.EqualValues(t, 42, testkit.MigrateDownOne(t, db, 42), "042 (the dead-letter retention index) sits on top of 034 and must be rolled back first; asserting which migration came off is what stops this test drifting onto a newer one") require.EqualValues(t, 41, testkit.MigrateDownOne(t, db, 41), diff --git a/internal/db/postgres/bridged_votes_repo.go b/internal/db/postgres/bridged_votes_repo.go new file mode 100644 index 0000000..66731c8 --- /dev/null +++ b/internal/db/postgres/bridged_votes_repo.go @@ -0,0 +1,233 @@ +package postgres + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "time" + + "Coves/internal/core/bridgedvotes" + + "github.com/lib/pq" +) + +// BridgedVotesRepository implements bridgedvotes.Store over posts, comments and +// communities. +type BridgedVotesRepository struct { + db *sql.DB +} + +// NewBridgedVotesRepository builds the postgres-backed bridgedvotes.Store. +func NewBridgedVotesRepository(db *sql.DB) *BridgedVotesRepository { + return &BridgedVotesRepository{db: db} +} + +// SelectCandidates implements bridgedvotes.Store: it selects the oldest eligible subjects in poll-rotation order. +func (r *BridgedVotesRepository) SelectCandidates(ctx context.Context, storedHosts []string, lookback time.Duration, limit int) ([]bridgedvotes.Candidate, error) { + // Exact pds_url equality is deliberate here. The poller normalizes operator + // config, matches it to stored community values, then passes those exact stored + // strings back so this persistence layer never turns an untrusted URL into a dial + // target. + // + // Comments carry no community column, so their community and PDS must be inherited + // from an indexed root post. The inner join and root soft-delete gate also prevent a + // dangling comment or a comment on removed content from entering the poll rotation. + query := ` + SELECT uri, pds_url + FROM ( + SELECT p.uri, co.pds_url, p.bridged_polled_at, p.created_at + FROM posts p + JOIN communities co ON co.did = p.community_did + WHERE co.pds_url = ANY($1) + AND p.deleted_at IS NULL + AND p.created_at > NOW() - make_interval(secs => $2) + + UNION ALL + + SELECT c.uri, co.pds_url, c.bridged_polled_at, c.created_at + FROM comments c + JOIN posts p ON p.uri = c.root_uri AND p.deleted_at IS NULL + JOIN communities co ON co.did = p.community_did + WHERE co.pds_url = ANY($1) + AND c.deleted_at IS NULL + AND c.created_at > NOW() - make_interval(secs => $2) + ) AS candidates + -- A sweep cap smaller than the never-polled cohort used to leave which + -- rows entered rotation first up to the query plan. Creation time and URI + -- make that first pass stable across indexes, restarts and equal stamps. + ORDER BY bridged_polled_at ASC NULLS FIRST, created_at ASC, uri ASC + LIMIT $3 + ` + + rows, err := r.db.QueryContext(ctx, query, pq.Array(storedHosts), lookback.Seconds(), limit) + if err != nil { + return nil, fmt.Errorf("failed to select bridged vote candidates: %w", err) + } + defer func() { + if closeErr := rows.Close(); closeErr != nil { + slog.Warn("failed to close bridged vote candidate rows", "error", closeErr) + } + }() + + candidates := make([]bridgedvotes.Candidate, 0) + for rows.Next() { + var candidate bridgedvotes.Candidate + if err := rows.Scan(&candidate.URI, &candidate.StoredPDSURL); err != nil { + return nil, fmt.Errorf("failed to scan bridged vote candidate: %w", err) + } + candidates = append(candidates, candidate) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating bridged vote candidates: %w", err) + } + + return candidates, nil +} + +// DistinctCommunityPDSURLs implements bridgedvotes.Store: it lists the stored community PDS values eligible for trust matching. +func (r *BridgedVotesRepository) DistinctCommunityPDSURLs(ctx context.Context) ([]string, error) { + rows, err := r.db.QueryContext(ctx, ` + SELECT DISTINCT pds_url + FROM communities + WHERE pds_url IS NOT NULL AND pds_url <> '' + `) + if err != nil { + return nil, fmt.Errorf("failed to list distinct community PDS URLs: %w", err) + } + defer func() { + if closeErr := rows.Close(); closeErr != nil { + slog.Warn("failed to close community PDS URL rows", "error", closeErr) + } + }() + + urls := make([]string, 0) + for rows.Next() { + var pdsURL string + if err := rows.Scan(&pdsURL); err != nil { + return nil, fmt.Errorf("failed to scan community PDS URL: %w", err) + } + urls = append(urls, pdsURL) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("error iterating community PDS URLs: %w", err) + } + + return urls, nil +} + +// ApplyAggregate implements bridgedvotes.Store: it applies a non-regressing bridged tally to its post or comment. +func (r *BridgedVotesRepository) ApplyAggregate(ctx context.Context, agg bridgedvotes.Aggregate) error { + if agg.AsOf.IsZero() { + // The client never produces one (ParseAsOf rejects the zero time), so + // this is a caller bug, and a silent success would let counts land + // without the sampling instant the >= guard depends on. + return fmt.Errorf("apply bridged vote aggregate to %q: %w", agg.URI, bridgedvotes.ErrMissingAsOf) + } + + // Jetstream record stamps and this poller race through the same bridged columns. + // Keeping the >= guard, count replacement, and score recomputation in one UPDATE + // prevents a read-then-write race from letting an older aggregate overwrite a newer + // one or recomputing score from counts that did not win the guard. + result, err := r.db.ExecContext(ctx, ` + UPDATE posts + SET bridged_upvote_count = $2, + bridged_downvote_count = $3, + bridged_stats_as_of = $4, + score = (upvote_count + $2) - (downvote_count + $3) + WHERE uri = $1 + AND deleted_at IS NULL + AND (bridged_stats_as_of IS NULL OR $4 >= bridged_stats_as_of) + `, agg.URI, agg.Upvotes, agg.Downvotes, agg.AsOf) + if err != nil { + return fmt.Errorf("failed to apply bridged vote aggregate to post: %w", err) + } + + rowsAffected, err := result.RowsAffected() + if err != nil { + return fmt.Errorf("failed to check bridged vote post aggregate result: %w", err) + } + if rowsAffected > 0 { + return nil + } + + commentResult, err := r.db.ExecContext(ctx, ` + UPDATE comments + SET bridged_upvote_count = $2, + bridged_downvote_count = $3, + bridged_stats_as_of = $4, + score = (upvote_count + $2) - (downvote_count + $3) + WHERE uri = $1 + AND deleted_at IS NULL + AND (bridged_stats_as_of IS NULL OR $4 >= bridged_stats_as_of) + `, agg.URI, agg.Upvotes, agg.Downvotes, agg.AsOf) + if err != nil { + return fmt.Errorf("failed to apply bridged vote aggregate to comment: %w", err) + } + commentRowsAffected, err := commentResult.RowsAffected() + if err != nil { + return fmt.Errorf("failed to check bridged vote comment aggregate result: %w", err) + } + if commentRowsAffected == 0 { + // The poller selected this subject as existing and non-deleted moments + // ago, so zero rows in both tables is almost always the stale-guard + // case: a stored bridged_stats_as_of newer than what the bridge just + // served. Once is a race with a Jetstream stamp; a sustained stream on + // one URI is a stored stamp that will never be beaten, and Warn is the + // level production actually emits. + slog.Warn("bridged vote aggregate matched no writable subject", + "uri", agg.URI, + "incoming_as_of", agg.AsOf, + ) + } + + // A subject may disappear after selection, and stale asOf values are expected to + // lose the guard. Neither case may wedge the sweep, so zero matches in both tables + // are a successful no-op just like a bridge response that omits a subject. + return nil +} + +// MarkPolled implements bridgedvotes.Store: it advances rotation watermarks for every attempted subject. +func (r *BridgedVotesRepository) MarkPolled(ctx context.Context, uris []string) error { + if len(uris) == 0 { + return nil + } + + // Every attempted candidate advances even when the bridge omitted it from its + // response, otherwise permanently absent aggregates would monopolize the oldest + // rotation slots. Unknown or concurrently removed URIs therefore deliberately + // affect zero rows without turning the sweep into an error. Both tables move + // in one transaction so a batch is never left half-advanced, with its posts + // out of the oldest slot and its comments still in it. + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin bridged vote mark transaction: %w", err) + } + defer func() { + if rollbackErr := tx.Rollback(); rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) { + slog.Warn("failed to roll back bridged vote mark transaction", "error", rollbackErr) + } + }() + + if _, err := tx.ExecContext(ctx, ` + UPDATE posts + SET bridged_polled_at = NOW() + WHERE uri = ANY($1) + `, pq.Array(uris)); err != nil { + return fmt.Errorf("failed to mark bridged vote posts polled: %w", err) + } + + if _, err := tx.ExecContext(ctx, ` + UPDATE comments + SET bridged_polled_at = NOW() + WHERE uri = ANY($1) + `, pq.Array(uris)); err != nil { + return fmt.Errorf("failed to mark bridged vote comments polled: %w", err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("failed to commit bridged vote mark transaction: %w", err) + } + return nil +} diff --git a/internal/db/postgres/bridged_votes_repo_test.go b/internal/db/postgres/bridged_votes_repo_test.go new file mode 100644 index 0000000..169fff4 --- /dev/null +++ b/internal/db/postgres/bridged_votes_repo_test.go @@ -0,0 +1,651 @@ +//go:build integration + +package postgres + +import ( + "context" + "database/sql" + "testing" + "time" + + "Coves/internal/core/bridgedvotes" + "Coves/tests/testkit" + + "github.com/stretchr/testify/require" +) + +const ( + bridgeAPDSURL = "https://bridge-a.test" // coves:allow-host-literal: exact inert communities.pds_url fixture under test; never dialled + otherPDSURL = "https://other.test" // coves:allow-host-literal: exact inert communities.pds_url fixture under test; never dialled +) + +type bridgedVotesCandidatesFixture struct { + p1 string + p2 string + p3 string + p4 string + c1 string + c2 string + c3 string + c4 string +} + +func TestBridgedVotesRepository_SelectCandidates(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedBridgedVotesCandidates(t, ctx, db) + + got, err := NewBridgedVotesRepository(db).SelectCandidates(ctx, []string{bridgeAPDSURL}, 90*24*time.Hour, 50) + require.NoError(t, err) + require.ElementsMatch(t, []bridgedvotes.Candidate{ + {URI: fixture.p1, StoredPDSURL: bridgeAPDSURL}, + {URI: fixture.c1, StoredPDSURL: bridgeAPDSURL}, + }, got) +} + +func TestBridgedVotesRepository_SelectCandidatesHonorsLimit(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedBridgedVotesCandidates(t, ctx, db) + + got, err := NewBridgedVotesRepository(db).SelectCandidates(ctx, []string{bridgeAPDSURL}, 90*24*time.Hour, 1) + require.NoError(t, err) + require.Len(t, got, 1) + require.Contains(t, []string{fixture.p1, fixture.c1}, got[0].URI) + require.Equal(t, bridgeAPDSURL, got[0].StoredPDSURL) +} + +func TestBridgedVotesRepository_DistinctCommunityPDSURLs(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + seedBridgedVotesCandidates(t, ctx, db) + _, err := db.ExecContext(ctx, ` + INSERT INTO communities + (did, handle, name, owner_did, created_by_did, hosted_by_did, pds_url, created_at) + VALUES + ('did:plc:bridgedvotescandidated', '!candidate-d@local.test', 'candidate-d', + 'did:plc:bridgedvotescandidated', 'did:plc:bridgedvotescandidated', + 'did:plc:bridgedvotescandidated', $1, NOW()) + `, bridgeAPDSURL) + require.NoError(t, err, "seed duplicate community PDS URL") + + got, err := NewBridgedVotesRepository(db).DistinctCommunityPDSURLs(ctx) + require.NoError(t, err) + require.ElementsMatch(t, []string{bridgeAPDSURL, otherPDSURL}, got) +} + +func seedBridgedVotesCandidates(t *testing.T, ctx context.Context, db *sql.DB) bridgedVotesCandidatesFixture { + t.Helper() + + const ( + communityA = "did:plc:bridgedvotescandidatea" + communityB = "did:plc:bridgedvotescandidateb" + communityC = "did:plc:bridgedvotescandidatec" + ) + fixture := bridgedVotesCandidatesFixture{ + p1: "at://did:plc:author1/social.coves.community.postv2/p1", + p2: "at://did:plc:author2/social.coves.community.postv2/p2", + p3: "at://did:plc:author3/social.coves.community.postv2/p3", + p4: "at://did:plc:author4/social.coves.community.postv2/p4", + c1: "at://did:plc:commenter1/social.coves.community.comment/c1", + c2: "at://did:plc:commenter2/social.coves.community.comment/c2", + c3: "at://did:plc:commenter3/social.coves.community.comment/c3", + c4: "at://did:plc:commenter4/social.coves.community.comment/c4", + } + recent := time.Now().UTC() + old := recent.Add(-120 * 24 * time.Hour) + deletedAt := recent.Add(-time.Hour) + + _, err := db.ExecContext(ctx, ` + INSERT INTO communities + (did, handle, name, owner_did, created_by_did, hosted_by_did, pds_url, created_at) + VALUES + ($1, '!candidate-a@local.test', 'candidate-a', $1, $1, $1, $2, $6), + ($3, '!candidate-b@local.test', 'candidate-b', $3, $3, $3, $4, $6), + ($5, '!candidate-c@local.test', 'candidate-c', $5, $5, $5, '', $6) + `, communityA, bridgeAPDSURL, communityB, otherPDSURL, communityC, recent) + require.NoError(t, err, "seed candidate communities") + + _, err = db.ExecContext(ctx, ` + INSERT INTO posts + (uri, cid, rkey, author_did, community_did, title, created_at, deleted_at) + VALUES + ($1, 'bafycandidatep1', 'p1', 'did:plc:author1', $2, 'p1', $7, NULL), + ($3, 'bafycandidatep2', 'p2', 'did:plc:author2', $2, 'p2', $7, $8), + ($4, 'bafycandidatep3', 'p3', 'did:plc:author3', $2, 'p3', $9, NULL), + ($5, 'bafycandidatep4', 'p4', 'did:plc:author4', $6, 'p4', $7, NULL) + `, fixture.p1, communityA, fixture.p2, fixture.p3, fixture.p4, communityB, recent, deletedAt, old) + require.NoError(t, err, "seed candidate posts") + + const ghostRootURI = "at://did:plc:ghost/social.coves.community.postv2/unindexed" + _, err = db.ExecContext(ctx, ` + INSERT INTO comments + (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, created_at, deleted_at) + VALUES + ($1, 'bafycandidatec1', 'c1', 'did:plc:commenter1', $5, 'bafycandidatep1', $5, 'bafycandidatep1', 'c1', $8, NULL), + ($2, 'bafycandidatec2', 'c2', 'did:plc:commenter2', $6, 'bafycandidatep2', $6, 'bafycandidatep2', 'c2', $8, NULL), + ($3, 'bafycandidatec3', 'c3', 'did:plc:commenter3', $5, 'bafycandidatep1', $5, 'bafycandidatep1', 'c3', $8, $9), + ($4, 'bafycandidatec4', 'c4', 'did:plc:commenter4', $7, 'bafyghost', $7, 'bafyghost', 'c4', $8, NULL) + `, fixture.c1, fixture.c2, fixture.c3, fixture.c4, fixture.p1, fixture.p2, ghostRootURI, recent, deletedAt) + require.NoError(t, err, "seed candidate comments") + + return fixture +} + +func TestBridgedVotesRepository_MarkPolledAdvancesPostsAndComments(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedBridgedVotesCandidates(t, ctx, db) + const missingURI = "at://did:plc:missing/social.coves.community.postv2/not-indexed" + + require.NoError(t, NewBridgedVotesRepository(db).MarkPolled(ctx, []string{fixture.p1, fixture.c1, missingURI})) + + postPolledAt := readBridgedPolledAt(t, ctx, db, "posts", fixture.p1) + commentPolledAt := readBridgedPolledAt(t, ctx, db, "comments", fixture.c1) + unnamedPostPolledAt := readBridgedPolledAt(t, ctx, db, "posts", fixture.p4) + + require.True(t, postPolledAt.Valid, "named post bridged_polled_at must be populated") + require.WithinDuration(t, time.Now().UTC(), postPolledAt.Time.UTC(), time.Minute) + require.True(t, commentPolledAt.Valid, "named comment bridged_polled_at must be populated") + require.WithinDuration(t, time.Now().UTC(), commentPolledAt.Time.UTC(), time.Minute) + require.False(t, unnamedPostPolledAt.Valid, "post omitted from MarkPolled must retain a NULL watermark") +} + +func TestBridgedVotesRepository_SelectCandidatesRotatesNeverPolledThenOldest(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedBridgedVotesRotation(t, ctx, db) + repo := NewBridgedVotesRepository(db) + + got, err := repo.SelectCandidates(ctx, []string{bridgeAPDSURL}, 90*24*time.Hour, 50) + require.NoError(t, err) + require.Equal(t, []bridgedvotes.Candidate{ + {URI: fixture.pNever, StoredPDSURL: bridgeAPDSURL}, + {URI: fixture.pOld, StoredPDSURL: bridgeAPDSURL}, + {URI: fixture.pNew, StoredPDSURL: bridgeAPDSURL}, + }, got) + + got, err = repo.SelectCandidates(ctx, []string{bridgeAPDSURL}, 90*24*time.Hour, 1) + require.NoError(t, err) + require.Equal(t, []bridgedvotes.Candidate{ + {URI: fixture.pNever, StoredPDSURL: bridgeAPDSURL}, + }, got) +} + +func readBridgedPolledAt(t *testing.T, ctx context.Context, db *sql.DB, table, uri string) sql.NullTime { + t.Helper() + + query := `SELECT bridged_polled_at FROM posts WHERE uri = $1` + if table == "comments" { + query = `SELECT bridged_polled_at FROM comments WHERE uri = $1` + } + + var polledAt sql.NullTime + require.NoError(t, db.QueryRowContext(ctx, query, uri).Scan(&polledAt), "read %s watermark for %s", table, uri) + return polledAt +} + +type bridgedVotesRotationFixture struct { + pNever string + pOld string + pNew string +} + +func seedBridgedVotesRotation(t *testing.T, ctx context.Context, db *sql.DB) bridgedVotesRotationFixture { + t.Helper() + + const communityDID = "did:plc:bridgedvotesrotation" + fixture := bridgedVotesRotationFixture{ + pNever: "at://did:plc:rotationnever/social.coves.community.postv2/never", + pOld: "at://did:plc:rotationold/social.coves.community.postv2/old", + pNew: "at://did:plc:rotationnew/social.coves.community.postv2/new", + } + now := time.Now().UTC() + + _, err := db.ExecContext(ctx, ` + INSERT INTO communities + (did, handle, name, owner_did, created_by_did, hosted_by_did, pds_url, created_at) + VALUES + ($1, '!rotation@local.test', 'rotation', $1, $1, $1, $2, $3) + `, communityDID, bridgeAPDSURL, now) + require.NoError(t, err, "seed rotation community") + + _, err = db.ExecContext(ctx, ` + INSERT INTO posts + (uri, cid, rkey, author_did, community_did, title, created_at) + VALUES + ($1, 'bafyrotationnever', 'never', 'did:plc:rotationnever', $4, 'never', $5), + ($2, 'bafyrotationold', 'old', 'did:plc:rotationold', $4, 'old', $5), + ($3, 'bafyrotationnew', 'new', 'did:plc:rotationnew', $4, 'new', $5) + `, fixture.pNever, fixture.pOld, fixture.pNew, communityDID, now) + require.NoError(t, err, "seed rotation posts") + + _, err = db.ExecContext(ctx, ` + UPDATE posts + SET bridged_polled_at = CASE uri + WHEN $1 THEN $3::timestamptz + WHEN $2 THEN $4::timestamptz + END + WHERE uri IN ($1, $2) + `, fixture.pOld, fixture.pNew, now.Add(-2*time.Hour), now.Add(-time.Minute)) + require.NoError(t, err, "seed rotation watermarks") + + return fixture +} + +func TestBridgedVotesRepository_ApplyAggregateFirstApply(t *testing.T) { + t.Parallel() + + t1 := time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC) + tests := []struct { + name string + table string + uri func(applyAggregateFixture) string + }{ + {name: "post", table: "posts", uri: func(f applyAggregateFixture) string { return f.postURI }}, + {name: "comment", table: "comments", uri: func(f applyAggregateFixture) string { return f.commentURI }}, + } + + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + uri := test.uri(fixture) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: uri, Upvotes: 5, Downvotes: 2, AsOf: t1, + })) + requireAggregateStats(t, ctx, db, test.table, uri, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 5, bridgedDown: 2, score: 5, asOf: &t1, + }) + }) + } +} + +func TestBridgedVotesRepository_ApplyAggregateNewerAsOfOverwrites(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + t1 := time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC) + t2 := t1.Add(time.Minute) + seedStoredAggregate(t, ctx, db, "posts", fixture.postURI, 5, 2, t1) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.postURI, Upvotes: 7, Downvotes: 2, AsOf: t2, + })) + requireAggregateStats(t, ctx, db, "posts", fixture.postURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 7, bridgedDown: 2, score: 7, asOf: &t2, + }) +} + +func TestBridgedVotesRepository_ApplyAggregateEqualAsOfReapplies(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + t2 := time.Date(2026, 8, 31, 2, 5, 1, 80_000_000, time.UTC) + seedStoredAggregate(t, ctx, db, "posts", fixture.postURI, 7, 2, t2) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.postURI, Upvotes: 7, Downvotes: 2, AsOf: t2, + })) + requireAggregateStats(t, ctx, db, "posts", fixture.postURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 7, bridgedDown: 2, score: 7, asOf: &t2, + }) +} + +func TestBridgedVotesRepository_ApplyAggregateStaleAsOfIsNoOp(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + t1 := time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC) + t0 := t1.Add(-time.Minute) + seedStoredAggregate(t, ctx, db, "posts", fixture.postURI, 5, 2, t1) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.postURI, Upvotes: 99, Downvotes: 41, AsOf: t0, + })) + requireAggregateStats(t, ctx, db, "posts", fixture.postURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 5, bridgedDown: 2, score: 5, asOf: &t1, + }) +} + +func TestBridgedVotesRepository_ApplyAggregateDeletedRowIsNoOpSuccess(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + t1 := time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.deletedPostURI, Upvotes: 5, Downvotes: 2, AsOf: t1, + })) + requireAggregateStats(t, ctx, db, "posts", fixture.deletedPostURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 0, bridgedDown: 0, score: 2, + }) +} + +func TestBridgedVotesRepository_ApplyAggregateAbsentRowIsNoOpSuccess(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + seedApplyAggregateFixture(t, ctx, db) + t1 := time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: "at://did:plc:missing/social.coves.community.postv2/not-indexed", + Upvotes: 5, Downvotes: 2, AsOf: t1, + })) +} + +func TestBridgedVotesRepository_ApplyAggregateZeroAsOfIsAnError(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + + // Counts and their stamp form one atomic validated trio. The client never + // produces a zero AsOf, so one reaching the store is a caller bug and must + // surface as an error rather than a silent no-op that marks the subject polled. + err := NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.postURI, Upvotes: 5, Downvotes: 2, + }) + require.ErrorIs(t, err, bridgedvotes.ErrMissingAsOf) + requireAggregateStats(t, ctx, db, "posts", fixture.postURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 0, bridgedDown: 0, score: 2, + }) +} + +type applyAggregateFixture struct { + postURI string + commentURI string + deletedPostURI string +} + +func seedApplyAggregateFixture(t *testing.T, ctx context.Context, db *sql.DB) applyAggregateFixture { + t.Helper() + + const communityDID = "did:plc:applyaggregatecommunity" + fixture := applyAggregateFixture{ + postURI: "at://did:plc:applyaggregateauthor/social.coves.community.postv2/post", + commentURI: "at://did:plc:applyaggregatecommenter/social.coves.community.comment/comment", + deletedPostURI: "at://did:plc:applyaggregatedeleted/social.coves.community.postv2/deleted", + } + now := time.Now().UTC() + deletedAt := now.Add(-time.Minute) + + _, err := db.ExecContext(ctx, ` + INSERT INTO communities + (did, handle, name, owner_did, created_by_did, hosted_by_did, pds_url, created_at) + VALUES + ($1, '!apply-aggregate@local.test', 'apply-aggregate', $1, $1, $1, $2, $3) + `, communityDID, bridgeAPDSURL, now) + require.NoError(t, err, "seed apply-aggregate community") + + _, err = db.ExecContext(ctx, ` + INSERT INTO posts + (uri, cid, rkey, author_did, community_did, title, created_at, deleted_at, + upvote_count, downvote_count, score, bridged_upvote_count, bridged_downvote_count, bridged_stats_as_of) + VALUES + ($1, 'bafyapplyaggregatepost', 'post', 'did:plc:applyaggregateauthor', $3, 'post', $4, NULL, + 3, 1, 2, 0, 0, NULL), + ($2, 'bafyapplyaggregatedeleted', 'deleted', 'did:plc:applyaggregatedeleted', $3, 'deleted', $4, $5, + 3, 1, 2, 0, 0, NULL) + `, fixture.postURI, fixture.deletedPostURI, communityDID, now, deletedAt) + require.NoError(t, err, "seed apply-aggregate posts") + + _, err = db.ExecContext(ctx, ` + INSERT INTO comments + (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, created_at, + upvote_count, downvote_count, score, bridged_upvote_count, bridged_downvote_count, bridged_stats_as_of) + VALUES + ($1, 'bafyapplyaggregatecomment', 'comment', 'did:plc:applyaggregatecommenter', + $2, 'bafyapplyaggregatepost', $2, 'bafyapplyaggregatepost', 'comment', $3, + 3, 1, 2, 0, 0, NULL) + `, fixture.commentURI, fixture.postURI, now) + require.NoError(t, err, "seed apply-aggregate comment") + + return fixture +} + +func seedStoredAggregate( + t *testing.T, + ctx context.Context, + db *sql.DB, + table string, + uri string, + bridgedUp int, + bridgedDown int, + asOf time.Time, +) { + t.Helper() + + query := ` + UPDATE posts + SET bridged_upvote_count = $2, + bridged_downvote_count = $3, + bridged_stats_as_of = $4, + score = (upvote_count + $2::int) - (downvote_count + $3::int) + WHERE uri = $1 + ` + if table == "comments" { + query = ` + UPDATE comments + SET bridged_upvote_count = $2, + bridged_downvote_count = $3, + bridged_stats_as_of = $4, + score = (upvote_count + $2::int) - (downvote_count + $3::int) + WHERE uri = $1 + ` + } + + result, err := db.ExecContext(ctx, query, uri, bridgedUp, bridgedDown, asOf) + require.NoError(t, err, "seed stored %s aggregate for %s", table, uri) + rows, err := result.RowsAffected() + require.NoError(t, err) + require.EqualValues(t, 1, rows, "seeded aggregate must update exactly one %s row", table) +} + +type storedAggregateStats struct { + nativeUp int + nativeDown int + bridgedUp int + bridgedDown int + score int + asOf sql.NullTime +} + +type expectedAggregateStats struct { + nativeUp int + nativeDown int + bridgedUp int + bridgedDown int + score int + asOf *time.Time +} + +func requireAggregateStats( + t *testing.T, + ctx context.Context, + db *sql.DB, + table string, + uri string, + want expectedAggregateStats, +) { + t.Helper() + + query := ` + SELECT upvote_count, downvote_count, bridged_upvote_count, + bridged_downvote_count, score, bridged_stats_as_of + FROM posts WHERE uri = $1 + ` + if table == "comments" { + query = ` + SELECT upvote_count, downvote_count, bridged_upvote_count, + bridged_downvote_count, score, bridged_stats_as_of + FROM comments WHERE uri = $1 + ` + } + + var got storedAggregateStats + require.NoError(t, db.QueryRowContext(ctx, query, uri).Scan( + &got.nativeUp, + &got.nativeDown, + &got.bridgedUp, + &got.bridgedDown, + &got.score, + &got.asOf, + ), "read %s aggregate stats for %s", table, uri) + require.Equal(t, want.nativeUp, got.nativeUp, "%s native upvotes", uri) + require.Equal(t, want.nativeDown, got.nativeDown, "%s native downvotes", uri) + require.Equal(t, want.bridgedUp, got.bridgedUp, "%s bridged upvotes", uri) + require.Equal(t, want.bridgedDown, got.bridgedDown, "%s bridged downvotes", uri) + require.Equal(t, want.score, got.score, "%s score", uri) + if want.asOf == nil { + require.False(t, got.asOf.Valid, "%s bridged stats as-of must remain NULL", uri) + return + } + require.True(t, got.asOf.Valid, "%s bridged stats as-of must be populated", uri) + require.True(t, got.asOf.Time.UTC().Equal(want.asOf.UTC()), + "%s bridged stats as-of: got %s, want %s", uri, got.asOf.Time.UTC(), want.asOf.UTC()) +} + +func TestBridgedVotesRepository_SelectCandidatesOrdersNeverPolledByCreatedAt(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedNeverPolledCandidateOrder(t, ctx, db) + repo := NewBridgedVotesRepository(db) + want := []bridgedvotes.Candidate{ + {URI: fixture.oldest, StoredPDSURL: bridgeAPDSURL}, + {URI: fixture.middle, StoredPDSURL: bridgeAPDSURL}, + {URI: fixture.newest, StoredPDSURL: bridgeAPDSURL}, + } + + got, err := repo.SelectCandidates(ctx, []string{bridgeAPDSURL}, 90*24*time.Hour, 50) + require.NoError(t, err) + require.Equal(t, want, got, + "never-polled candidates must rotate oldest-created first, then by URI") + + got, err = repo.SelectCandidates(ctx, []string{bridgeAPDSURL}, 90*24*time.Hour, 1) + require.NoError(t, err) + require.Equal(t, want[:1], got, + "a bounded sweep must deterministically choose the oldest never-polled candidate") +} + +func TestBridgedVotesRepository_ApplyAggregateCommentNewerAsOfOverwrites(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + t1 := time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC) + t2 := t1.Add(time.Minute) + seedStoredAggregate(t, ctx, db, "comments", fixture.commentURI, 5, 2, t1) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.commentURI, Upvotes: 7, Downvotes: 2, AsOf: t2, + })) + requireAggregateStats(t, ctx, db, "comments", fixture.commentURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 7, bridgedDown: 2, score: 7, asOf: &t2, + }) +} + +func TestBridgedVotesRepository_ApplyAggregateCommentEqualAsOfReapplies(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + t2 := time.Date(2026, 8, 31, 2, 5, 1, 80_000_000, time.UTC) + seedStoredAggregate(t, ctx, db, "comments", fixture.commentURI, 7, 2, t2) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.commentURI, Upvotes: 7, Downvotes: 2, AsOf: t2, + })) + requireAggregateStats(t, ctx, db, "comments", fixture.commentURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 7, bridgedDown: 2, score: 7, asOf: &t2, + }) +} + +func TestBridgedVotesRepository_ApplyAggregateCommentStaleAsOfIsNoOp(t *testing.T) { + t.Parallel() + + db := testkit.DB(t) + ctx := context.Background() + fixture := seedApplyAggregateFixture(t, ctx, db) + t1 := time.Date(2026, 8, 31, 2, 4, 1, 80_000_000, time.UTC) + t0 := t1.Add(-time.Minute) + seedStoredAggregate(t, ctx, db, "comments", fixture.commentURI, 5, 2, t1) + + require.NoError(t, NewBridgedVotesRepository(db).ApplyAggregate(ctx, bridgedvotes.Aggregate{ + URI: fixture.commentURI, Upvotes: 99, Downvotes: 41, AsOf: t0, + })) + requireAggregateStats(t, ctx, db, "comments", fixture.commentURI, expectedAggregateStats{ + nativeUp: 3, nativeDown: 1, bridgedUp: 5, bridgedDown: 2, score: 5, asOf: &t1, + }) +} + +type neverPolledCandidateOrderFixture struct { + oldest string + middle string + newest string +} + +func seedNeverPolledCandidateOrder(t *testing.T, ctx context.Context, db *sql.DB) neverPolledCandidateOrderFixture { + t.Helper() + + const communityDID = "did:plc:neverpolledorder" + fixture := neverPolledCandidateOrderFixture{ + oldest: "at://did:plc:neverpolledorder/social.coves.community.postv2/zzz-oldest", + middle: "at://did:plc:neverpolledorder/social.coves.community.postv2/mmm-middle", + newest: "at://did:plc:neverpolledorder/social.coves.community.postv2/aaa-newest", + } + now := time.Now().UTC() + + _, err := db.ExecContext(ctx, ` + INSERT INTO communities + (did, handle, name, owner_did, created_by_did, hosted_by_did, pds_url, created_at) + VALUES + ($1, '!never-polled-order@local.test', 'never-polled-order', $1, $1, $1, $2, $3) + `, communityDID, bridgeAPDSURL, now) + require.NoError(t, err, "seed never-polled ordering community") + + // Deliberately insert newest, oldest, then middle. With every watermark NULL, + // heap/insertion order differs from the required created_at order. + _, err = db.ExecContext(ctx, ` + INSERT INTO posts + (uri, cid, rkey, author_did, community_did, title, created_at, bridged_polled_at) + VALUES + ($1, 'bafyneverpollednewest', 'aaa-newest', 'did:plc:neverpollednewest', $4, 'newest', $5, NULL), + ($2, 'bafyneverpolledoldest', 'zzz-oldest', 'did:plc:neverpolledoldest', $4, 'oldest', $6, NULL), + ($3, 'bafyneverpolledmiddle', 'mmm-middle', 'did:plc:neverpolledmiddle', $4, 'middle', $7, NULL) + `, fixture.newest, fixture.oldest, fixture.middle, communityDID, + now.Add(-time.Minute), now.Add(-time.Hour), now.Add(-30*time.Minute)) + require.NoError(t, err, "seed never-polled ordering posts") + + return fixture +} diff --git a/internal/db/postgres/future_comment_created_at_migration_test.go b/internal/db/postgres/future_comment_created_at_migration_test.go index 0c26771..85f7d66 100644 --- a/internal/db/postgres/future_comment_created_at_migration_test.go +++ b/internal/db/postgres/future_comment_created_at_migration_test.go @@ -18,6 +18,8 @@ func TestMigration041_ClampsFutureCommentCreatedAt(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 43, testkit.MigrateDownOne(t, db, 43), + "043 (the bridged-vote poll watermark) sits on top and must be rolled back first") require.EqualValues(t, 42, testkit.MigrateDownOne(t, db, 42), "042 (the dead-letter retention index) sits on top of 041 and must be rolled back first") require.EqualValues(t, 41, testkit.MigrateDownOne(t, db, 41), diff --git a/internal/db/postgres/rematerialize_ledger_schema_test.go b/internal/db/postgres/rematerialize_ledger_schema_test.go index 6aec8cd..bcff7e6 100644 --- a/internal/db/postgres/rematerialize_ledger_schema_test.go +++ b/internal/db/postgres/rematerialize_ledger_schema_test.go @@ -198,6 +198,8 @@ func TestRematerializeLedgerMigration_RollsBack(t *testing.T) { // top of 037 and come off first, one asserted step at a time. Asserting which // migration rolled back is what keeps this pointed at 037's Down rather than // drifting onto a newer one later. + require.EqualValues(t, 43, testkit.MigrateDownOne(t, db, 43), + "043 (the bridged-vote poll watermark) sits on top and must be rolled back first") require.EqualValues(t, 42, testkit.MigrateDownOne(t, db, 42), "042 (the dead-letter retention index) sits on top of 041 and must be rolled back first") require.EqualValues(t, 41, testkit.MigrateDownOne(t, db, 41), diff --git a/internal/db/postgres/vote_drift_recount_migration_test.go b/internal/db/postgres/vote_drift_recount_migration_test.go index 930252a..e3ea578 100644 --- a/internal/db/postgres/vote_drift_recount_migration_test.go +++ b/internal/db/postgres/vote_drift_recount_migration_test.go @@ -80,6 +80,8 @@ func TestMigration040_RecountsVoteDriftAndSweepsLegacyOrphans(t *testing.T) { // point of a repair migration and cannot be observed by seeding after it has // run. Asserting the version that came off is the tripwire that keeps this // pointed at 040 when later migrations land. + require.EqualValues(t, 43, testkit.MigrateDownOne(t, db, 43), + "043 (the bridged-vote poll watermark) sits on top and must be rolled back first") require.EqualValues(t, 42, testkit.MigrateDownOne(t, db, 42), "042 (the dead-letter retention index) sits on top of 041 and must be rolled back first") require.EqualValues(t, 41, testkit.MigrateDownOne(t, db, 41), -- 2.51.2