From 01c3bc391933546d4fca2bbeebf5bc7bf8cfdb1c Mon Sep 17 00:00:00 2001 From: Bretton Date: Fri, 2 Oct 2026 00:28:13 -0700 Subject: [PATCH] feat(moderation): durable CDN purge targets with retry sweep Make chunk 15's Cloudflare edge purge durable. With purge configured, a removal records its owner-scoped (owner DID, CID) blocks as purge targets and retries them until a purge lands at least HTTP_WRITE_TIMEOUT after the block committed. A purge failure still never fails a removal, and with nothing configured there are no target rows, no sweep and no outbound request. - Migration 052 adds moderation_media_purges: one row per (owner DID, CID) with pending/completed state, attempts, next attempt time, earliest completion time and last failure code. Head-down migration test chains gain MigrateDownOne(052). - Targets are written in the transaction that inserts the block: the removal transaction and the consumer's reconcile transaction (MediaTransaction). A new block on a completed pair re-pends it; restore writes no targets. - After commit, removeContent and MediaReconciler.Purge attempt due targets once, replacing the direct purger call. Rows are claimed with a 2-minute lease so concurrent workers skip them and a crash leaves them due again; attempts are bounded and outcomes are written after cancellation. - Completion requires a success at or after the earliest completion time (an early success schedules one more purge) and is monotonic. Failures back off exponentially from 1 minute to a 1-hour cap and keep the DID/CID/failure-code Error line. - cmd/server wiring builds the retry sweep only when purge is configured: once at boot, then every minute, stopping on shutdown. - T1 coverage for durable targets, claims, racing sweeps, late failures and restart recovery against a fake Cloudflare endpoint on an injected clock; PRD_CSAM_SCANNING WS4/WS5 updated. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/server/cdn_purge_wiring.go | 61 +- cmd/server/cdn_purge_wiring_test.go | 249 ++++++-- cmd/server/main.go | 17 +- cmd/server/wiring.go | 16 +- docs/PRD_CSAM_SCANNING.md | 4 +- .../moderation_cdn_purge_integration_test.go | 604 ++++++++++++++++++ ...ration_cdn_purge_retry_integration_test.go | 364 +++++++++++ .../moderation_media_integration_test.go | 95 +-- .../moderation_post_media_integration_test.go | 82 ++- .../atproto/jetstream/comment_consumer.go | 4 +- internal/atproto/jetstream/post_consumer.go | 2 +- .../post_moderation_consumer_test.go | 216 +++++-- internal/core/moderation/cdn_purge.go | 23 +- .../cdn_purge_retry_internal_test.go | 26 + internal/core/moderation/cdn_purge_targets.go | 314 +++++++++ internal/core/moderation/cdn_purge_test.go | 344 ---------- internal/core/moderation/fake_store_test.go | 8 + internal/core/moderation/media.go | 34 +- internal/core/moderation/post_rules_test.go | 7 +- internal/core/moderation/remove.go | 15 +- internal/core/moderation/remove_rules_test.go | 8 - internal/core/moderation/store.go | 3 + internal/core/moderation/types.go | 2 +- .../052_moderation_media_purges.sql | 21 + .../db/postgres/admission_repo_schema_test.go | 2 + ...unity_subscriber_recount_migration_test.go | 2 + .../credential_reencrypt_migration_test.go | 4 + .../postgres/discover_hot_migration_test.go | 2 + ...uture_comment_created_at_migration_test.go | 2 + ...moderation_concurrency_integration_test.go | 4 +- internal/db/postgres/moderation_repo.go | 120 ++++ .../read_path_indexes_migration_test.go | 2 + .../rematerialize_ledger_schema_test.go | 2 + .../vote_drift_recount_migration_test.go | 2 + tests/testkit/cloudflare_purge.go | 46 ++ 35 files changed, 2120 insertions(+), 587 deletions(-) create mode 100644 internal/api/routes/moderation_cdn_purge_integration_test.go create mode 100644 internal/api/routes/moderation_cdn_purge_retry_integration_test.go create mode 100644 internal/core/moderation/cdn_purge_retry_internal_test.go create mode 100644 internal/core/moderation/cdn_purge_targets.go delete mode 100644 internal/core/moderation/cdn_purge_test.go create mode 100644 internal/db/migrations/052_moderation_media_purges.sql diff --git a/cmd/server/cdn_purge_wiring.go b/cmd/server/cdn_purge_wiring.go index 22a9ae9..a5ce30a 100644 --- a/cmd/server/cdn_purge_wiring.go +++ b/cmd/server/cdn_purge_wiring.go @@ -1,8 +1,12 @@ package main import ( + "context" + "errors" "fmt" "log/slog" + "reflect" + "sync" "time" "Coves/internal/config" @@ -16,22 +20,21 @@ const cloudflareAPIBase = "https://api.cloudflare.com/client/v4" // coves:allow- // moderationDependencies are the storage, media and resolution seams the // moderation service and media reconciler are built from. type moderationDependencies struct { - subjectReader moderation.SubjectReader - store moderation.Store - mediaBinder moderation.TransactionBinder - mediaPurger moderation.MediaPurger - communityResolver moderation.CommunityResolver - handleResolver moderation.HandleResolver + subjectReader moderation.SubjectReader + store moderation.Store + mediaBinder moderation.TransactionBinder + mediaPurger moderation.MediaPurger + communityResolver moderation.CommunityResolver + handleResolver moderation.HandleResolver + cdnPurgeTargetStore moderation.CDNPurgeTargetStore } -// buildModeration builds the moderation service and the media reconciler, and -// gives both the CDN purger when Cloudflare purge is configured: the service -// purges after a removal commits, the reconciler after a consumer reconcile -// commits. apiBase is cloudflareAPIBase in production. -func buildModeration(cfg *config.Config, apiBase string, dependencies moderationDependencies) (moderation.Service, *moderation.MediaReconciler, error) { +// buildModeration shares the same durable queue between admin removals and +// consumer reconciliation when Cloudflare purge is configured. +func buildModeration(cfg *config.Config, apiBase string, dependencies moderationDependencies) (moderation.Service, *moderation.MediaReconciler, *moderation.CDNPurgeQueue, error) { cdnPurger, err := buildCDNPurger(cfg.Media.CDNPurge, apiBase) if err != nil { - return nil, nil, fmt.Errorf("creating CDN purger: %w", err) + return nil, nil, nil, fmt.Errorf("creating CDN purger: %w", err) } moderationConfig := moderation.Config{ InstanceDID: cfg.Instance.DID, @@ -43,14 +46,20 @@ func buildModeration(cfg *config.Config, apiBase string, dependencies moderation HandleResolver: dependencies.handleResolver, } var reconcilerOptions []moderation.MediaReconcilerOption + var queue *moderation.CDNPurgeQueue if cdnPurger != nil { - moderationConfig.CDNPurger = cdnPurger - reconcilerOptions = append(reconcilerOptions, moderation.WithCDNPurger(cdnPurger)) + if dependencies.cdnPurgeTargetStore == nil { + return nil, nil, nil, errors.New("CDN purge target store is required when CDN purge is configured") + } + queue = moderation.NewCDNPurgeQueue(dependencies.cdnPurgeTargetStore, cdnPurger, + moderation.CDNPurgeQueueConfig{WriteTimeout: cfg.Server.WriteTimeout}) + moderationConfig.CDNPurgeTargets = queue + reconcilerOptions = append(reconcilerOptions, moderation.WithCDNPurgeTargets(queue)) } service := moderation.NewService(dependencies.subjectReader, dependencies.store, moderationConfig) reconciler := moderation.NewMediaReconciler( dependencies.mediaBinder, cfg.Instance.DID, dependencies.mediaPurger, reconcilerOptions...) - return service, reconciler, nil + return service, reconciler, queue, nil } // buildCDNPurger returns nil when no Cloudflare edge invalidation is configured. @@ -67,3 +76,25 @@ func buildCDNPurger(purge config.CDNPurgeConfig, apiBase string) (moderation.CDN Timeout: 10 * time.Second, }) } + +// cdnPurgeSweepInterval is how often the CDN retry sweep runs after boot. +const cdnPurgeSweepInterval = time.Minute + +// cdnPurgeSweeper processes due durable CDN purge targets. +type cdnPurgeSweeper interface { + Sweep(ctx context.Context) error +} + +// startCDNPurgeSweepJob retries due targets at boot and each interval. +func startCDNPurgeSweepJob(ctx context.Context, waitGroup *sync.WaitGroup, sweeper cdnPurgeSweeper, interval time.Duration) { + if sweeper == nil || (reflect.ValueOf(sweeper).Kind() == reflect.Ptr && reflect.ValueOf(sweeper).IsNil()) || interval <= 0 { + return + } + runTicker(ctx, waitGroup, "cdn-purge-sweep", interval, func(ctx context.Context) { + // An ended cycle context (shutdown or the cycle deadline) surfaces as + // driver errors that need not wrap ctx.Err(); the next cycle resumes. + if err := sweeper.Sweep(ctx); err != nil && ctx.Err() == nil { + slog.Error("moderation CDN purge sweep failed", "error", err) + } + }) +} diff --git a/cmd/server/cdn_purge_wiring_test.go b/cmd/server/cdn_purge_wiring_test.go index 478611f..ed806a8 100644 --- a/cmd/server/cdn_purge_wiring_test.go +++ b/cmd/server/cdn_purge_wiring_test.go @@ -2,13 +2,14 @@ package main import ( "context" + "database/sql" + "errors" "log/slog" "sync" "testing" "time" "Coves/internal/config" - "Coves/internal/core/blobs" "Coves/internal/core/imageproxy" "Coves/internal/core/moderation" "Coves/tests/testkit" @@ -69,7 +70,8 @@ func (store *cdnWiringStore) InTransaction(ctx context.Context, fn func(context. type cdnWiringTransaction struct { moderation.Transaction - comment moderation.IndexedComment + comment moderation.IndexedComment + purgeTargets []imageproxy.BlockedBlob } func (*cdnWiringTransaction) LockActor(context.Context, string) error { return nil } @@ -105,10 +107,71 @@ func (*cdnWiringTransaction) InsertMediaBlocks(context.Context, []moderation.Med return nil } -// TestBuildModerationPurgesCloudflareOnRemovalAndReconcile pins the production -// wiring: with CDN purge configured, both the removal path (moderation service) -// and the consumer reconcile path (media reconciler) must reach Cloudflare. -func TestBuildModerationPurgesCloudflareOnRemovalAndReconcile(t *testing.T) { +func (transaction *cdnWiringTransaction) RecordCDNPurgeTargets(_ context.Context, blobs []imageproxy.BlockedBlob) error { + transaction.purgeTargets = append(transaction.purgeTargets, blobs...) + return nil +} + +func (store *cdnWiringStore) ReadSubject(_ context.Context, uri string) (*moderation.IndexedRecord, error) { + return &moderation.IndexedRecord{URI: uri, CID: store.transaction.comment.CID}, nil +} + +type cdnWiringMediaTransaction struct { + moderation.MediaTransaction + comment moderation.IndexedComment + purgeTargets []imageproxy.BlockedBlob +} + +func (*cdnWiringMediaTransaction) ActiveRemoval(context.Context, string, string) (*moderation.Action, error) { + return &moderation.Action{ID: "cdn-wiring-action"}, nil +} + +func (transaction *cdnWiringMediaTransaction) ReadIndexedComment(context.Context, string) (*moderation.IndexedComment, error) { + return &transaction.comment, nil +} + +func (*cdnWiringMediaTransaction) InsertNewMediaBlocks(_ context.Context, blocks []moderation.MediaBlock) ([]moderation.MediaBlock, error) { + return blocks, nil +} + +func (transaction *cdnWiringMediaTransaction) RecordCDNPurgeTargets(_ context.Context, blobs []imageproxy.BlockedBlob) error { + transaction.purgeTargets = append(transaction.purgeTargets, blobs...) + return nil +} + +type cdnWiringMediaBinder struct{ transaction *cdnWiringMediaTransaction } + +func (binder cdnWiringMediaBinder) BindTransaction(*sql.Tx) moderation.MediaTransaction { + return binder.transaction +} + +type cdnWiringPurgeTargetStore struct { + mu sync.Mutex + claims []moderation.CDNPurgeClaim +} + +func (store *cdnWiringPurgeTargetStore) ClaimDueCDNPurgeTargets(_ context.Context, claim moderation.CDNPurgeClaim) ([]moderation.CDNPurgeTarget, error) { + store.mu.Lock() + defer store.mu.Unlock() + store.claims = append(store.claims, claim) + return nil, nil +} + +func (*cdnWiringPurgeTargetStore) CompleteCDNPurgeTarget(context.Context, moderation.CDNPurgeTarget) error { + return nil +} + +func (*cdnWiringPurgeTargetStore) RescheduleCDNPurgeTarget(context.Context, moderation.CDNPurgeTarget, time.Time, string) error { + return nil +} + +func (store *cdnWiringPurgeTargetStore) snapshot() []moderation.CDNPurgeClaim { + store.mu.Lock() + defer store.mu.Unlock() + return append([]moderation.CDNPurgeClaim(nil), store.claims...) +} + +func TestBuildModerationRecordsCDNPurgeTargetsAndConfiguresQueue(t *testing.T) { const ( ownerDID = "did:plc:cdnwiringauthor" subjectURI = "at://did:plc:cdnwiringauthor/social.coves.community.comment/3kcdnwiring" @@ -116,44 +179,150 @@ func TestBuildModerationPurgesCloudflareOnRemovalAndReconcile(t *testing.T) { reconciledBlob = "bafyreicdnwiringreconciled" imageBaseURL = "https://img.cdn-wiring.test" ) - endpoint := testkit.NewCloudflarePurgeEndpoint(t) - cfg := &config.Config{ - Instance: config.InstanceConfig{DID: "did:web:cdn-wiring.test"}, - Moderation: config.ModerationConfig{IdempotencyRetention: time.Hour, MaxLiveIdempotencyKeys: 10}, - Media: config.MediaConfig{ - ImageProxy: imageproxy.Config{BaseURL: imageBaseURL}, - CDNPurge: config.CDNPurgeConfig{ZoneID: "zone-cdn-wiring", APIToken: "token-cdn-wiring", BaseURLs: []string{imageBaseURL}}, - }, + for _, configured := range []bool{true, false} { + name := "unconfigured" + if configured { + name = "configured" + } + t.Run(name, func(t *testing.T) { + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + cfg := &config.Config{ + Instance: config.InstanceConfig{DID: "did:web:cdn-wiring.test"}, + Moderation: config.ModerationConfig{IdempotencyRetention: time.Hour, MaxLiveIdempotencyKeys: 10}, + Media: config.MediaConfig{ImageProxy: imageproxy.Config{BaseURL: imageBaseURL}}, + } + cfg.Server.WriteTimeout = 77 * time.Second + if configured { + cfg.Media.CDNPurge = config.CDNPurgeConfig{ZoneID: "zone-cdn-wiring", APIToken: "token-cdn-wiring", BaseURLs: []string{imageBaseURL}} + } + comment := moderation.IndexedComment{URI: subjectURI, CID: "bafyreicdnwiringsubject", OwnerDID: ownerDID, ImageCIDs: []string{removedImage}} + store := &cdnWiringStore{transaction: &cdnWiringTransaction{comment: comment}} + media := &cdnWiringMediaTransaction{comment: moderation.IndexedComment{URI: subjectURI, OwnerDID: ownerDID, ImageCIDs: []string{reconciledBlob}}} + targets := &cdnWiringPurgeTargetStore{} + service, reconciler, queue, err := buildModeration(cfg, endpoint.URL(), moderationDependencies{ + subjectReader: store, store: store, mediaBinder: cdnWiringMediaBinder{transaction: media}, cdnPurgeTargetStore: targets, + }) + require.NoError(t, err) + if configured { + assert.NotNil(t, queue, "configured CDN purge needs a durable queue") + } else { + assert.Nil(t, queue) + } + + result, err := service.RemoveContent(t.Context(), "did:plc:cdnwiringadmin", moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: subjectURI, CID: comment.CID}, + ExpectedVersion: moderation.InitialVersion, IdempotencyKey: "cdn-wiring-removal", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + blocks, err := reconciler.ReconcileTx(t.Context(), nil, subjectURI) + require.NoError(t, err) + require.Equal(t, []moderation.MediaBlock{{OwnerDID: ownerDID, BlobCID: reconciledBlob, ActionID: "cdn-wiring-action"}}, blocks) + if configured { + assert.Equal(t, []imageproxy.BlockedBlob{{OwnerDID: ownerDID, CID: removedImage}}, store.transaction.purgeTargets) + assert.Equal(t, []imageproxy.BlockedBlob{{OwnerDID: ownerDID, CID: reconciledBlob}}, media.purgeTargets) + if queue != nil { + queueCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + require.NoError(t, queue.Wait(queueCtx)) + beforeSweep := len(targets.snapshot()) + require.NoError(t, queue.Sweep(queueCtx)) + claims := targets.snapshot() + require.Greater(t, len(claims), beforeSweep) + assert.Equal(t, 77*time.Second, claims[beforeSweep].WriteTimeout) + assert.Equal(t, 2*time.Minute, claims[beforeSweep].Lease) + } + } else { + assert.Empty(t, store.transaction.purgeTargets) + assert.Empty(t, media.purgeTargets) + } + }) } - store := &cdnWiringStore{transaction: &cdnWiringTransaction{comment: moderation.IndexedComment{ - URI: subjectURI, CID: "bafyreicdnwiringsubject", OwnerDID: ownerDID, ImageCIDs: []string{removedImage}, - }}} +} - service, reconciler, err := buildModeration(cfg, endpoint.URL(), moderationDependencies{store: store}) - require.NoError(t, err) +type cdnWiringSweeper struct{ called chan struct{} } + +func (sweeper *cdnWiringSweeper) Sweep(context.Context) error { + sweeper.called <- struct{}{} + return nil +} - presets := imageproxy.ListPresets() - require.NotEmpty(t, presets) - assertPurged := func(t *testing.T, requests []testkit.CloudflarePurgeRequest, cid string) { - t.Helper() - require.Len(t, requests, 1) - assert.Equal(t, "/zones/zone-cdn-wiring/purge_cache", requests[0].Path) - assert.Equal(t, "Bearer token-cdn-wiring", requests[0].Authorization) - require.NoError(t, requests[0].DecodeError) - assert.Contains(t, requests[0].Files, blobs.HydrateImageProxyURL(imageBaseURL, presets[0].Name, ownerDID, cid)) +func TestStartCDNPurgeSweepJobNilSweeper(t *testing.T) { + for _, test := range []struct { + name string + sweeper cdnPurgeSweeper + }{ + {name: "nil interface"}, + {name: "typed nil queue", sweeper: (*moderation.CDNPurgeQueue)(nil)}, + } { + t.Run(test.name, func(t *testing.T) { + var waitGroup sync.WaitGroup + startCDNPurgeSweepJob(t.Context(), &waitGroup, test.sweeper, time.Hour) + requireCDNPurgeSweepJobStops(t, &waitGroup) + }) } +} - result, err := service.RemoveContent(t.Context(), "did:plc:cdnwiringadmin", moderation.RemoveContentRequest{ - Subject: moderation.StrongRef{URI: subjectURI, CID: "bafyreicdnwiringsubject"}, - ExpectedVersion: moderation.InitialVersion, IdempotencyKey: "cdn-wiring-removal", - Reason: "social.coves.moderation.defs#reasonSpam", - }) - require.NoError(t, err) - require.Equal(t, moderation.OutcomeApplied, result.Outcome) - assertPurged(t, endpoint.Requests(), removedImage) +func TestStartCDNPurgeSweepJobBootAndCancel(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var waitGroup sync.WaitGroup + sweeper := &cdnWiringSweeper{called: make(chan struct{}, 2)} + startCDNPurgeSweepJob(ctx, &waitGroup, sweeper, time.Hour) + select { + case <-sweeper.called: + case <-time.After(10 * time.Second): + t.Fatal("CDN purge sweep did not run on boot") + } + cancel() + requireCDNPurgeSweepJobStops(t, &waitGroup) + assert.Empty(t, sweeper.called) +} + +func requireCDNPurgeSweepJobStops(t *testing.T, waitGroup *sync.WaitGroup) { + t.Helper() + done := make(chan struct{}) + go func() { + waitGroup.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(10 * time.Second): + t.Fatal("CDN purge sweep job did not stop") + } +} + +// cdnInterruptedSweeper fails the way lib/pq reports a statement cancelled by +// the caller's context: with an error that does not wrap context.Canceled. +type cdnInterruptedSweeper struct{ entered chan struct{} } + +func (sweeper *cdnInterruptedSweeper) Sweep(ctx context.Context) error { + close(sweeper.entered) + <-ctx.Done() + return errors.New("pq: canceling statement due to user request") +} - reconciler.Purge([]moderation.MediaBlock{{OwnerDID: ownerDID, BlobCID: reconciledBlob, ActionID: "cdn-wiring-action"}}) - requests := endpoint.Requests() - require.Len(t, requests, 2, "the media reconciler must purge Cloudflare after a reconcile commit") - assertPurged(t, requests[1:], reconciledBlob) +// Not parallel: this test replaces slog.Default while the job runs. +func TestStartCDNPurgeSweepJobEndedContextIsNotAnError(t *testing.T) { + previous := slog.Default() + handler := &cdnBootLogHandler{} + slog.SetDefault(slog.New(handler)) + t.Cleanup(func() { slog.SetDefault(previous) }) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var waitGroup sync.WaitGroup + sweeper := &cdnInterruptedSweeper{entered: make(chan struct{})} + startCDNPurgeSweepJob(ctx, &waitGroup, sweeper, time.Hour) + select { + case <-sweeper.entered: + case <-time.After(10 * time.Second): + t.Fatal("CDN purge sweep did not run on boot") + } + cancel() + requireCDNPurgeSweepJobStops(t, &waitGroup) + for _, record := range handler.snapshot() { + assert.NotEqual(t, slog.LevelError, record.Level, "a sweep stopped by its context is not a failure") + } } diff --git a/cmd/server/main.go b/cmd/server/main.go index 23008e7..3d21a0e 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -117,6 +117,9 @@ func run() error { discoverHotCleanupInterval, discoverHotCleanupDerivedRowBatchSize) startModerationIdempotencySweepJob(backgroundCtx, &backgroundWG, postgresRepo.NewModerationRepository(db), moderationIdempotencySweepInterval) + if app.cdnPurgeQueue != nil { + startCDNPurgeSweepJob(backgroundCtx, &backgroundWG, app.cdnPurgeQueue, cdnPurgeSweepInterval) + } // Nil when the driver is disabled, and passed as a typed nil would be a // non-nil interface — so the guard is here rather than inside the job. @@ -217,6 +220,10 @@ func serve( case <-ctx.Done(): slog.Info("shutdown signal received") } + if app.cdnPurgeQueue != nil { + // Commits from here on leave their CDN purge targets to the next sweep. + app.cdnPurgeQueue.Close() + } // Drain background work concurrently with the listener drain, not after // it. They are independent — background work runs on its own context, @@ -242,9 +249,17 @@ func serve( slog.Error("HTTP server shutdown error", "error", err) } } - drained := <-drainResult + if app.cdnPurgeQueue != nil { + // Best effort: an attempt cut off here keeps its durable target, which + // the boot sweep retries once its lease expires. + if err := app.cdnPurgeQueue.Wait(shutdownCtx); err != nil { + slog.Warn("CDN purge attempts still in flight at shutdown", + "in_flight", app.cdnPurgeQueue.InFlight(), "error", err) + } + } + // Stop the image proxy cleanup job here rather than leaving it to run's // deferred Close, so it does not outlive the drain. Close is idempotent, // so the deferred call is a no-op. diff --git a/cmd/server/wiring.go b/cmd/server/wiring.go index d5a0b4f..738359a 100644 --- a/cmd/server/wiring.go +++ b/cmd/server/wiring.go @@ -140,6 +140,7 @@ type application struct { adminReportService adminreports.Service moderationService moderation.Service mediaReconciler *moderation.MediaReconciler + cdnPurgeQueue *moderation.CDNPurgeQueue communitySuggestionService communitysuggestions.Service feedService communityFeeds.Service timelineService timeline.Service @@ -212,13 +213,14 @@ func buildApplication( return nil, err } moderationStore := postgresRepo.NewModerationRepository(app.db) - app.moderationService, app.mediaReconciler, err = buildModeration(app.cfg, cloudflareAPIBase, moderationDependencies{ - subjectReader: moderation.NewRepositorySubjectReader(app.postRepo, app.commentRepo), - store: moderationStore, - mediaBinder: moderationStore, - mediaPurger: purger, - communityResolver: app.communityService, - handleResolver: app.identityResolver, + app.moderationService, app.mediaReconciler, app.cdnPurgeQueue, err = buildModeration(app.cfg, cloudflareAPIBase, moderationDependencies{ + subjectReader: moderation.NewRepositorySubjectReader(app.postRepo, app.commentRepo), + store: moderationStore, + mediaBinder: moderationStore, + mediaPurger: purger, + communityResolver: app.communityService, + handleResolver: app.identityResolver, + cdnPurgeTargetStore: moderationStore, }) if err != nil { return nil, err diff --git a/docs/PRD_CSAM_SCANNING.md b/docs/PRD_CSAM_SCANNING.md index 3c04c98..fbb1778 100644 --- a/docs/PRD_CSAM_SCANNING.md +++ b/docs/PRD_CSAM_SCANNING.md @@ -89,7 +89,7 @@ Build an admin takedown flow (CLI or admin endpoint), input = DID + CID: - Delete the record + blob from the owning PDS (native: PDS admin API; bridged: tidepool bridge admin path). - Remove/tombstone the post in the AppView index. - Purge the imageproxy disk cache for **all presets** of that DID+CID (add a purge-by-blob admin method to `imageproxy.DiskCache` — cache keys are preset-scoped). - - Purge the Cloudflare edge cache by URL for each preset variant (single-file purge is available on free plans). Cloudflare's own block covers the exact matched URL; we purge the sibling preset URLs. Moderation removal already purges the imageproxy disk cache and, when configured, the edge through `imageproxy.CloudflarePurger`. Set both `CLOUDFLARE_CACHE_PURGE_ZONE_ID` and `CLOUDFLARE_CACHE_PURGE_API_TOKEN`; the token needs Zone → Cache Purge → Purge on the `coves.social` zone only, separate from Caddy's `CLOUDFLARE_API_TOKEN`. The DID+CID takedown tooling can reuse this purger. + - Purge the Cloudflare edge cache by URL for each preset variant (single-file purge is available on free plans). Cloudflare's own block covers the exact matched URL; we purge the sibling preset URLs. Moderation removal already purges the imageproxy disk cache and, when configured, records durable owner/CID purge targets in the removal transaction. It attempts an edge purge after commit, retries failures, and makes a final purge no earlier than `HTTP_WRITE_TIMEOUT` after the block committed. That bounds how long the origin can still write a response authorized before the commit; it does not bound downstream buffering at the edge. Set both `CLOUDFLARE_CACHE_PURGE_ZONE_ID` and `CLOUDFLARE_CACHE_PURGE_API_TOKEN`; the token needs Zone → Cache Purge → Purge on the `coves.social` zone only, separate from Caddy's `CLOUDFLARE_API_TOKEN`. The DID+CID takedown tooling can reuse this purger. 4. **Report**: file our own NCMEC CyberTipline report (Cloudflare's third-party report does not replace the provider's own obligation). 5. **Act on the source**: ban the native account, or for bridged content: report to the origin Lemmy instance's admins and, on repeat, drop the instance at the bridge (instance blocklist) — this is where "rely on Lemmy moderation" plugs in. 6. **Log** the entire action (who, what, when) to an audit table. Never log or store the image content outside the preservation store. @@ -106,8 +106,6 @@ Phase 1 can be a documented manual runbook using existing tools (psql, PDS admin | Video blobs unscanned | Image proxy is stills-only | Track as separate workstream | | Browser caches, and any shared cache nobody purges, keep a removed image for up to one day | A moderation removal purges the proxy's disk cache and optionally Cloudflare's edge; a copy already served with `Cache-Control: public, max-age=86400` stays valid in browser caches and any shared cache not purged | The one-day `max-age` bounds the exposure without any configuration, including for self-hosters with no CDN | | Other owners' edge copies of an `illegal-content` CID are not purged | The edge purge names only the owner of each owner-scoped block, not the other owners covered by an ownerless block | The one-day cache lifetime bounds the gap; a planned follow-up names the other owners | -| A failed edge purge is not retried | The removal logs the failure but does not queue a retry | The one-day cache lifetime bounds the gap; a planned follow-up adds durable purge targets and retries | -| An in-flight response can reach the edge after its purge | A response authorized before the removal commits can still be written after the edge purge | The one-day cache lifetime bounds the gap; a planned follow-up adds a final purge after `HTTP_WRITE_TIMEOUT` | | Responses cached under the pre-deploy `public, max-age=31536000, immutable` header | Copies stored before the one-day header shipped keep their original lifetime | Browsers keep them for up to a year and nothing server-side can reach them; the edge keeps them until a one-time Purge Everything of the `coves.social` zone after deploy (Rollout order step 4; WS3 proxies only `img.coves.social`, so this drops only cached images) | | Bridge PDS stores blobs regardless of scanning | Blobs land before any serve-time scan | Phase 2 ingest scanning; instance allow/blocklist at the bridge is the coarse control | | `record.embed` still carries blob references | Post and comment responses include the verbatim atproto record, whose embed is unprojected by design (the lexicon calls it verbatim). A client *could* build a `getBlob` URL from it | Neither client reads `record.embed` today. Coves image URLs use the proxy, with the explicit foreign Bluesky CDN exception below; this does not mean no blob reference reaches a client. The same record bytes are public on the PDS regardless. Revisit if a client starts reading it | diff --git a/internal/api/routes/moderation_cdn_purge_integration_test.go b/internal/api/routes/moderation_cdn_purge_integration_test.go new file mode 100644 index 0000000..da0480c --- /dev/null +++ b/internal/api/routes/moderation_cdn_purge_integration_test.go @@ -0,0 +1,604 @@ +//go:build integration + +package routes_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "log/slog" + "net/http" + "sync" + "testing" + "time" + + "Coves/internal/core/imageproxy" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const cdnPurgeSpam = "social.coves.moderation.defs#reasonSpam" + +type moderationCDNClock struct { + mu sync.Mutex + at time.Time +} + +func (clock *moderationCDNClock) Now() time.Time { + clock.mu.Lock() + defer clock.mu.Unlock() + return clock.at +} + +func (clock *moderationCDNClock) Set(at time.Time) { + clock.mu.Lock() + defer clock.mu.Unlock() + clock.at = at +} + +func newModerationCDNPurgeHarness(t *testing.T, endpoint *testkit.CloudflarePurgeEndpoint, clock *moderationCDNClock, writeTimeout time.Duration, wrapStore ...func(moderation.Store) moderation.Store) *moderationMediaHarness { + t.Helper() + purger := newMediaCloudflarePurger(t, endpoint, "route-cdn-token-SENTINEL") + h, _ := newModerationMediaHarness(t, false, func(options *moderationMediaHarnessConfig) { + options.now = clock.Now + options.targets = func(db *sql.DB, now func() time.Time) moderation.CDNPurgeTargets { + return moderation.NewCDNPurgeQueue(postgres.NewModerationRepository(db), purger, moderation.CDNPurgeQueueConfig{ + WriteTimeout: writeTimeout, Now: now, BatchSize: 100, + }) + } + if len(wrapStore) != 0 { + options.wrapStore = wrapStore[0] + } + }) + require.NotNil(t, h.queue) + return h +} + +func moderationCDNTarget(t *testing.T, db *sql.DB, owner, cid string) (target struct { + state, failureCode string + attempts int + nextAttemptAt time.Time +}) { + t.Helper() + var failure sql.NullString + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT state, attempts, next_attempt_at, last_failure_code + FROM moderation_media_purges WHERE owner_did = $1 AND blob_cid = $2 + `, owner, cid).Scan(&target.state, &target.attempts, &target.nextAttemptAt, &failure)) + target.failureCode = failure.String + return target +} + +func moderationCDNTargetCount(t *testing.T, db *sql.DB) int { + t.Helper() + var count int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_media_purges`).Scan(&count)) + return count +} + +// These are the Cloudflare files for the six configured image presets, written +// from the fixture base, owner DID and CID rather than using the URL hydrator. +func moderationCDNFiles(owner, cid string) []string { + return []string{ + "https://img.example.test/img/avatar/plain/" + owner + "/" + cid, + "https://img.example.test/img/avatar_small/plain/" + owner + "/" + cid, + "https://img.example.test/img/banner/plain/" + owner + "/" + cid, + "https://img.example.test/img/content_preview/plain/" + owner + "/" + cid, + "https://img.example.test/img/content_full/plain/" + owner + "/" + cid, + "https://img.example.test/img/embed_thumbnail/plain/" + owner + "/" + cid, + } +} + +func awaitModerationCDN[T any](t *testing.T, event string, ch <-chan T) T { + t.Helper() + select { + case result := <-ch: + return result + case <-time.After(10 * time.Second): + t.Fatalf("timed out waiting for %s", event) + var zero T + return zero + } +} + +func waitModerationCDN(t *testing.T, queue *moderation.CDNPurgeQueue) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + require.NoError(t, queue.Wait(ctx)) +} + +// The second handoff blocks after the removal commits, before its immediate +// attempt can claim the newly re-pended target. +type gatedModerationCDNTargets struct { + queue *moderation.CDNPurgeQueue + mu sync.Mutex + calls int + entered chan struct{} + release chan struct{} +} + +func (targets *gatedModerationCDNTargets) PurgeAfterCommit(ctx context.Context, blobs []imageproxy.BlockedBlob) { + targets.mu.Lock() + targets.calls++ + second := targets.calls == 2 + targets.mu.Unlock() + if second { + close(targets.entered) + <-targets.release + } + targets.queue.PurgeAfterCommit(ctx, blobs) +} + +func TestModerationCDNPurgeRependDuringInflightClaim(t *testing.T) { + for _, testCase := range []struct { + name string + status int + body string + }{ + {name: "held success cannot complete new generation", status: http.StatusOK, body: `{"success":true}`}, + {name: "held failure cannot reschedule new claim", status: http.StatusInternalServerError, body: `{"success":false}`}, + } { + t.Run(testCase.name, func(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + gate := &gatedModerationCDNTargets{entered: make(chan struct{}), release: make(chan struct{})} + var releaseGate sync.Once + releaseFresh := func() { releaseGate.Do(func() { close(gate.release) }) } + t.Cleanup(releaseFresh) + purger := newMediaCloudflarePurger(t, endpoint, "route-cdn-token-SENTINEL") + h, _ := newModerationMediaHarness(t, false, func(options *moderationMediaHarnessConfig) { + options.now = clock.Now + options.targets = func(db *sql.DB, now func() time.Time) moderation.CDNPurgeTargets { + gate.queue = moderation.NewCDNPurgeQueue(postgres.NewModerationRepository(db), purger, moderation.CDNPurgeQueueConfig{ + WriteTimeout: 90 * time.Second, Now: now, BatchSize: 100, + }) + return gate + } + }) + h.queue = gate.queue + imageCID := mediaImageCID("cdn repend during claim " + testCase.name) + subject := h.comment(t, h.ownerA, imageCID) + removed := h.remove(t, subject, cdnPurgeSpam) + waitModerationCDN(t, h.queue) + initial := moderationCDNTarget(t, h.db, h.ownerA, imageCID) + require.Equal(t, "pending", initial.state) + require.Equal(t, 0, initial.attempts) + require.Equal(t, time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC), initial.nextAttemptAt) + require.Len(t, endpoint.Requests(), 1) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[0].Files) + + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + endpoint.QueueResponse(testCase.status, testCase.body) + entered, releaseHeld := endpoint.HoldNextRequest(t) + swept := make(chan error, 1) + go func() { swept <- h.queue.Sweep(t.Context()) }() + awaitModerationCDN(t, "held sweep request", entered) + clock.Set(time.Date(2026, 9, 28, 12, 3, 0, 0, time.UTC)) + restored := h.restore(t, subject, removed) + fresh := make(chan error, 1) + go func() { + result, err := h.moderation.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: restored.State.Version, + IdempotencyKey: "remove-" + testkit.UniqueID(t), Reason: cdnPurgeSpam, + }) + if err == nil && (result == nil || result.Outcome != moderation.OutcomeApplied) { + err = errors.New("fresh removal did not apply") + } + fresh <- err + }() + awaitModerationCDN(t, "committed fresh removal before immediate attempt", gate.entered) + // A second connection sees the reset target while the new handoff is gated. + var state, nextAttempt, failureCode string + var attempts int + require.NoError(t, h.db.QueryRowContext(t.Context(), ` + SELECT state, attempts, next_attempt_at::text, COALESCE(last_failure_code, '') + FROM moderation_media_purges WHERE owner_did = $1 AND blob_cid = $2 + `, h.ownerA, imageCID).Scan(&state, &attempts, &nextAttempt, &failureCode)) + require.Equal(t, "pending", state) + require.Equal(t, 0, attempts) + require.Equal(t, "-infinity", nextAttempt) + require.Empty(t, failureCode) + require.Len(t, endpoint.Requests(), 2, "the fresh immediate attempt is still gated") + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[1].Files) + + releaseHeld() + require.NoError(t, awaitModerationCDN(t, "stale sweep to return", swept)) + require.NoError(t, h.db.QueryRowContext(t.Context(), ` + SELECT state, attempts, next_attempt_at::text, COALESCE(last_failure_code, '') + FROM moderation_media_purges WHERE owner_did = $1 AND blob_cid = $2 + `, h.ownerA, imageCID).Scan(&state, &attempts, &nextAttempt, &failureCode)) + assert.Equal(t, "pending", state) + assert.Equal(t, 0, attempts) + assert.Equal(t, "-infinity", nextAttempt) + assert.Empty(t, failureCode) + assert.Len(t, endpoint.Requests(), 2) + + releaseFresh() + require.NoError(t, awaitModerationCDN(t, "fresh removal to return", fresh)) + waitModerationCDN(t, h.queue) + repended := moderationCDNTarget(t, h.db, h.ownerA, imageCID) + require.Equal(t, "pending", repended.state) + assert.Equal(t, 0, repended.attempts) + assert.Empty(t, repended.failureCode) + assert.Equal(t, time.Date(2026, 9, 28, 12, 4, 30, 0, time.UTC), repended.nextAttemptAt) + require.Len(t, endpoint.Requests(), 3) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[2].Files) + clock.Set(time.Date(2026, 9, 28, 12, 4, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 4) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[3].Files) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + }) + } +} + +func TestModerationCDNPurgeAfterCommittedRemoval(t *testing.T) { + imageCID := mediaImageCID("cdn committed removal") + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + entered, release := endpoint.HoldNextRequest(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + subject := h.comment(t, h.ownerA, imageCID) + callerCtx, cancelCaller := context.WithCancel(t.Context()) + defer cancelCaller() + removed := make(chan error, 1) + actorDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")) + idempotencyKey := "remove-" + testkit.UniqueID(t) + go func() { + result, err := h.moderation.RemoveContent(callerCtx, actorDID, moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: idempotencyKey, Reason: cdnPurgeSpam, + }) + if err == nil && (result == nil || result.Outcome != moderation.OutcomeApplied) { + err = errors.New("removal did not apply") + } + removed <- err + }() + require.NoError(t, awaitModerationCDN(t, "removal to return while the CDN request is held", removed)) + awaitModerationCDN(t, "immediate CDN purge request", entered) + blocked, err := postgres.NewModerationRepository(h.db).IsBlocked(t.Context(), h.ownerA, imageCID) + require.NoError(t, err) + assert.True(t, blocked, "a separate connection must see the committed block before Cloudflare responds") + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + cancelCaller() + release() + waitModerationCDN(t, h.queue) + require.Len(t, endpoint.Requests(), 1) + assert.NoError(t, endpoint.Requests()[0].DecodeError) + assert.Equal(t, http.MethodPost, endpoint.Requests()[0].Method) + assert.Equal(t, "/zones/zone-abc/purge_cache", endpoint.Requests()[0].Path) + assert.Equal(t, "Bearer route-cdn-token-SENTINEL", endpoint.Requests()[0].Authorization) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[0].Files) + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + clock.Set(time.Date(2026, 9, 28, 12, 1, 29, 999999000, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + assert.Len(t, endpoint.Requests(), 1) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 2) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[1].Files) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + clock.Set(time.Date(2026, 9, 28, 13, 0, 0, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + assert.Len(t, endpoint.Requests(), 2) +} + +func TestModerationCDNPurgeWaitHonorsCallerDeadline(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + entered, release := endpoint.HoldNextRequest(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + imageCID := mediaImageCID("cdn bounded wait") + h.remove(t, h.comment(t, h.ownerA, imageCID), cdnPurgeSpam) + awaitModerationCDN(t, "immediate CDN purge request", entered) + ctx, cancel := context.WithCancel(t.Context()) + cancel() + waited := make(chan error, 1) + go func() { waited <- h.queue.Wait(ctx) }() + assert.ErrorIs(t, awaitModerationCDN(t, "cancelled Wait to return", waited), context.Canceled) + release() + waitModerationCDN(t, h.queue) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + assert.Len(t, endpoint.Requests(), 2) +} + +func TestModerationCDNPurgeRemovalAndRestoreLifecycle(t *testing.T) { + t.Run("stale version", func(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + imageCID := mediaImageCID("cdn stale version") + subject := h.comment(t, h.ownerA, imageCID) + result, err := h.moderation.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v7", IdempotencyKey: "remove-" + testkit.UniqueID(t), Reason: cdnPurgeSpam, + }) + require.ErrorIs(t, err, moderation.ErrStateConflict) + assert.Nil(t, result) + assert.Zero(t, moderationCDNTargetCount(t, h.db)) + assert.Empty(t, endpoint.Requests()) + }) + t.Run("pending restore, completion, and fresh removal", func(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + endpoint.QueueResponse(http.StatusInternalServerError, `{"success":false}`) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + imageCID := mediaImageCID("cdn restore pending") + subject := h.comment(t, h.ownerA, imageCID) + removed := h.remove(t, subject, cdnPurgeSpam) + waitModerationCDN(t, h.queue) + failed := moderationCDNTarget(t, h.db, h.ownerA, imageCID) + assert.Equal(t, "pending", failed.state) + assert.Equal(t, 1, failed.attempts) + assert.Equal(t, "http_500", failed.failureCode) + assert.Equal(t, time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC), failed.nextAttemptAt) + require.Len(t, endpoint.Requests(), 1) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[0].Files) + restored := h.restore(t, subject, removed) + assert.Len(t, endpoint.Requests(), 1, "restore makes no request") + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 2, "a pending target remains eligible after restore") + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[1].Files) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + result, err := h.moderation.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: restored.State.Version, + IdempotencyKey: "remove-" + testkit.UniqueID(t), Reason: cdnPurgeSpam, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + waitModerationCDN(t, h.queue) + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + require.Len(t, endpoint.Requests(), 3) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[2].Files) + }) + t.Run("restore after completion", func(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + imageCID := mediaImageCID("cdn restore completed") + subject := h.comment(t, h.ownerA, imageCID) + removed := h.remove(t, subject, cdnPurgeSpam) + waitModerationCDN(t, h.queue) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + h.restore(t, subject, removed) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + assert.Len(t, endpoint.Requests(), 2) + }) +} + +func TestModerationCDNPurgeIllegalContentRecordsOnlyOwner(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + imageCID := mediaImageCID("cdn illegal content") + h.remove(t, h.comment(t, h.ownerA, imageCID), "social.coves.moderation.defs#reasonIllegalContent") + waitModerationCDN(t, h.queue) + assert.Equal(t, 1, moderationCDNTargetCount(t, h.db)) + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) + require.Len(t, endpoint.Requests(), 1) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[0].Files) +} + +type failingCDNRecordStore struct{ moderation.Store } +type failingCDNRecordTransaction struct{ moderation.Transaction } + +func (store failingCDNRecordStore) InTransaction(ctx context.Context, fn func(context.Context, moderation.Transaction) error) error { + return store.Store.InTransaction(ctx, func(ctx context.Context, tx moderation.Transaction) error { + return fn(ctx, failingCDNRecordTransaction{tx}) + }) +} + +func (tx failingCDNRecordTransaction) RecordCDNPurgeTargets(ctx context.Context, blobs []imageproxy.BlockedBlob) error { + if err := tx.Transaction.RecordCDNPurgeTargets(ctx, blobs); err != nil { + return err + } + return errors.New("record targets failed after insert") +} + +func TestModerationCDNPurgeRecordFailureRollsBackRemoval(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second, func(store moderation.Store) moderation.Store { + return failingCDNRecordStore{store} + }) + imageCID := mediaImageCID("cdn atomicity") + subject := h.comment(t, h.ownerA, imageCID) + result, err := h.moderation.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-" + testkit.UniqueID(t), Reason: cdnPurgeSpam, + }) + require.Error(t, err) + assert.Nil(t, result) + blocked, err := postgres.NewModerationRepository(h.db).IsBlocked(t.Context(), h.ownerA, imageCID) + require.NoError(t, err) + assert.False(t, blocked) + assert.Zero(t, moderationCDNTargetCount(t, h.db)) + assert.Empty(t, endpoint.Requests()) +} + +func TestModerationCDNPurgeImmediateAttemptIsRestrictedToRemoval(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + endpoint.QueueResponse(http.StatusInternalServerError, `{"success":false}`) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + imageCID := mediaImageCID("cdn shared cid") + h.remove(t, h.comment(t, h.ownerB, imageCID), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + otherBefore := moderationCDNTarget(t, h.db, h.ownerB, imageCID) + assert.Equal(t, "pending", otherBefore.state) + assert.Equal(t, 1, otherBefore.attempts) + assert.Equal(t, time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC), otherBefore.nextAttemptAt) + require.Len(t, endpoint.Requests(), 1) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerB, imageCID), endpoint.Requests()[0].Files) + clock.Set(time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC)) + h.remove(t, h.comment(t, h.ownerA, imageCID), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + require.Len(t, endpoint.Requests(), 2) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[1].Files) + assert.Equal(t, otherBefore, moderationCDNTarget(t, h.db, h.ownerB, imageCID)) + h.remove(t, h.comment(t, h.ownerA), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + assert.Len(t, endpoint.Requests(), 2, "a removal without an image makes no purge request") +} + +func TestModerationCDNPurgeUnconfiguredWritesNoTargets(t *testing.T) { + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h, _ := newModerationMediaHarness(t, false) + imageCID := mediaImageCID("cdn unconfigured restore") + subject := h.comment(t, h.ownerA, imageCID) + removed := h.remove(t, subject, cdnPurgeSpam) + h.restore(t, subject, removed) + assert.Zero(t, moderationCDNTargetCount(t, h.db)) + assert.Empty(t, endpoint.Requests()) +} + +// assertModerationCDNUnclaimed checks a recorded target no attempt has claimed: +// still pending, no attempts, and due immediately. +func assertModerationCDNUnclaimed(t *testing.T, db *sql.DB, owner, cid string) { + t.Helper() + var state, nextAttempt string + var attempts int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT state, attempts, next_attempt_at::text + FROM moderation_media_purges WHERE owner_did = $1 AND blob_cid = $2 + `, owner, cid).Scan(&state, &attempts, &nextAttempt)) + assert.Equal(t, "pending", state) + assert.Zero(t, attempts) + assert.Equal(t, "-infinity", nextAttempt) +} + +// interruptingCDNPurger acknowledges one pair, then holds until the caller's +// context ends and reports every other pair as a transport failure, the way a +// Cloudflare request cut off by the sweep's cycle deadline does. +type interruptingCDNPurger struct { + acknowledged imageproxy.BlockedBlob + entered chan struct{} +} + +func (purger *interruptingCDNPurger) PurgeBlobs(ctx context.Context, blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + close(purger.entered) + <-ctx.Done() + var result imageproxy.CDNPurgeResult + for _, blob := range blobs { + if blob == purger.acknowledged { + result.Acknowledged = append(result.Acknowledged, blob) + continue + } + result.Failed = append(result.Failed, imageproxy.CDNPurgeFailure{Blob: blob, Code: "transport"}) + } + return result +} + +// Not parallel: this test replaces slog.Default while the sweep runs. +func TestModerationCDNPurgeSweepRecordsOutcomesAfterContextEnds(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + first, second := mediaImageCID("cdn interrupted first"), mediaImageCID("cdn interrupted second") + h.remove(t, moderationCDNTwoImagePost(t, h, first, second), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + require.Equal(t, time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC), moderationCDNTarget(t, h.db, h.ownerA, second).nextAttemptAt) + + logs := &postMediaLogCapture{} + previousLogger := slog.Default() + slog.SetDefault(slog.New(logs)) + t.Cleanup(func() { slog.SetDefault(previousLogger) }) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + purger := &interruptingCDNPurger{acknowledged: imageproxy.BlockedBlob{OwnerDID: h.ownerA, CID: first}, entered: make(chan struct{})} + queue := moderation.NewCDNPurgeQueue(postgres.NewModerationRepository(h.db), purger, + moderation.CDNPurgeQueueConfig{Now: clock.Now, WriteTimeout: 90 * time.Second, BatchSize: 100}) + sweepCtx, cancelSweep := context.WithCancel(t.Context()) + defer cancelSweep() + swept := make(chan error, 1) + go func() { swept <- queue.Sweep(sweepCtx) }() + awaitModerationCDN(t, "sweep purge request", purger.entered) + cancelSweep() + awaitModerationCDN(t, "interrupted sweep to return", swept) + + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, first).state, + "an acknowledged pair completes even though the sweep's context ended") + interrupted := moderationCDNTarget(t, h.db, h.ownerA, second) + assert.Equal(t, "pending", interrupted.state) + assert.Equal(t, 1, interrupted.attempts) + assert.Equal(t, "interrupted", interrupted.failureCode) + assert.Equal(t, time.Date(2026, 9, 28, 12, 2, 30, 0, time.UTC), interrupted.nextAttemptAt) + var warnings int + for _, record := range logs.snapshot() { + attrs := make(map[string]slog.Value) + record.Attrs(func(attr slog.Attr) bool { + attrs[attr.Key] = attr.Value + return true + }) + if record.Level == slog.LevelError { + assert.NotEqual(t, second, attrs["cid"].String(), "an interrupted pair is not logged as a purge failure") + } + if record.Level == slog.LevelWarn && attrs["count"].Kind() == slog.KindInt64 && attrs["count"].Int64() == 1 { + warnings++ + } + } + assert.Equal(t, 1, warnings, "one warning counts the interrupted pairs") + + clock.Set(time.Date(2026, 9, 28, 12, 2, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 2) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, second), endpoint.Requests()[1].Files) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, second).state) +} + +func TestModerationCDNPurgeSaturatedImmediateAttemptsLeaveTargetToSweep(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + held := make([]func(), 0, 4) + for slot := range 4 { + entered, release := endpoint.HoldNextRequest(t) + held = append(held, release) + h.remove(t, h.comment(t, h.ownerA, mediaImageCID(fmt.Sprintf("cdn saturated held %d", slot))), cdnPurgeSpam) + awaitModerationCDN(t, "held immediate attempt", entered) + } + extra := mediaImageCID("cdn saturated extra") + h.remove(t, h.comment(t, h.ownerA, extra), cdnPurgeSpam) + for _, release := range held { + release() + } + waitModerationCDN(t, h.queue) + require.Len(t, endpoint.Requests(), 4, "a saturated queue makes no immediate request") + assertModerationCDNUnclaimed(t, h.db, h.ownerA, extra) + + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 5, "the sweep purges the target the saturated queue skipped") + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, extra), endpoint.Requests()[4].Files) + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, extra).state) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, extra).state) +} + +func TestModerationCDNPurgeClosedQueueLeavesTargetsToSweep(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + h.queue.Close() + imageCID := mediaImageCID("cdn closed queue") + h.remove(t, h.comment(t, h.ownerA, imageCID), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + assert.Empty(t, endpoint.Requests(), "a closed queue makes no immediate request") + assertModerationCDNUnclaimed(t, h.db, h.ownerA, imageCID) + + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 1) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, imageCID), endpoint.Requests()[0].Files) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 2) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, imageCID).state) +} diff --git a/internal/api/routes/moderation_cdn_purge_retry_integration_test.go b/internal/api/routes/moderation_cdn_purge_retry_integration_test.go new file mode 100644 index 0000000..bc5c6a5 --- /dev/null +++ b/internal/api/routes/moderation_cdn_purge_retry_integration_test.go @@ -0,0 +1,364 @@ +//go:build integration + +package routes_test + +import ( + "context" + "database/sql" + "encoding/json" + "net/http" + "sync" + "sync/atomic" + "testing" + "time" + + "Coves/internal/core/imageproxy" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type moderationCDNScriptedPurger struct { + mu sync.Mutex + calls [][]imageproxy.BlockedBlob + result func([]imageproxy.BlockedBlob) imageproxy.CDNPurgeResult + enter chan struct{} + release chan struct{} +} + +func (purger *moderationCDNScriptedPurger) PurgeBlobs(_ context.Context, blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + copyOfBlobs := append([]imageproxy.BlockedBlob(nil), blobs...) + purger.mu.Lock() + purger.calls = append(purger.calls, copyOfBlobs) + purger.mu.Unlock() + if purger.enter != nil { + close(purger.enter) + <-purger.release + } + return purger.result(copyOfBlobs) +} + +func (purger *moderationCDNScriptedPurger) Calls() [][]imageproxy.BlockedBlob { + purger.mu.Lock() + defer purger.mu.Unlock() + calls := make([][]imageproxy.BlockedBlob, len(purger.calls)) + for i, blobs := range purger.calls { + calls[i] = append([]imageproxy.BlockedBlob(nil), blobs...) + } + return calls +} + +func moderationCDNRetryQueue(h *moderationMediaHarness, clock *moderationCDNClock, purger moderation.CDNPurger, timeout time.Duration, batchSize int) *moderation.CDNPurgeQueue { + return moderation.NewCDNPurgeQueue(postgres.NewModerationRepository(h.db), purger, + moderation.CDNPurgeQueueConfig{Now: clock.Now, WriteTimeout: timeout, BatchSize: batchSize}) +} + +func moderationCDNTwoImagePost(t *testing.T, h *moderationMediaHarness, first, second string) moderation.StrongRef { + t.Helper() + subject, owner := h.indexedImagePost(t, moderation.PostV2Collection, first) + images := make([]any, 0, 2) + for _, cid := range []string{first, second} { + images = append(images, map[string]any{"alt": "test image", "image": map[string]any{ + "$type": "blob", "ref": map[string]any{"$link": cid}, "mimeType": "image/png", "size": 10, + }}) + } + embed, err := json.Marshal(map[string]any{"$type": "social.coves.embed.images", "images": images}) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), `UPDATE posts SET embed = $2::jsonb WHERE uri = $1`, subject.URI, string(embed)) + require.NoError(t, err) + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{owner, second}] = true + h.pds.mu.Unlock() + return subject +} + +func TestModerationCDNPurgeSweepDrainsAllDueBatches(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + endpoint.SetResponse(http.StatusInternalServerError, `{"success":false}`) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 30*time.Second) + first, second := mediaImageCID("cdn batch first"), mediaImageCID("cdn batch second") + h.remove(t, moderationCDNTwoImagePost(t, h, first, second), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + require.Len(t, endpoint.Requests(), 1) + assert.ElementsMatch(t, append(moderationCDNFiles(h.ownerA, first), moderationCDNFiles(h.ownerA, second)...), endpoint.Requests()[0].Files) + for _, cid := range []string{first, second} { + row := moderationCDNTarget(t, h.db, h.ownerA, cid) + assert.Equal(t, "pending", row.state) + assert.Equal(t, 1, row.attempts) + assert.Equal(t, time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC), row.nextAttemptAt) + } + endpoint.SetResponse(http.StatusOK, `{"success":true}`) + clock.Set(time.Date(2026, 9, 28, 12, 2, 0, 0, time.UTC)) + before := len(endpoint.Requests()) + queue := moderationCDNRetryQueue(h, clock, newMediaCloudflarePurger(t, endpoint, "batch-token"), 30*time.Second, 1) + require.NoError(t, queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), before+2, "one Sweep must drain both batches") + assert.Len(t, endpoint.Requests()[before].Files, 6) + assert.Len(t, endpoint.Requests()[before+1].Files, 6) + assert.ElementsMatch(t, append(moderationCDNFiles(h.ownerA, first), moderationCDNFiles(h.ownerA, second)...), + append(endpoint.Requests()[before].Files, endpoint.Requests()[before+1].Files...)) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, first).state) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, second).state) +} + +func TestModerationCDNPurgeAttemptUsesClockBeforeHeldResponse(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + endpoint.QueueResponse(http.StatusInternalServerError, `{"success":false}`) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + cid := mediaImageCID("cdn crossed anchor") + h.remove(t, h.comment(t, h.ownerA, cid), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + row := moderationCDNTarget(t, h.db, h.ownerA, cid) + assert.Equal(t, 1, row.attempts) + assert.Equal(t, time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC), row.nextAttemptAt) + clock.Set(time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC)) + entered, release := endpoint.HoldNextRequest(t) + finished := make(chan error, 1) + go func() { finished <- h.queue.Sweep(t.Context()) }() + awaitModerationCDN(t, "early purge request", entered) + clock.Set(time.Date(2026, 9, 28, 12, 2, 0, 0, time.UTC)) + release() + require.NoError(t, awaitModerationCDN(t, "early sweep to finish", finished)) + require.Len(t, endpoint.Requests(), 2, "one Sweep attempts the target once") + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, cid), endpoint.Requests()[1].Files) + row = moderationCDNTarget(t, h.db, h.ownerA, cid) + assert.Equal(t, "pending", row.state, "an early attempt cannot complete merely because its response arrives after the anchor") + assert.Equal(t, 1, row.attempts) + assert.Equal(t, time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC), row.nextAttemptAt) + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 3) + assert.ElementsMatch(t, moderationCDNFiles(h.ownerA, cid), endpoint.Requests()[2].Files) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, cid).state) +} + +func TestModerationCDNPurgeMixedResultsRetryOnlyFailedPair(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + first, second := mediaImageCID("cdn mixed first"), mediaImageCID("cdn mixed second") + h.remove(t, moderationCDNTwoImagePost(t, h, first, second), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + require.Len(t, endpoint.Requests(), 1) + assert.ElementsMatch(t, append(moderationCDNFiles(h.ownerA, first), moderationCDNFiles(h.ownerA, second)...), endpoint.Requests()[0].Files) + for _, cid := range []string{first, second} { + row := moderationCDNTarget(t, h.db, h.ownerA, cid) + assert.Equal(t, "pending", row.state) + assert.Zero(t, row.attempts) + assert.Equal(t, time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC), row.nextAttemptAt) + } + var phase atomic.Int32 + purger := &moderationCDNScriptedPurger{result: func(blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + if phase.Load() == 0 { + return imageproxy.CDNPurgeResult{ + Acknowledged: []imageproxy.BlockedBlob{{OwnerDID: h.ownerA, CID: first}}, + Failed: []imageproxy.CDNPurgeFailure{{Blob: imageproxy.BlockedBlob{OwnerDID: h.ownerA, CID: second}, Code: "http_500"}}, + } + } + return imageproxy.CDNPurgeResult{Acknowledged: blobs} + }} + queue := moderationCDNRetryQueue(h, clock, purger, 90*time.Second, 100) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + require.NoError(t, queue.Sweep(t.Context())) + require.Len(t, purger.Calls(), 1) + assert.ElementsMatch(t, []imageproxy.BlockedBlob{{OwnerDID: h.ownerA, CID: first}, {OwnerDID: h.ownerA, CID: second}}, purger.Calls()[0]) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, first).state) + failed := moderationCDNTarget(t, h.db, h.ownerA, second) + assert.Equal(t, "pending", failed.state) + assert.Equal(t, 1, failed.attempts) + assert.Equal(t, "http_500", failed.failureCode) + assert.Equal(t, time.Date(2026, 9, 28, 12, 2, 30, 0, time.UTC), failed.nextAttemptAt) + phase.Store(1) + clock.Set(time.Date(2026, 9, 28, 12, 2, 30, 0, time.UTC)) + require.NoError(t, queue.Sweep(t.Context())) + require.Len(t, purger.Calls(), 2) + assert.ElementsMatch(t, []imageproxy.BlockedBlob{{OwnerDID: h.ownerA, CID: second}}, purger.Calls()[1]) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, second).state) +} + +func TestModerationCDNPurgeExpiredLeaseFencesStaleWorker(t *testing.T) { + for _, lateFailure := range []bool{false, true} { + name := "stale failure before completion" + if lateFailure { + name = "late failure after completion" + } + t.Run(name, func(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + cid := mediaImageCID("cdn fenced " + name) + h.remove(t, h.comment(t, h.ownerA, cid), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, cid).state) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + first := &moderationCDNScriptedPurger{enter: make(chan struct{}), release: make(chan struct{}), + result: func(blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + return imageproxy.CDNPurgeResult{Failed: []imageproxy.CDNPurgeFailure{{Blob: blobs[0], Code: "http_500"}}} + }} + var releaseFirst sync.Once + t.Cleanup(func() { releaseFirst.Do(func() { close(first.release) }) }) + firstFinished := make(chan error, 1) + go func() { + firstFinished <- moderationCDNRetryQueue(h, clock, first, 90*time.Second, 100).Sweep(t.Context()) + }() + awaitModerationCDN(t, "first worker to claim", first.enter) + assert.ElementsMatch(t, []imageproxy.BlockedBlob{{OwnerDID: h.ownerA, CID: cid}}, first.Calls()[0]) + clock.Set(time.Date(2026, 9, 28, 12, 3, 30, 0, time.UTC)) + second := &moderationCDNScriptedPurger{enter: make(chan struct{}), release: make(chan struct{}), + result: func(blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + return imageproxy.CDNPurgeResult{Acknowledged: blobs} + }} + var releaseSecond sync.Once + t.Cleanup(func() { releaseSecond.Do(func() { close(second.release) }) }) + secondFinished := make(chan error, 1) + go func() { + secondFinished <- moderationCDNRetryQueue(h, clock, second, 90*time.Second, 100).Sweep(t.Context()) + }() + awaitModerationCDN(t, "replacement worker to claim expired lease", second.enter) + third := &moderationCDNScriptedPurger{result: func(blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + return imageproxy.CDNPurgeResult{Acknowledged: blobs} + }} + if !lateFailure { + releaseFirst.Do(func() { close(first.release) }) + require.NoError(t, awaitModerationCDN(t, "stale failure", firstFinished)) + } + require.NoError(t, moderationCDNRetryQueue(h, clock, third, 90*time.Second, 100).Sweep(t.Context())) + assert.Empty(t, third.Calls(), "the stale failure cannot steal the replacement worker's lease") + releaseSecond.Do(func() { close(second.release) }) + require.NoError(t, awaitModerationCDN(t, "replacement worker success", secondFinished)) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, cid).state) + if lateFailure { + releaseFirst.Do(func() { close(first.release) }) + require.NoError(t, awaitModerationCDN(t, "late failure", firstFinished)) + } + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, cid).state) + clock.Set(time.Date(2026, 9, 28, 14, 0, 0, 0, time.UTC)) + require.NoError(t, moderationCDNRetryQueue(h, clock, third, 90*time.Second, 100).Sweep(t.Context())) + assert.Empty(t, third.Calls()) + }) + } +} + +func TestModerationCDNPurgeRestartRecoversAbandonedLeaseAtDeadline(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + cid := mediaImageCID("cdn abandoned lease") + h.remove(t, h.comment(t, h.ownerA, cid), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, cid).state) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + first := &moderationCDNScriptedPurger{enter: make(chan struct{}), release: make(chan struct{}), + result: func(blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + return imageproxy.CDNPurgeResult{Acknowledged: blobs} + }} + var releaseFirst sync.Once + firstFinished := make(chan error, 1) + t.Cleanup(func() { + releaseFirst.Do(func() { close(first.release) }) + require.NoError(t, awaitModerationCDN(t, "abandoned worker cleanup", firstFinished)) + }) + go func() { + firstFinished <- moderationCDNRetryQueue(h, clock, first, 90*time.Second, 100).Sweep(context.Background()) + }() + awaitModerationCDN(t, "abandoned worker to claim", first.enter) + restarted := &moderationCDNScriptedPurger{result: func(blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + return imageproxy.CDNPurgeResult{Acknowledged: blobs} + }} + queue := moderationCDNRetryQueue(h, clock, restarted, 90*time.Second, 100) + clock.Set(time.Date(2026, 9, 28, 12, 3, 29, 999999000, time.UTC)) + require.NoError(t, queue.Sweep(t.Context())) + assert.Empty(t, restarted.Calls()) + clock.Set(time.Date(2026, 9, 28, 12, 3, 30, 0, time.UTC)) + require.NoError(t, queue.Sweep(t.Context())) + assert.ElementsMatch(t, []imageproxy.BlockedBlob{{OwnerDID: h.ownerA, CID: cid}}, restarted.Calls()[0]) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, cid).state) +} + +type moderationCDNNoopTargets struct{} + +func (moderationCDNNoopTargets) PurgeAfterCommit(context.Context, []imageproxy.BlockedBlob) {} + +func TestModerationCDNPurgeAnchorIsSampledAfterRemovalCommit(t *testing.T) { + firstInstant := time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC) + committedInstant := time.Date(2026, 9, 28, 12, 0, 10, 0, time.UTC) + clock := &moderationCDNClock{at: firstInstant} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + var armed atomic.Bool + var h *moderationMediaHarness + var subject moderation.StrongRef + now := func() time.Time { + if armed.Swap(false) { + clock.Set(committedInstant) + h.remove(t, subject, cdnPurgeSpam) + return firstInstant + } + return clock.Now() + } + h, _ = newModerationMediaHarness(t, false, func(config *moderationMediaHarnessConfig) { + config.now = now + config.targets = func(*sql.DB, func() time.Time) moderation.CDNPurgeTargets { return moderationCDNNoopTargets{} } + }) + cid := mediaImageCID("cdn after commit anchor") + subject = h.comment(t, h.ownerA, cid) + queue := moderation.NewCDNPurgeQueue(postgres.NewModerationRepository(h.db), + newMediaCloudflarePurger(t, endpoint, "anchor-token"), + moderation.CDNPurgeQueueConfig{Now: now, WriteTimeout: 90 * time.Second, BatchSize: 100}) + armed.Store(true) + require.NoError(t, queue.Sweep(t.Context())) + require.False(t, armed.Load(), "the first Sweep must invoke the clock hook") + assert.Equal(t, "pending", moderationCDNTarget(t, h.db, h.ownerA, cid).state) + clock.Set(time.Date(2026, 9, 28, 12, 1, 39, 999999000, time.UTC)) + require.NoError(t, queue.Sweep(t.Context())) + assert.NotEqual(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, cid).state, + "a post-commit target cannot complete before C2+90s") + clock.Set(time.Date(2026, 9, 28, 12, 1, 40, 0, time.UTC)) + require.NoError(t, queue.Sweep(t.Context())) + if moderationCDNTarget(t, h.db, h.ownerA, cid).state != "completed" { + clock.Set(time.Date(2026, 9, 28, 12, 3, 40, 0, time.UTC)) + require.NoError(t, queue.Sweep(t.Context())) + } + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, cid).state) +} + +func TestModerationCDNPurgeSweepSkipsRowLockedByConcurrentClaim(t *testing.T) { + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 90*time.Second) + cid := mediaImageCID("cdn locked by concurrent claim") + h.remove(t, h.comment(t, h.ownerA, cid), cdnPurgeSpam) + waitModerationCDN(t, h.queue) + row := moderationCDNTarget(t, h.db, h.ownerA, cid) + require.Equal(t, "pending", row.state) + require.Equal(t, time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC), row.nextAttemptAt) + clock.Set(time.Date(2026, 9, 28, 12, 1, 30, 0, time.UTC)) + + holder, err := h.db.BeginTx(context.Background(), nil) + require.NoError(t, err) + var releaseHolder sync.Once + rollbackHolder := func() { releaseHolder.Do(func() { require.NoError(t, holder.Rollback()) }) } + t.Cleanup(rollbackHolder) + var lockedCID string + require.NoError(t, holder.QueryRowContext(t.Context(), ` + SELECT blob_cid FROM moderation_media_purges WHERE owner_did = $1 AND blob_cid = $2 FOR UPDATE + `, h.ownerA, cid).Scan(&lockedCID)) + + purger := &moderationCDNScriptedPurger{result: func(blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { + return imageproxy.CDNPurgeResult{Acknowledged: blobs} + }} + queue := moderationCDNRetryQueue(h, clock, purger, 90*time.Second, 100) + sweepCtx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + require.NoError(t, queue.Sweep(sweepCtx), "a sweep must skip a row another claim holds instead of waiting on it") + assert.Empty(t, purger.Calls(), "a row held by a concurrent claim must not be purged twice") + + rollbackHolder() + require.NoError(t, queue.Sweep(t.Context())) + require.Len(t, purger.Calls(), 1, "the row was due, so only the lock kept the first sweep away") + assert.ElementsMatch(t, []imageproxy.BlockedBlob{{OwnerDID: h.ownerA, CID: cid}}, purger.Calls()[0]) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, h.ownerA, cid).state) +} diff --git a/internal/api/routes/moderation_media_integration_test.go b/internal/api/routes/moderation_media_integration_test.go index 4453717..107b8eb 100644 --- a/internal/api/routes/moderation_media_integration_test.go +++ b/internal/api/routes/moderation_media_integration_test.go @@ -21,7 +21,6 @@ import ( imagehandler "Coves/internal/api/handlers/imageproxy" "Coves/internal/api/routes" "Coves/internal/atproto/identity" - "Coves/internal/core/blobs" "Coves/internal/core/imageproxy" "Coves/internal/core/moderation" "Coves/internal/db/postgres" @@ -148,6 +147,7 @@ type moderationMediaHarness struct { proxy *httptest.Server pds *mediaPDS moderation moderation.Service + queue *moderation.CDNPurgeQueue postURI string postCID string ownerA string @@ -157,8 +157,18 @@ type moderationMediaHarness struct { proxyService *imageproxy.ImageProxyService } -func newModerationMediaHarness(t *testing.T, blockFetch bool, cdnPurgers ...moderation.CDNPurger) (*moderationMediaHarness, *waitingMediaFetcher) { +type moderationMediaHarnessConfig struct { + now func() time.Time + targets func(*sql.DB, func() time.Time) moderation.CDNPurgeTargets + wrapStore func(moderation.Store) moderation.Store +} + +func newModerationMediaHarness(t *testing.T, blockFetch bool, configure ...func(*moderationMediaHarnessConfig)) (*moderationMediaHarness, *waitingMediaFetcher) { t.Helper() + options := moderationMediaHarnessConfig{now: time.Now} + for _, configureHarness := range configure { + configureHarness(&options) + } db := testkit.DB(t) ownerName := testkit.UniqueIDWithPrefix(t, "mediaowner") otherName := testkit.UniqueIDWithPrefix(t, "otherowner") @@ -196,19 +206,24 @@ func newModerationMediaHarness(t *testing.T, blockFetch bool, cdnPurgers ...mode store := postgres.NewModerationRepository(db) proxyService, err := imageproxy.NewService(cache, processor, fetcher, store, imageproxy.DefaultConfig()) require.NoError(t, err) - config := moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000, Purger: proxyService} - if len(cdnPurgers) > 0 { - config.CDNPurger = cdnPurgers[0] + config := moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000, Purger: proxyService, Now: options.now} + if options.targets != nil { + config.CDNPurgeTargets = options.targets(db, options.now) + } + var serviceStore moderation.Store = store + if options.wrapStore != nil { + serviceStore = options.wrapStore(store) } service := moderation.NewService( moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), - store, config, + serviceStore, config, ) router := chi.NewRouter() routes.RegisterImageProxyRoutes(router, imagehandler.NewHandler(proxyService, mediaPDSResolver{url: pdsServer.URL})) proxy := httptest.NewServer(router) t.Cleanup(proxy.Close) - return &moderationMediaHarness{db: db, cache: cache, cacheDir: cacheDir, proxy: proxy, proxyService: proxyService, pds: pds, + queue, _ := config.CDNPurgeTargets.(*moderation.CDNPurgeQueue) + return &moderationMediaHarness{db: db, cache: cache, cacheDir: cacheDir, proxy: proxy, proxyService: proxyService, pds: pds, queue: queue, moderation: service, postURI: postURI, postCID: post.CID, ownerA: ownerA, ownerB: ownerB}, waiting } @@ -223,69 +238,6 @@ func newMediaCloudflarePurger(t *testing.T, endpoint *testkit.CloudflarePurgeEnd return purger } -func TestModerationCommentCDNPurgeAfterPostgresCommit(t *testing.T) { - const token = "route-cdn-token-SENTINEL" - const base = "https://img.example.test" - const spam = "social.coves.moderation.defs#reasonSpam" - for _, scenario := range []string{"removal", "stale version", "restore"} { - t.Run(scenario, func(t *testing.T) { - endpoint := testkit.NewCloudflarePurgeEndpoint(t) - purger := newMediaCloudflarePurger(t, endpoint, token) - h, _ := newModerationMediaHarness(t, false, purger) - imageCID := mediaImageCID("cdn comment " + scenario) - subject := h.comment(t, h.ownerA, imageCID) - observed := make(chan struct { - blocked bool - err error - }, 1) - endpoint.SetHook(func([]string) { - blocked, err := postgres.NewModerationRepository(h.db).IsBlocked(t.Context(), h.ownerA, imageCID) - // Non-blocking: an unexpected second request must not wedge the handler. - select { - case observed <- struct { - blocked bool - err error - }{blocked, err}: - default: - } - }) - if scenario == "stale version" { - result, err := h.moderation.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RemoveContentRequest{ - Subject: subject, ExpectedVersion: "v7", IdempotencyKey: "remove-" + testkit.UniqueID(t), Reason: spam, - }) - require.ErrorIs(t, err, moderation.ErrStateConflict) - assert.Nil(t, result) - assert.Empty(t, endpoint.Requests()) - return - } - removed := h.remove(t, subject, spam) - requests := endpoint.Requests() - require.Len(t, requests, 1) - request := requests[0] - assert.NoError(t, request.DecodeError) - assert.Equal(t, http.MethodPost, request.Method) - assert.Equal(t, "/zones/zone-abc/purge_cache", request.Path) - assert.Equal(t, "Bearer "+token, request.Authorization) - var expected []string - for _, preset := range []string{"avatar", "avatar_small", "banner", "content_preview", "content_full", "embed_thumbnail"} { - expected = append(expected, blobs.HydrateImageProxyURL(base, preset, h.ownerA, imageCID)) - } - assert.ElementsMatch(t, expected, request.Files) - select { - case committed := <-observed: - require.NoError(t, committed.err) - assert.True(t, committed.blocked, "the endpoint must see the block on a separate connection") - default: - t.Fatal("Cloudflare request hook did not check the committed block") - } - if scenario == "restore" { - h.restore(t, subject, removed) - assert.Len(t, endpoint.Requests(), 1, "restore must not make another purge request") - } - }) - } -} - func (h *moderationMediaHarness) comment(t *testing.T, owner string, imageCIDs ...string) moderation.StrongRef { t.Helper() images := make([]any, 0, len(imageCIDs)) @@ -343,7 +295,7 @@ func (h *moderationMediaHarness) remove(t *testing.T, subject moderation.StrongR return result } -func (h *moderationMediaHarness) restore(t *testing.T, subject moderation.StrongRef, removed *moderation.MutationResult) { +func (h *moderationMediaHarness) restore(t *testing.T, subject moderation.StrongRef, removed *moderation.MutationResult) *moderation.MutationResult { t.Helper() result, err := h.moderation.RestoreContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RestoreContentRequest{ ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: removed.State.Version, @@ -351,6 +303,7 @@ func (h *moderationMediaHarness) restore(t *testing.T, subject moderation.Strong }) require.NoError(t, err) assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + return result } func (h *moderationMediaHarness) cachePath(preset, owner, cid string) string { diff --git a/internal/api/routes/moderation_post_media_integration_test.go b/internal/api/routes/moderation_post_media_integration_test.go index 0213426..6c0b3df 100644 --- a/internal/api/routes/moderation_post_media_integration_test.go +++ b/internal/api/routes/moderation_post_media_integration_test.go @@ -5,11 +5,9 @@ package routes_test import ( "context" "encoding/json" - "fmt" "log/slog" "net/http" "os" - "slices" "strings" "sync" "testing" @@ -107,12 +105,12 @@ func (capture *postMediaLogCapture) snapshot() []slog.Record { // Not parallel: this test replaces slog.Default while the removal runs. func TestModerationPostCDNFailureDoesNotUndoLocalRemoval(t *testing.T) { - const token = "post-cdn-token-SENTINEL" + const token = "route-cdn-token-SENTINEL" const bodySentinel = "cf-response-body-SENTINEL" + clock := &moderationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} endpoint := testkit.NewCloudflarePurgeEndpoint(t) endpoint.SetResponse(http.StatusInternalServerError, bodySentinel) - cdn := newMediaCloudflarePurger(t, endpoint, token) - h, _ := newModerationMediaHarness(t, false, cdn) + h := newModerationCDNPurgeHarness(t, endpoint, clock, 30*time.Second) imageCID := mediaImageCID("post cdn failure") subject, authorDID := h.indexedImagePost(t, moderation.PostV2Collection, imageCID) require.Equal(t, h.ownerA, authorDID) @@ -125,31 +123,83 @@ func TestModerationPostCDNFailureDoesNotUndoLocalRemoval(t *testing.T) { slog.SetDefault(slog.New(logs)) t.Cleanup(func() { slog.SetDefault(previousLogger) }) removed := h.remove(t, subject, postMediaSpamReason) + waitModerationCDN(t, h.queue) require.Equal(t, moderation.OutcomeApplied, removed.Outcome) - assert.Len(t, endpoint.Requests(), 1, "the failing purge must have been attempted") + require.Len(t, endpoint.Requests(), 1, "the failing purge must have been attempted") + assert.ElementsMatch(t, moderationCDNFiles(authorDID, imageCID), endpoint.Requests()[0].Files) h.assertNoCachedBlob(t, imageCID, authorDID) require.Equal(t, http.StatusNotFound, h.request(t, postMediaPreset, authorDID, imageCID)) + blocked, err := postgres.NewModerationRepository(h.db).IsBlocked(t.Context(), authorDID, imageCID) + require.NoError(t, err) + assert.True(t, blocked) + first := moderationCDNTarget(t, h.db, authorDID, imageCID) + assert.Equal(t, "pending", first.state) + assert.Equal(t, 1, first.attempts) + assert.Equal(t, "http_500", first.failureCode) + assert.Equal(t, time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC), first.nextAttemptAt) - var foundError bool + var matchingErrors int for _, record := range logs.snapshot() { - var values []string var text strings.Builder text.WriteString(record.Message) + attrs := make(map[string]string) record.Attrs(func(attr slog.Attr) bool { - values = append(values, attr.Value.String()) - fmt.Fprint(&text, " ", attr) + attrs[attr.Key] = attr.Value.String() + text.WriteString(" " + attr.Key + "=" + attr.Value.String()) return true }) assert.NotContains(t, text.String(), token, "no log record may expose the Cloudflare token") assert.NotContains(t, text.String(), bodySentinel, "no log record may expose the Cloudflare response") - if record.Level == slog.LevelError && - slices.Contains(values, authorDID) && - slices.Contains(values, imageCID) && - slices.Contains(values, "http_500") { - foundError = true + if record.Level == slog.LevelError && attrs["did"] == authorDID && attrs["cid"] == imageCID && attrs["code"] == "http_500" { + matchingErrors++ } } - assert.True(t, foundError, "an Error record must carry the author DID, CID and http_500") + assert.Equal(t, 1, matchingErrors, "exactly one Error record must carry did, cid and http_500 for the first failure") + slog.SetDefault(previousLogger) + + newQueue := func() *moderation.CDNPurgeQueue { + return moderation.NewCDNPurgeQueue(postgres.NewModerationRepository(h.db), + newMediaCloudflarePurger(t, endpoint, token), moderation.CDNPurgeQueueConfig{WriteTimeout: 30 * time.Second, Now: clock.Now}) + } + clock.Set(time.Date(2026, 9, 28, 12, 0, 59, 999999000, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + assert.Len(t, endpoint.Requests(), 1) + clock.Set(time.Date(2026, 9, 28, 12, 1, 0, 0, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + require.Len(t, endpoint.Requests(), 2) + assert.ElementsMatch(t, moderationCDNFiles(authorDID, imageCID), endpoint.Requests()[1].Files) + second := moderationCDNTarget(t, h.db, authorDID, imageCID) + assert.Equal(t, "pending", second.state) + assert.Equal(t, 2, second.attempts) + assert.Equal(t, "http_500", second.failureCode) + assert.Equal(t, time.Date(2026, 9, 28, 12, 3, 0, 0, time.UTC), second.nextAttemptAt) + clock.Set(time.Date(2026, 9, 28, 12, 2, 59, 999999000, time.UTC)) + require.NoError(t, h.queue.Sweep(t.Context())) + assert.Len(t, endpoint.Requests(), 2) + + clock.Set(time.Date(2026, 9, 28, 12, 3, 0, 0, time.UTC)) + entered, release := endpoint.HoldNextRequest(t) + firstSweep := make(chan error, 1) + go func() { firstSweep <- newQueue().Sweep(t.Context()) }() + awaitModerationCDN(t, "first concurrent sweep request", entered) + require.NoError(t, newQueue().Sweep(t.Context())) + assert.Len(t, endpoint.Requests(), 3, "the second queue cannot claim an in-flight target") + release() + require.NoError(t, awaitModerationCDN(t, "first concurrent sweep to finish", firstSweep)) + assert.ElementsMatch(t, moderationCDNFiles(authorDID, imageCID), endpoint.Requests()[2].Files) + third := moderationCDNTarget(t, h.db, authorDID, imageCID) + assert.Equal(t, "pending", third.state) + assert.Equal(t, 3, third.attempts) + assert.Equal(t, time.Date(2026, 9, 28, 12, 7, 0, 0, time.UTC), third.nextAttemptAt) + endpoint.SetResponse(http.StatusOK, `{"success":true}`) + clock.Set(time.Date(2026, 9, 28, 12, 7, 0, 0, time.UTC)) + require.NoError(t, newQueue().Sweep(t.Context()), "a restarted queue must recover the due target") + require.Len(t, endpoint.Requests(), 4) + assert.ElementsMatch(t, moderationCDNFiles(authorDID, imageCID), endpoint.Requests()[3].Files) + assert.Equal(t, "completed", moderationCDNTarget(t, h.db, authorDID, imageCID).state) + clock.Set(time.Date(2026, 9, 28, 14, 0, 0, 0, time.UTC)) + require.NoError(t, newQueue().Sweep(t.Context())) + assert.Len(t, endpoint.Requests(), 4) } func TestModerationPostV2MediaRemovalColdAndWarm(t *testing.T) { diff --git a/internal/atproto/jetstream/comment_consumer.go b/internal/atproto/jetstream/comment_consumer.go index c4c035f..3f8ad77 100644 --- a/internal/atproto/jetstream/comment_consumer.go +++ b/internal/atproto/jetstream/comment_consumer.go @@ -48,7 +48,7 @@ type CommentEventConsumerOption func(*CommentEventConsumer) // MediaReconciler blocks images introduced on removed comments or posts. type MediaReconciler interface { ReconcileTx(ctx context.Context, tx *sql.Tx, subjectURI string) ([]moderation.MediaBlock, error) - Purge(blocks []moderation.MediaBlock) + Purge(ctx context.Context, blocks []moderation.MediaBlock) } // WithCommentMediaReconciler reconciles media blocks when a removed comment is rewritten. @@ -77,7 +77,7 @@ func commitMediaWrite(ctx context.Context, tx *sql.Tx, uri string, reconciler Me return err } if reconciler != nil { - reconciler.Purge(blocks) + reconciler.Purge(ctx, blocks) } return nil } diff --git a/internal/atproto/jetstream/post_consumer.go b/internal/atproto/jetstream/post_consumer.go index 4c680e9..2975997 100644 --- a/internal/atproto/jetstream/post_consumer.go +++ b/internal/atproto/jetstream/post_consumer.go @@ -723,7 +723,7 @@ func (c *PostEventConsumer) commitIncomingMediaWrite(ctx context.Context, tx *sq return err } if c.mediaReconciler != nil { - c.mediaReconciler.Purge(blocks) + c.mediaReconciler.Purge(ctx, blocks) } return nil } diff --git a/internal/atproto/jetstream/post_moderation_consumer_test.go b/internal/atproto/jetstream/post_moderation_consumer_test.go index 652d818..f1fb084 100644 --- a/internal/atproto/jetstream/post_moderation_consumer_test.go +++ b/internal/atproto/jetstream/post_moderation_consumer_test.go @@ -6,10 +6,12 @@ import ( "context" "database/sql" "encoding/json" + "errors" + "net/http" + "sync" "testing" "time" - "Coves/internal/core/blobs" "Coves/internal/core/communityFeeds" "Coves/internal/core/imageproxy" "Coves/internal/core/moderation" @@ -26,6 +28,7 @@ import ( const ( postModerationCIDOne = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" postModerationCIDTwo = "bafkreicy44vctf2bgqnn5wwzdern7bc2khwi7ku2r66bozl4x6bsrvuj2q" + postModerationCIDThree = "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" postModerationRecordCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" ) @@ -80,13 +83,14 @@ type postModerationConsumerFixture struct { postService posts.Service moderator moderation.Service purger *postModerationPurger + queue *moderation.CDNPurgeQueue uri, rkey string createdAt int64 revs []string create *JetstreamEvent } -func newPostModerationConsumerFixture(t *testing.T, cdnPurgers ...moderation.CDNPurger) postModerationConsumerFixture { +func newPostModerationConsumerFixture(t *testing.T, targets ...func(*sql.DB) moderation.CDNPurgeTargets) postModerationConsumerFixture { t.Helper() db := testkit.DB(t) f := newPV2Fixture(t, db) @@ -94,8 +98,11 @@ func newPostModerationConsumerFixture(t *testing.T, cdnPurgers ...moderation.CDN moderationRepository := postgres.NewModerationRepository(db) purger := &postModerationPurger{db: db} var options []moderation.MediaReconcilerOption - if len(cdnPurgers) > 0 { - options = append(options, moderation.WithCDNPurger(cdnPurgers[0])) + var queue *moderation.CDNPurgeQueue + if len(targets) > 0 { + configured := targets[0](db) + queue, _ = configured.(*moderation.CDNPurgeQueue) + options = append(options, moderation.WithCDNPurgeTargets(configured)) } f.consumer = NewPostEventConsumer( postRepository, postgres.NewCommunityRepository(db, credentialciphertest.Fixed()), f.users, db, @@ -131,65 +138,180 @@ func newPostModerationConsumerFixture(t *testing.T, cdnPurgers ...moderation.CDN require.Equal(t, posts.AdmissionApplied, accepted.Outcome) return postModerationConsumerFixture{ pv2Fixture: f, postRepository: postRepository, postService: postService, - moderator: moderator, purger: purger, uri: uri, rkey: rkey, + moderator: moderator, purger: purger, queue: queue, uri: uri, rkey: rkey, createdAt: createdAt, revs: revs, create: create, } } -func TestModerationPostConsumerCDNPurgeAfterEditCommit(t *testing.T) { - const token = "consumer-cdn-token-SENTINEL" - const base = "https://img.example.test" - endpoint := testkit.NewCloudflarePurgeEndpoint(t) +type postModerationCDNClock struct { + mu sync.Mutex + at time.Time +} + +func (clock *postModerationCDNClock) Now() time.Time { + clock.mu.Lock() + defer clock.mu.Unlock() + return clock.at +} + +func newPostModerationCDNFixture(t *testing.T, endpoint *testkit.CloudflarePurgeEndpoint) postModerationConsumerFixture { + t.Helper() + clock := &postModerationCDNClock{at: time.Date(2026, 9, 28, 12, 0, 0, 0, time.UTC)} cdn, err := imageproxy.NewCloudflarePurger(imageproxy.CloudflarePurgerConfig{ - APIBase: endpoint.URL(), ZoneID: "zone-abc", APIToken: token, - BaseURLs: []string{base}, Timeout: 5 * time.Second, + APIBase: endpoint.URL(), ZoneID: "zone-abc", APIToken: "consumer-cdn-token-SENTINEL", + BaseURLs: []string{"https://img.example.test"}, Timeout: 5 * time.Second, }) require.NoError(t, err) - require.NotNil(t, cdn) - f := newPostModerationConsumerFixture(t, cdn) - removed := f.remove(t, "social.coves.moderation.defs#reasonSpam") - assert.Empty(t, endpoint.Requests(), "the moderator has no CDN purger") - observed := make(chan struct { - blocked bool - err error - }, 1) - endpoint.SetHook(func([]string) { - blocked, err := postgres.NewModerationRepository(f.db).IsBlocked(t.Context(), pv2Author, postModerationCIDTwo) - // Non-blocking: an unexpected second request must not wedge the handler. - select { - case observed <- struct { - blocked bool - err error - }{blocked, err}: - default: - } + return newPostModerationConsumerFixture(t, func(db *sql.DB) moderation.CDNPurgeTargets { + return moderation.NewCDNPurgeQueue(postgres.NewModerationRepository(db), cdn, + moderation.CDNPurgeQueueConfig{WriteTimeout: 90 * time.Second, Now: clock.Now}) }) +} + +func postModerationCDNTargetCount(t *testing.T, db *sql.DB) int { + t.Helper() + var count int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_media_purges`).Scan(&count)) + return count +} + +func postModerationCDNFiles(cid string) []string { + return []string{ + "https://img.example.test/img/avatar/plain/" + pv2Author + "/" + cid, + "https://img.example.test/img/avatar_small/plain/" + pv2Author + "/" + cid, + "https://img.example.test/img/banner/plain/" + pv2Author + "/" + cid, + "https://img.example.test/img/content_preview/plain/" + pv2Author + "/" + cid, + "https://img.example.test/img/content_full/plain/" + pv2Author + "/" + cid, + "https://img.example.test/img/embed_thumbnail/plain/" + pv2Author + "/" + cid, + } +} + +func assertPostModerationCDNCommitted(t *testing.T, f postModerationConsumerFixture, cid string) { + t.Helper() + blocked, err := postgres.NewModerationRepository(f.db).IsBlocked(t.Context(), pv2Author, cid) + require.NoError(t, err) + assert.True(t, blocked, "a separate connection must see the committed incoming image block") + var state string + require.NoError(t, f.db.QueryRowContext(t.Context(), ` + SELECT state FROM moderation_media_purges WHERE owner_did = $1 AND blob_cid = $2 + `, pv2Author, cid).Scan(&state)) + assert.Equal(t, "pending", state) +} + +func assertPostModerationCDNRequest(t *testing.T, endpoint *testkit.CloudflarePurgeEndpoint, cid string) { + t.Helper() + requests := endpoint.Requests() + require.Len(t, requests, 1) + assert.NoError(t, requests[0].DecodeError) + assert.Equal(t, http.MethodPost, requests[0].Method) + assert.Equal(t, "/zones/zone-abc/purge_cache", requests[0].Path) + assert.Equal(t, "Bearer consumer-cdn-token-SENTINEL", requests[0].Authorization) + assert.ElementsMatch(t, postModerationCDNFiles(cid), requests[0].Files) +} + +func waitPostModerationCDN(t *testing.T, queue *moderation.CDNPurgeQueue) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + require.NoError(t, queue.Wait(ctx)) +} + +func awaitPostModerationCDNRequest(t *testing.T, entered <-chan struct{}) { + t.Helper() + select { + case <-entered: + case <-time.After(10 * time.Second): + t.Fatal("timed out waiting for immediate CDN purge request") + } +} + +func TestModerationPostConsumerCDNPurgeAfterEditCommit(t *testing.T) { + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + entered, release := endpoint.HoldNextRequest(t) + f := newPostModerationCDNFixture(t, endpoint) + removed := f.remove(t, "social.coves.moderation.defs#reasonSpam") + assert.Empty(t, endpoint.Requests(), "the moderator has no CDN target seam") update := pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)) require.NoError(t, f.consumer.HandleEvent(t.Context(), update)) + awaitPostModerationCDNRequest(t, entered) assert.Equal(t, 1, countRows(t, f.db, ` SELECT count(*) FROM moderation_media_blocks WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active `, removed.Action.ID, pv2Author, postModerationCIDTwo)) - requests := endpoint.Requests() - require.Len(t, requests, 1) - request := requests[0] - assert.NoError(t, request.DecodeError) - assert.Equal(t, "POST", request.Method) - assert.Equal(t, "/zones/zone-abc/purge_cache", request.Path) - assert.Equal(t, "Bearer "+token, request.Authorization) - var expected []string - for _, preset := range []string{"avatar", "avatar_small", "banner", "content_preview", "content_full", "embed_thumbnail"} { - expected = append(expected, blobs.HydrateImageProxyURL(base, preset, pv2Author, postModerationCIDTwo)) - } - assert.ElementsMatch(t, expected, request.Files) - select { - case committed := <-observed: - require.NoError(t, committed.err) - assert.True(t, committed.blocked, "the endpoint must see the edited image block on a separate connection") - default: - t.Fatal("Cloudflare request hook did not check the committed edited image block") + assertPostModerationCDNCommitted(t, f, postModerationCIDTwo) + assertPostModerationCDNRequest(t, endpoint, postModerationCIDTwo) + release() + waitPostModerationCDN(t, f.queue) + assertPostModerationCDNCommitted(t, f, postModerationCIDTwo) +} + +func TestModerationPostConsumerCDNPurgeIncomingCreateCommit(t *testing.T) { + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + entered, release := endpoint.HoldNextRequest(t) + f := newPostModerationCDNFixture(t, endpoint) + removed := f.remove(t, "social.coves.moderation.defs#reasonSpam") + embed, err := json.Marshal(postModerationRecord(postModerationCIDThree)["embed"]) + require.NoError(t, err) + embedJSON := string(embed) + applied, err := f.consumer.indexPostIfRevWins(t.Context(), &posts.Post{ + URI: f.uri, CID: postModerationCIDOne, RKey: f.rkey, AuthorDID: pv2Author, CommunityDID: pv2Community, + Embed: &embedJSON, CreatedAt: time.Now(), IndexedAt: time.Now(), + }, f.revs[1]) + require.NoError(t, err) + assert.False(t, applied, "the incoming create must leave the stored row intact") + awaitPostModerationCDNRequest(t, entered) + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDThree)) + assertPostModerationCDNCommitted(t, f, postModerationCIDThree) + assertPostModerationCDNRequest(t, endpoint, postModerationCIDThree) + release() + waitPostModerationCDN(t, f.queue) + assertPostModerationCDNCommitted(t, f, postModerationCIDThree) +} + +type failingPostModerationCDNBinder struct{ moderation.TransactionBinder } +type failingPostModerationCDNTransaction struct{ moderation.MediaTransaction } + +func (binder failingPostModerationCDNBinder) BindTransaction(tx *sql.Tx) moderation.MediaTransaction { + return failingPostModerationCDNTransaction{binder.TransactionBinder.BindTransaction(tx)} +} + +func (tx failingPostModerationCDNTransaction) RecordCDNPurgeTargets(ctx context.Context, targets []imageproxy.BlockedBlob) error { + if err := tx.MediaTransaction.RecordCDNPurgeTargets(ctx, targets); err != nil { + return err } + return errors.New("record targets failed after insert") +} + +func TestModerationPostConsumerCDNPurgeRecordFailureRollsBackEdit(t *testing.T) { + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + f := newPostModerationCDNFixture(t, endpoint) + f.remove(t, "social.coves.moderation.defs#reasonSpam") + f.consumer.mediaReconciler = moderation.NewMediaReconciler( + failingPostModerationCDNBinder{postgres.NewModerationRepository(f.db)}, fixtures.InstanceDID(), f.purger, + moderation.WithCDNPurgeTargets(f.queue)) + update := pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)) + require.Error(t, f.consumer.HandleEvent(t.Context(), update), "record failure must abort the edit") + blocked, err := postgres.NewModerationRepository(f.db).IsBlocked(t.Context(), pv2Author, postModerationCIDTwo) + require.NoError(t, err) + assert.False(t, blocked) + assert.Zero(t, postModerationCDNTargetCount(t, f.db)) + assert.Empty(t, endpoint.Requests()) +} + +func TestModerationPostConsumerCDNPurgeUnconfiguredWritesNoTargets(t *testing.T) { + endpoint := testkit.NewCloudflarePurgeEndpoint(t) + f := newPostModerationConsumerFixture(t) + f.remove(t, "social.coves.moderation.defs#reasonSpam") + update := pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)) + require.NoError(t, f.consumer.HandleEvent(t.Context(), update)) + assert.Zero(t, postModerationCDNTargetCount(t, f.db)) + assert.Empty(t, endpoint.Requests()) } func (f postModerationConsumerFixture) remove(t *testing.T, reason string) *moderation.MutationResult { diff --git a/internal/core/moderation/cdn_purge.go b/internal/core/moderation/cdn_purge.go index 7835e1c..d5e900a 100644 --- a/internal/core/moderation/cdn_purge.go +++ b/internal/core/moderation/cdn_purge.go @@ -2,7 +2,6 @@ package moderation import ( "context" - "log/slog" "Coves/internal/core/imageproxy" ) @@ -15,28 +14,14 @@ type CDNPurger interface { // MediaReconcilerOption configures optional post-commit media actions. type MediaReconcilerOption func(*MediaReconciler) -// WithCDNPurger enables CDN invalidation after media reconciliation commits. -func WithCDNPurger(purger CDNPurger) MediaReconcilerOption { - return func(reconciler *MediaReconciler) { reconciler.cdnPurger = purger } -} - -// purgeCDNMediaBlocks invalidates only the URLs belonging to owner-scoped -// blocks. CDN failures are best effort; the committed block already prevents -// serving the blob at the origin. -func purgeCDNMediaBlocks(ctx context.Context, purger CDNPurger, blocks []MediaBlock) { - if purger == nil { - return - } +// ownerCDNPurgeBlobs excludes every-owner blocks: only pairs with an origin +// repository can be turned into CDN image URLs. +func ownerCDNPurgeBlobs(blocks []MediaBlock) []imageproxy.BlockedBlob { var owned []imageproxy.BlockedBlob for _, block := range blocks { if block.OwnerDID != "" { owned = append(owned, imageproxy.BlockedBlob{OwnerDID: block.OwnerDID, CID: block.BlobCID}) } } - if len(owned) == 0 { - return - } - for _, failure := range purger.PurgeBlobs(ctx, owned).Failed { - slog.Error("moderation CDN purge failed", "did", failure.Blob.OwnerDID, "cid", failure.Blob.CID, "code", failure.Code) - } + return owned } diff --git a/internal/core/moderation/cdn_purge_retry_internal_test.go b/internal/core/moderation/cdn_purge_retry_internal_test.go new file mode 100644 index 0000000..f9ef14c --- /dev/null +++ b/internal/core/moderation/cdn_purge_retry_internal_test.go @@ -0,0 +1,26 @@ +package moderation + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestCDNPurgeRetryDelay(t *testing.T) { + for _, test := range []struct { + failedAttempts int + want time.Duration + }{ + {0, time.Minute}, + {1, time.Minute}, + {2, 2 * time.Minute}, + {3, 4 * time.Minute}, + {6, 32 * time.Minute}, + {7, time.Hour}, + {8, time.Hour}, + {1000, time.Hour}, + } { + assert.Equal(t, test.want, cdnPurgeRetryDelay(test.failedAttempts), "failed attempts %d", test.failedAttempts) + } +} diff --git a/internal/core/moderation/cdn_purge_targets.go b/internal/core/moderation/cdn_purge_targets.go new file mode 100644 index 0000000..e7b83df --- /dev/null +++ b/internal/core/moderation/cdn_purge_targets.go @@ -0,0 +1,314 @@ +package moderation + +import ( + "context" + "errors" + "fmt" + "log/slog" + "sync" + "time" + + "Coves/internal/core/imageproxy" +) + +// CDNPurgeTargets receives the owner-scoped pairs a committed transaction recorded. +type CDNPurgeTargets interface { + PurgeAfterCommit(ctx context.Context, blobs []imageproxy.BlockedBlob) +} + +// CDNPurgeTarget is one claimed durable edge-purge target. +type CDNPurgeTarget struct { + Blob imageproxy.BlockedBlob + Attempts int + EarliestCompletionAt time.Time + Generation int64 + Claim int64 +} + +// CDNPurgeClaim selects due targets for one attempt. +type CDNPurgeClaim struct { + // DueAt is the cutoff: only targets whose next attempt is at or before it + // are claimed. A sweep freezes it at the start of its pass. + DueAt time.Time + // Now is sampled once after the claimed rows are locked. That sample + // anchors each claim's lease and, on a target's first claim, its earliest + // completion, so the anchor is never earlier than the commit that made + // the target visible. It must not be DueAt: a target re-pended after a + // sweep began would then complete before its commit plus WriteTimeout. + Now func() time.Time + Lease time.Duration + WriteTimeout time.Duration + Limit int + Blobs []imageproxy.BlockedBlob + excluded []imageproxy.BlockedBlob +} + +// ExcludedBlobs lists targets already attempted by this sweep. A target +// re-pended during the pass is due at -infinity, so without this list the same +// pass would claim it again whatever DueAt is. The queue sets this only for +// unrestricted claims; it does not change the restriction made by Blobs on +// immediate post-commit claims. +func (claim CDNPurgeClaim) ExcludedBlobs() []imageproxy.BlockedBlob { + return claim.excluded +} + +// CDNPurgeTargetStore persists durable edge-purge targets. +type CDNPurgeTargetStore interface { + ClaimDueCDNPurgeTargets(ctx context.Context, claim CDNPurgeClaim) ([]CDNPurgeTarget, error) + CompleteCDNPurgeTarget(ctx context.Context, target CDNPurgeTarget) error + RescheduleCDNPurgeTarget(ctx context.Context, target CDNPurgeTarget, nextAttemptAt time.Time, failureCode string) error +} + +const ( + // cdnPurgeLease is how long a claim holds a target before another worker + // may claim it. + cdnPurgeLease = 2 * time.Minute + // cdnPurgeImmediateAttemptTimeout bounds an immediate post-commit attempt. + // With cdnPurgeOutcomeWriteTimeout it stays below cdnPurgeLease, so the + // attempt's outcome is written before its lease can pass to a sweep. + cdnPurgeImmediateAttemptTimeout = 90 * time.Second + // cdnPurgeOutcomeWriteTimeout bounds writing an attempt's outcomes, which + // runs after the attempt's own context may have ended. + cdnPurgeOutcomeWriteTimeout = 5 * time.Second + // cdnPurgeImmediateAttemptLimit caps immediate attempts in flight. Beyond + // it, recorded targets wait for the next sweep. + cdnPurgeImmediateAttemptLimit = 4 + // cdnPurgeInterruptedCode records an attempt cut off by its context. + cdnPurgeInterruptedCode = "interrupted" +) + +// CDNPurgeQueueConfig configures a CDNPurgeQueue. +type CDNPurgeQueueConfig struct { + WriteTimeout time.Duration + Now func() time.Time + BatchSize int +} + +// CDNPurgeQueue purges durable targets after commit and on sweeps. +type CDNPurgeQueue struct { + store CDNPurgeTargetStore + purger CDNPurger + config CDNPurgeQueueConfig + slots chan struct{} + mu sync.Mutex + closed bool + inFlight int + idle chan struct{} +} + +// NewCDNPurgeQueue builds a durable purge queue. +func NewCDNPurgeQueue(store CDNPurgeTargetStore, purger CDNPurger, config CDNPurgeQueueConfig) *CDNPurgeQueue { + if config.Now == nil { + config.Now = time.Now + } + if config.BatchSize <= 0 { + config.BatchSize = 100 + } + idle := make(chan struct{}) + close(idle) + return &CDNPurgeQueue{ + store: store, purger: purger, config: config, idle: idle, + slots: make(chan struct{}, cdnPurgeImmediateAttemptLimit), + } +} + +// PurgeAfterCommit schedules a bounded, cancellation-independent attempt for +// just the pairs recorded by the committing transaction. After Close, or while +// cdnPurgeImmediateAttemptLimit attempts are in flight, it returns without +// attempting: the targets are durable and the next sweep purges them. +func (q *CDNPurgeQueue) PurgeAfterCommit(ctx context.Context, blobs []imageproxy.BlockedBlob) { + if len(blobs) == 0 { + return + } + owned := append([]imageproxy.BlockedBlob(nil), blobs...) + q.mu.Lock() + if q.closed { + q.mu.Unlock() + return + } + select { + case q.slots <- struct{}{}: + default: + q.mu.Unlock() + slog.Info("moderation CDN purge immediate attempts saturated; the sweep will purge these targets", "count", len(owned)) + return + } + if q.inFlight == 0 { + q.idle = make(chan struct{}) + } + q.inFlight++ + q.mu.Unlock() + go func() { + defer func() { + <-q.slots + q.mu.Lock() + q.inFlight-- + if q.inFlight == 0 { + close(q.idle) + } + q.mu.Unlock() + }() + attemptCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cdnPurgeImmediateAttemptTimeout) + defer cancel() + for start := 0; start < len(owned); start += q.config.BatchSize { + if attemptCtx.Err() != nil { + slog.Warn("moderation CDN purge immediate attempt ran out of time; the sweep will purge the rest", + "count", len(owned)-start) + return + } + end := min(start+q.config.BatchSize, len(owned)) + targets, err := q.store.ClaimDueCDNPurgeTargets(attemptCtx, q.claim(owned[start:end], q.config.Now())) + if err != nil { + slog.Error("moderation CDN purge claim failed", "error", err) + return + } + if err := q.attempt(attemptCtx, targets); err != nil { + slog.Error("moderation CDN purge target update failed", "error", err) + return + } + } + }() +} + +func (q *CDNPurgeQueue) claim(blobs []imageproxy.BlockedBlob, dueAt time.Time) CDNPurgeClaim { + return CDNPurgeClaim{ + DueAt: dueAt, Now: q.config.Now, Lease: cdnPurgeLease, WriteTimeout: q.config.WriteTimeout, + Limit: q.config.BatchSize, Blobs: blobs, + } +} + +// Sweep drains targets due at the start of this pass. Freezing the due cutoff +// keeps a target rescheduled while a request is held from being retried by the +// same sweep if the clock advances during that request. The store still samples +// the live clock after locking rows for each target's completion anchor. +func (q *CDNPurgeQueue) Sweep(ctx context.Context) error { + dueAt := q.config.Now() + var attempted []imageproxy.BlockedBlob + for { + claim := q.claim(nil, dueAt) + claim.excluded = attempted + targets, err := q.store.ClaimDueCDNPurgeTargets(ctx, claim) + if err != nil { + return err + } + if len(targets) == 0 { + return ctx.Err() + } + if err := q.attempt(ctx, targets); err != nil { + return err + } + for _, target := range targets { + attempted = append(attempted, target.Blob) + } + } +} + +// Close stops new immediate attempts. Targets recorded afterwards stay pending +// for the sweep. Attempts already in flight continue; Wait observes them. +func (q *CDNPurgeQueue) Close() { + q.mu.Lock() + defer q.mu.Unlock() + q.closed = true +} + +// InFlight reports how many immediate attempts are running. +func (q *CDNPurgeQueue) InFlight() int { + q.mu.Lock() + defer q.mu.Unlock() + return q.inFlight +} + +// Wait observes all immediate attempts in flight when called. +func (q *CDNPurgeQueue) Wait(ctx context.Context) error { + q.mu.Lock() + idle := q.idle + q.mu.Unlock() + select { + case <-ctx.Done(): + return ctx.Err() + case <-idle: + return nil + } +} + +func (q *CDNPurgeQueue) attempt(ctx context.Context, targets []CDNPurgeTarget) error { + if len(targets) == 0 { + return nil + } + blobs := make([]imageproxy.BlockedBlob, len(targets)) + for i, target := range targets { + blobs[i] = target.Blob + } + attemptedAt := q.config.Now() + result := q.purger.PurgeBlobs(ctx, blobs) + // A request cut off by ctx fails as transport, as does every request the + // purger starts after ctx ended. Those pairs were interrupted, not refused. + interrupted := ctx.Err() != nil + acknowledged := make(map[imageproxy.BlockedBlob]bool, len(result.Acknowledged)) + failures := make(map[imageproxy.BlockedBlob]string, len(result.Failed)) + for _, blob := range result.Acknowledged { + acknowledged[blob] = true + } + for _, failure := range result.Failed { + failures[failure.Blob] = failure.Code + } + codes := make([]string, len(targets)) + interruptedCount := 0 + for i, target := range targets { + if acknowledged[target.Blob] { + continue + } + code, ok := failures[target.Blob] + if interrupted && (!ok || code == "transport") { + codes[i] = cdnPurgeInterruptedCode + interruptedCount++ + continue + } + if !ok { + code = "unacknowledged" + } + codes[i] = code + slog.Error("moderation CDN purge failed", "did", target.Blob.OwnerDID, "cid", target.Blob.CID, "code", code) + } + if interruptedCount > 0 { + slog.Warn("moderation CDN purge interrupted; targets rescheduled", "count", interruptedCount) + } + // The attempt's context may have ended during the purge; the outcomes are + // still written so acknowledged pairs complete and failures back off. + writeCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), cdnPurgeOutcomeWriteTimeout) + defer cancel() + var writeErrors []error + for i, target := range targets { + var err error + operation := "reschedule" + switch { + case codes[i] != "": + err = q.store.RescheduleCDNPurgeTarget(writeCtx, target, attemptedAt.Add(cdnPurgeRetryDelay(target.Attempts+1)), codes[i]) + case !attemptedAt.Before(target.EarliestCompletionAt): + operation = "complete" + err = q.store.CompleteCDNPurgeTarget(writeCtx, target) + default: + err = q.store.RescheduleCDNPurgeTarget(writeCtx, target, target.EarliestCompletionAt, "") + } + if err != nil { + writeErrors = append(writeErrors, fmt.Errorf("%s CDN purge target did=%s cid=%s: %w", + operation, target.Blob.OwnerDID, target.Blob.CID, err)) + } + } + return errors.Join(writeErrors...) +} + +// WithCDNPurgeTargets enables transactionally recorded CDN targets. +func WithCDNPurgeTargets(targets CDNPurgeTargets) MediaReconcilerOption { + return func(reconciler *MediaReconciler) { reconciler.cdnPurgeTargets = targets } +} + +func cdnPurgeRetryDelay(failedAttempts int) time.Duration { + if failedAttempts <= 1 { + return time.Minute + } + if failedAttempts >= 7 { + return time.Hour + } + return time.Minute << (failedAttempts - 1) +} diff --git a/internal/core/moderation/cdn_purge_test.go b/internal/core/moderation/cdn_purge_test.go deleted file mode 100644 index b8adf69..0000000 --- a/internal/core/moderation/cdn_purge_test.go +++ /dev/null @@ -1,344 +0,0 @@ -package moderation_test - -import ( - "context" - "errors" - "log/slog" - "sync" - "testing" - - "Coves/internal/core/imageproxy" - "Coves/internal/core/moderation" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" -) - -type recordedCDNPurge struct { - blobs []imageproxy.BlockedBlob - contextLive bool - removalCommitted bool -} - -type recordingCDNPurger struct { - mu sync.Mutex - store *inMemoryModerationStore - result imageproxy.CDNPurgeResult - calls []recordedCDNPurge -} - -func (purger *recordingCDNPurger) PurgeBlobs(ctx context.Context, blobs []imageproxy.BlockedBlob) imageproxy.CDNPurgeResult { - call := recordedCDNPurge{blobs: append([]imageproxy.BlockedBlob(nil), blobs...), contextLive: ctx.Err() == nil} - if purger.store != nil && purger.store.mu.TryLock() { - call.removalCommitted = purger.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, removeRulesURI}] != "" - for _, blob := range blobs { - found := false - for block, active := range purger.store.state.mediaBlocks { - if active && block.OwnerDID == blob.OwnerDID && block.BlobCID == blob.CID { - found = true - } - } - call.removalCommitted = call.removalCommitted && found - } - purger.store.mu.Unlock() - } - purger.mu.Lock() - defer purger.mu.Unlock() - purger.calls = append(purger.calls, call) - return purger.result -} - -func (purger *recordingCDNPurger) snapshot() []recordedCDNPurge { - purger.mu.Lock() - defer purger.mu.Unlock() - return append([]recordedCDNPurge(nil), purger.calls...) -} - -type afterBlocksFailureStore struct { - *inMemoryModerationStore - err error -} - -func (store *afterBlocksFailureStore) InTransaction(ctx context.Context, fn func(context.Context, moderation.Transaction) error) error { - return store.inMemoryModerationStore.InTransaction(ctx, func(ctx context.Context, tx moderation.Transaction) error { - return fn(ctx, &afterBlocksFailureTransaction{Transaction: tx, err: store.err}) - }) -} - -type afterBlocksFailureTransaction struct { - moderation.Transaction - err error - inserted bool -} - -func (tx *afterBlocksFailureTransaction) InsertMediaBlocks(ctx context.Context, blocks []moderation.MediaBlock) error { - if err := tx.Transaction.InsertMediaBlocks(ctx, blocks); err != nil { - return err - } - tx.inserted = true - return nil -} - -func (tx *afterBlocksFailureTransaction) SaveIdempotencyRecord(ctx context.Context, record moderation.IdempotencyRecord) error { - if tx.inserted { - return tx.err - } - return tx.Transaction.SaveIdempotencyRecord(ctx, record) -} - -type cancelAfterCommitStore struct { - *inMemoryModerationStore - cancel context.CancelFunc -} - -func (store *cancelAfterCommitStore) InTransaction(ctx context.Context, fn func(context.Context, moderation.Transaction) error) error { - err := store.inMemoryModerationStore.InTransaction(ctx, fn) - if err == nil { - store.cancel() - } - return err -} - -type recordingCDNLogHandler struct { - mu sync.Mutex - records []slog.Record -} - -func (*recordingCDNLogHandler) Enabled(context.Context, slog.Level) bool { return true } -func (handler *recordingCDNLogHandler) Handle(_ context.Context, record slog.Record) error { - handler.mu.Lock() - defer handler.mu.Unlock() - handler.records = append(handler.records, record.Clone()) - return nil -} -func (handler *recordingCDNLogHandler) WithAttrs([]slog.Attr) slog.Handler { return handler } -func (handler *recordingCDNLogHandler) WithGroup(string) slog.Handler { return handler } -func (handler *recordingCDNLogHandler) snapshot() []slog.Record { - handler.mu.Lock() - defer handler.mu.Unlock() - return append([]slog.Record(nil), handler.records...) -} - -func TestRemoveContentCDNPurgeAfterCommit(t *testing.T) { - for _, test := range []struct { - name string - images []string - reason string - want []imageproxy.BlockedBlob - rollback bool - cancel bool - failures bool - }{ - { - name: "spam three images", reason: removeRulesSpam, - images: []string{"bafyreifirstimage", "bafyreisecondimage", "bafyreithirdimage"}, - want: []imageproxy.BlockedBlob{ - {OwnerDID: "did:plc:commentauthor", CID: "bafyreifirstimage"}, - {OwnerDID: "did:plc:commentauthor", CID: "bafyreisecondimage"}, - {OwnerDID: "did:plc:commentauthor", CID: "bafyreithirdimage"}, - }, - }, - { - name: "illegal content excludes ownerless blocks", reason: removeRulesIllegal, - images: []string{"bafyreifirstimage", "bafyreisecondimage"}, - want: []imageproxy.BlockedBlob{ - {OwnerDID: "did:plc:commentauthor", CID: "bafyreifirstimage"}, - {OwnerDID: "did:plc:commentauthor", CID: "bafyreisecondimage"}, - }, - }, - {name: "no images", reason: removeRulesSpam}, - { - name: "transaction rolls back after blocks", reason: removeRulesSpam, - images: []string{"bafyreifirstimage"}, rollback: true, - }, - { - name: "caller cancelled after commit", reason: removeRulesSpam, - images: []string{"bafyreifirstimage"}, cancel: true, - want: []imageproxy.BlockedBlob{{OwnerDID: "did:plc:commentauthor", CID: "bafyreifirstimage"}}, - }, - { - name: "two failed pairs leave mutation and local purge intact", reason: removeRulesSpam, - images: []string{"bafyreifirstimage", "bafyreisecondimage", "bafyreithirdimage"}, failures: true, - want: []imageproxy.BlockedBlob{ - {OwnerDID: "did:plc:commentauthor", CID: "bafyreifirstimage"}, - {OwnerDID: "did:plc:commentauthor", CID: "bafyreisecondimage"}, - {OwnerDID: "did:plc:commentauthor", CID: "bafyreithirdimage"}, - }, - }, - } { - t.Run(test.name, func(t *testing.T) { - scenario := newRemoveRulesScenario() - comment := scenario.store.state.indexedComments[removeRulesURI] - comment.ImageCIDs = test.images - scenario.store.state.indexedComments[removeRulesURI] = comment - scenario.request.Reason = test.reason - cdn := &recordingCDNPurger{store: scenario.store, result: imageproxy.CDNPurgeResult{Acknowledged: test.want}} - var store moderation.Store = scenario.store - var ctx context.Context = t.Context() - if test.rollback { - store = &afterBlocksFailureStore{inMemoryModerationStore: scenario.store, err: errCDNRollback} - } - if test.cancel { - var cancel context.CancelFunc - ctx, cancel = context.WithCancel(t.Context()) - t.Cleanup(cancel) - store = &cancelAfterCommitStore{inMemoryModerationStore: scenario.store, cancel: cancel} - } - if test.failures { - cdn.result = imageproxy.CDNPurgeResult{ - Acknowledged: []imageproxy.BlockedBlob{{OwnerDID: "did:plc:commentauthor", CID: "bafyreithirdimage"}}, - Failed: []imageproxy.CDNPurgeFailure{ - {Blob: imageproxy.BlockedBlob{OwnerDID: "did:plc:commentauthor", CID: "bafyreifirstimage"}, Code: "http_500"}, - {Blob: imageproxy.BlockedBlob{OwnerDID: "did:plc:commentauthor", CID: "bafyreisecondimage"}, Code: "transport"}, - }, - } - logs := &recordingCDNLogHandler{} - previous := slog.Default() - slog.SetDefault(slog.New(logs)) - t.Cleanup(func() { slog.SetDefault(previous) }) - scenario.withCDNPurger(store, cdn) - result, err := scenario.service.RemoveContent(ctx, removeRulesAdminDID, scenario.request) - require.NoError(t, err) - baseline := newRemoveRulesScenario() - baselineComment := baseline.store.state.indexedComments[removeRulesURI] - baselineComment.ImageCIDs = test.images - baseline.store.state.indexedComments[removeRulesURI] = baselineComment - baselineCDN := &recordingCDNPurger{store: baseline.store, result: imageproxy.CDNPurgeResult{Acknowledged: test.want}} - baseline.withCDNPurger(baseline.store, baselineCDN) - baselineResult, baselineErr := baseline.service.RemoveContent(t.Context(), removeRulesAdminDID, baseline.request) - require.NoError(t, baselineErr) - require.Equal(t, moderation.OutcomeApplied, baselineResult.Outcome) - require.Equal(t, moderation.OutcomeApplied, result.Outcome) - assert.Equal(t, baselineResult, result) - assert.ElementsMatch(t, []removeRulesOwnerPurge{ - {ownerDID: "did:plc:commentauthor", blobCID: "bafyreifirstimage"}, - {ownerDID: "did:plc:commentauthor", blobCID: "bafyreisecondimage"}, - {ownerDID: "did:plc:commentauthor", blobCID: "bafyreithirdimage"}, - }, scenario.purger.ownerPurges) - assertCDNPurgeCalls(t, cdn.snapshot(), test.want) - assertCDNFailureLogs(t, logs.snapshot()) - return - } - scenario.withCDNPurger(store, cdn) - result, err := scenario.service.RemoveContent(ctx, removeRulesAdminDID, scenario.request) - if test.rollback { - require.ErrorIs(t, err, errCDNRollback) - assert.Nil(t, result) - assert.Empty(t, scenario.store.state.mediaBlocks) - assert.Empty(t, scenario.store.state.activeRemovals) - assert.Empty(t, scenario.purger.ownerPurges) - assert.Empty(t, cdn.snapshot()) - return - } - require.NoError(t, err) - require.NotNil(t, result) - assert.Equal(t, moderation.OutcomeApplied, result.Outcome) - if test.cancel { - assert.ErrorIs(t, ctx.Err(), context.Canceled) - } - if len(test.want) == 0 { - assert.Empty(t, cdn.snapshot()) - return - } - assertCDNPurgeCalls(t, cdn.snapshot(), test.want) - }) - } -} - -var errCDNRollback = errors.New("injected after media insertion") - -func assertCDNPurgeCalls(t *testing.T, calls []recordedCDNPurge, want []imageproxy.BlockedBlob) { - t.Helper() - require.Len(t, calls, 1) - assert.ElementsMatch(t, want, calls[0].blobs) - assert.True(t, calls[0].removalCommitted, "removal and blocks must be visible outside the transaction") - assert.True(t, calls[0].contextLive, "purge must have a live context even after caller cancellation") -} - -func assertCDNFailureLogs(t *testing.T, records []slog.Record) { - t.Helper() - var errorsOnly []slog.Record - for _, record := range records { - if record.Level == slog.LevelError { - errorsOnly = append(errorsOnly, record) - } - } - require.Len(t, errorsOnly, 2) - want := []struct{ cid, code string }{{"bafyreifirstimage", "http_500"}, {"bafyreisecondimage", "transport"}} - seen := make(map[string]int) - for _, record := range errorsOnly { - values := make([]string, 0) - record.Attrs(func(attr slog.Attr) bool { - values = append(values, attr.Value.String()) - return true - }) - assert.Contains(t, values, "did:plc:commentauthor") - assert.NotContains(t, values, "bafyreithirdimage") - matched := false - for _, pair := range want { - if containsCDNLogValue(values, pair.cid) { - matched = true - seen[pair.cid]++ - assert.Contains(t, values, pair.code) - } - } - assert.True(t, matched, "unexpected CDN failure record attributes: %v", values) - } - assert.Equal(t, map[string]int{"bafyreifirstimage": 1, "bafyreisecondimage": 1}, seen) -} - -func containsCDNLogValue(values []string, wanted string) bool { - for _, value := range values { - if value == wanted { - return true - } - } - return false -} - -func TestRestoreContentDoesNotPurgeCDN(t *testing.T) { - scenario := newRemoveRulesScenario() - cdn := &recordingCDNPurger{store: scenario.store} - scenario.withCDNPurger(scenario.store, cdn) - removed, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) - require.NoError(t, err) - require.NotNil(t, removed) - require.NotNil(t, removed.Action) - before := cdn.snapshot() - require.Len(t, before, 1, "removal must exercise the configured purger") - restored, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, moderation.RestoreContentRequest{ - ActionID: removed.Action.ID, ReviewedSubject: &moderation.StrongRef{URI: removeRulesURI, CID: removeRulesCID}, - ExpectedVersion: removed.State.Version, IdempotencyKey: "restore-cdn", Reason: removeRulesSpam, - }) - require.NoError(t, err) - require.NotNil(t, restored) - assert.Equal(t, moderation.OutcomeApplied, restored.Outcome) - assert.Equal(t, before, cdn.snapshot()) -} - -func TestMediaReconcilerPurgesOwnerScopedBlobsFromCDN(t *testing.T) { - local := &removeRulesPurger{} - cdn := &recordingCDNPurger{} - reconciler := moderation.NewMediaReconciler(postRulesMediaBinder{&postRulesMediaTransaction{}}, removeRulesInstanceDID, local, moderation.WithCDNPurger(cdn)) - reconciler.Purge(nil) - reconciler.Purge([]moderation.MediaBlock{{BlobCID: "bafyreiownerlessonly"}}) - assert.Empty(t, cdn.snapshot()) - assert.Equal(t, []string{"bafyreiownerlessonly"}, local.blobPurges) - reconciler.Purge([]moderation.MediaBlock{ - {OwnerDID: "did:plc:postauthor", BlobCID: "bafyreifirstimage"}, - {BlobCID: "bafyreithirdimage"}, - {OwnerDID: "did:plc:postauthor", BlobCID: "bafyreisecondimage"}, - }) - calls := cdn.snapshot() - require.Len(t, calls, 1) - assert.ElementsMatch(t, []imageproxy.BlockedBlob{ - {OwnerDID: "did:plc:postauthor", CID: "bafyreifirstimage"}, - {OwnerDID: "did:plc:postauthor", CID: "bafyreisecondimage"}, - }, calls[0].blobs) - assert.True(t, calls[0].contextLive) - assert.ElementsMatch(t, []removeRulesOwnerPurge{ - {ownerDID: "did:plc:postauthor", blobCID: "bafyreifirstimage"}, - {ownerDID: "did:plc:postauthor", blobCID: "bafyreisecondimage"}, - }, local.ownerPurges) - assert.Equal(t, []string{"bafyreiownerlessonly", "bafyreithirdimage"}, local.blobPurges) -} diff --git a/internal/core/moderation/fake_store_test.go b/internal/core/moderation/fake_store_test.go index 68efa3d..d0d2bef 100644 --- a/internal/core/moderation/fake_store_test.go +++ b/internal/core/moderation/fake_store_test.go @@ -7,6 +7,7 @@ import ( "sync" "time" + "Coves/internal/core/imageproxy" "Coves/internal/core/moderation" ) @@ -41,6 +42,7 @@ type inMemoryModerationState struct { labelDecisions map[inMemoryModerationLabelKey]inMemoryModerationLabelDecision idempotency map[inMemoryModerationIdempotencyKey]moderation.IdempotencyRecord mediaBlocks map[moderation.MediaBlock]bool + cdnPurgeTargets []imageproxy.BlockedBlob nextActionID int } @@ -54,6 +56,7 @@ func (state inMemoryModerationState) copy() inMemoryModerationState { labelDecisions: make(map[inMemoryModerationLabelKey]inMemoryModerationLabelDecision, len(state.labelDecisions)), idempotency: make(map[inMemoryModerationIdempotencyKey]moderation.IdempotencyRecord, len(state.idempotency)), mediaBlocks: make(map[moderation.MediaBlock]bool, len(state.mediaBlocks)), + cdnPurgeTargets: append([]imageproxy.BlockedBlob(nil), state.cdnPurgeTargets...), nextActionID: state.nextActionID, } for key, comment := range state.indexedComments { @@ -351,6 +354,11 @@ func (transaction *inMemoryModerationTransaction) InsertMediaBlocks(_ context.Co return nil } +func (transaction *inMemoryModerationTransaction) RecordCDNPurgeTargets(_ context.Context, blobs []imageproxy.BlockedBlob) error { + transaction.state.cdnPurgeTargets = append(transaction.state.cdnPurgeTargets, blobs...) + return nil +} + func (transaction *inMemoryModerationTransaction) DeactivateMediaBlocks(_ context.Context, actionID string) error { transaction.store.writeCalls = append(transaction.store.writeCalls, "DeactivateMediaBlocks") for block := range transaction.state.mediaBlocks { diff --git a/internal/core/moderation/media.go b/internal/core/moderation/media.go index 6a3880e..5045c32 100644 --- a/internal/core/moderation/media.go +++ b/internal/core/moderation/media.go @@ -5,6 +5,8 @@ import ( "database/sql" "fmt" "log/slog" + + "Coves/internal/core/imageproxy" ) // MediaTransaction provides the operations needed to reconcile a subject's images. @@ -15,6 +17,7 @@ type MediaTransaction interface { // InsertNewMediaBlocks inserts only blocks not already active for the action // and returns the blocks it inserted. InsertNewMediaBlocks(ctx context.Context, blocks []MediaBlock) ([]MediaBlock, error) + RecordCDNPurgeTargets(ctx context.Context, blobs []imageproxy.BlockedBlob) error } // TransactionBinder binds media operations to a caller's transaction. @@ -28,10 +31,10 @@ type TransactionBinder interface { // removed the subject; reconciliation never adds one. Both reconcile methods // therefore refuse an empty owner DID, which the store records as ownerless. type MediaReconciler struct { - binder TransactionBinder - instanceDID string - purger MediaPurger - cdnPurger CDNPurger + binder TransactionBinder + instanceDID string + purger MediaPurger + cdnPurgeTargets CDNPurgeTargets } // NewMediaReconciler builds a MediaReconciler. @@ -62,7 +65,7 @@ func (r *MediaReconciler) ReconcileTx(ctx context.Context, tx *sql.Tx, subjectUR if subject.OwnerDID == "" { return nil, fmt.Errorf("%w: indexed subject has no owner DID", ErrInvalidSubject) } - return bound.InsertNewMediaBlocks(ctx, ownerImageMediaBlocks(subject, action)) + return r.insertAndRecord(ctx, bound, ownerImageMediaBlocks(subject, action)) } // ReconcileIncomingTx blocks blobs of incoming content the consumer did not @@ -82,13 +85,28 @@ func (r *MediaReconciler) ReconcileIncomingTx(ctx context.Context, tx *sql.Tx, s if ownerDID == "" { return nil, fmt.Errorf("%w: incoming content has no owner DID", ErrInvalidSubject) } - return bound.InsertNewMediaBlocks(ctx, ownerImageMediaBlocks(&indexedSubject{OwnerDID: ownerDID, BlobCIDs: blobCIDs}, action)) + return r.insertAndRecord(ctx, bound, ownerImageMediaBlocks(&indexedSubject{OwnerDID: ownerDID, BlobCIDs: blobCIDs}, action)) +} + +func (r *MediaReconciler) insertAndRecord(ctx context.Context, bound MediaTransaction, blocks []MediaBlock) ([]MediaBlock, error) { + inserted, err := bound.InsertNewMediaBlocks(ctx, blocks) + if err != nil { + return nil, err + } + if r.cdnPurgeTargets != nil && len(inserted) > 0 { + if err := bound.RecordCDNPurgeTargets(ctx, ownerCDNPurgeBlobs(inserted)); err != nil { + return nil, err + } + } + return inserted, nil } // Purge removes cached bytes of newly blocked blobs after commit. -func (r *MediaReconciler) Purge(blocks []MediaBlock) { +func (r *MediaReconciler) Purge(ctx context.Context, blocks []MediaBlock) { purgeMediaBlocks(r.purger, blocks) - purgeCDNMediaBlocks(context.Background(), r.cdnPurger, blocks) + if r.cdnPurgeTargets != nil && len(blocks) > 0 { + r.cdnPurgeTargets.PurgeAfterCommit(ctx, ownerCDNPurgeBlobs(blocks)) + } } func purgeMediaBlocks(purger MediaPurger, blocks []MediaBlock) { diff --git a/internal/core/moderation/post_rules_test.go b/internal/core/moderation/post_rules_test.go index 2bd3f68..270054f 100644 --- a/internal/core/moderation/post_rules_test.go +++ b/internal/core/moderation/post_rules_test.go @@ -5,6 +5,7 @@ import ( "database/sql" "testing" + "Coves/internal/core/imageproxy" "Coves/internal/core/moderation" "github.com/stretchr/testify/assert" @@ -262,6 +263,10 @@ func (transaction *postRulesMediaTransaction) InsertNewMediaBlocks(_ context.Con return blocks, nil } +func (*postRulesMediaTransaction) RecordCDNPurgeTargets(context.Context, []imageproxy.BlockedBlob) error { + return nil +} + type postRulesMediaBinder struct{ bound *postRulesMediaTransaction } func (binder postRulesMediaBinder) BindTransaction(*sql.Tx) moderation.MediaTransaction { @@ -340,7 +345,7 @@ func TestReconcileIllegalContentMediaIsOwnerScoped(t *testing.T) { } assert.Equal(t, wantBlocks, bound.blocks, "only owner-scoped blocks may be inserted") assert.Equal(t, wantBlocks, blocks, "only owner-scoped blocks may be returned") - reconciler.Purge(blocks) + reconciler.Purge(t.Context(), blocks) assert.Equal(t, wantPurges, purger.ownerPurges, "new blobs must use PurgeOwnerBlob") assert.Empty(t, purger.blobPurges, "reconciliation must never call PurgeBlob") }) diff --git a/internal/core/moderation/remove.go b/internal/core/moderation/remove.go index 98577e7..93ee5ce 100644 --- a/internal/core/moderation/remove.go +++ b/internal/core/moderation/remove.go @@ -6,6 +6,8 @@ import ( "fmt" "time" + "Coves/internal/core/imageproxy" + "github.com/bluesky-social/indigo/atproto/syntax" ) @@ -54,6 +56,7 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re fingerprint := mutationFingerprint(ActionRemove, "", request.Subject.URI, request.Subject.CID, request.ExpectedVersion, request.Reason, "", request.PrivateNote) var result *MutationResult var newlyBlocked []MediaBlock + var purgeBlobs []imageproxy.BlockedBlob var ruleError error err := s.store.InTransaction(ctx, func(ctx context.Context, tx Transaction) error { fail := func(err error) error { @@ -149,6 +152,14 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re if err := tx.InsertMediaBlocks(ctx, newlyBlocked); err != nil { return unavailable(err) } + if s.config.CDNPurgeTargets != nil { + purgeBlobs = ownerCDNPurgeBlobs(newlyBlocked) + if len(purgeBlobs) > 0 { + if err := tx.RecordCDNPurgeTargets(ctx, purgeBlobs); err != nil { + return unavailable(err) + } + } + } } state, err := newSubjectState(request.Subject.URI, version+1, recordState, current, action, activeLabels, s.config.InstanceDID) if err != nil { @@ -168,6 +179,8 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re return nil, fmt.Errorf("%w: %w", ErrModerationUnavailable, err) } purgeMediaBlocks(s.config.Purger, newlyBlocked) - purgeCDNMediaBlocks(context.WithoutCancel(ctx), s.config.CDNPurger, newlyBlocked) + if s.config.CDNPurgeTargets != nil && len(purgeBlobs) > 0 { + s.config.CDNPurgeTargets.PurgeAfterCommit(ctx, purgeBlobs) + } return result, nil } diff --git a/internal/core/moderation/remove_rules_test.go b/internal/core/moderation/remove_rules_test.go index a03a6fa..ad9e03c 100644 --- a/internal/core/moderation/remove_rules_test.go +++ b/internal/core/moderation/remove_rules_test.go @@ -82,14 +82,6 @@ func newRemoveRulesScenario() removeRulesScenario { } } -func (scenario *removeRulesScenario) withCDNPurger(store moderation.Store, cdn moderation.CDNPurger) { - scenario.service = moderation.NewService(scenario.reader, store, moderation.Config{ - InstanceDID: removeRulesInstanceDID, IdempotencyRetention: 24 * time.Hour, - MaxLiveIdempotencyKeys: 1000, Now: func() time.Time { return scenario.store.now }, - Purger: scenario.purger, CDNPurger: cdn, - }) -} - func assertRemoveRulesNoWrites(t *testing.T, scenario removeRulesScenario) { t.Helper() assert.Empty(t, scenario.store.writeCalls) diff --git a/internal/core/moderation/store.go b/internal/core/moderation/store.go index d274574..0608a8f 100644 --- a/internal/core/moderation/store.go +++ b/internal/core/moderation/store.go @@ -3,6 +3,8 @@ package moderation import ( "context" "time" + + "Coves/internal/core/imageproxy" ) // Action kinds, scope kinds and origins recorded on moderation actions. @@ -180,6 +182,7 @@ type Transaction interface { SetSubjectVersion(ctx context.Context, subjectURI string, version int64) error InsertMediaBlocks(ctx context.Context, blocks []MediaBlock) error DeactivateMediaBlocks(ctx context.Context, actionID string) error + RecordCDNPurgeTargets(ctx context.Context, blobs []imageproxy.BlockedBlob) error } // MediaPurger removes cached bytes of newly blocked blobs. diff --git a/internal/core/moderation/types.go b/internal/core/moderation/types.go index 808cbea..5e67b74 100644 --- a/internal/core/moderation/types.go +++ b/internal/core/moderation/types.go @@ -151,7 +151,7 @@ type Config struct { IdempotencyRetention time.Duration MaxLiveIdempotencyKeys int Purger MediaPurger - CDNPurger CDNPurger + CDNPurgeTargets CDNPurgeTargets Now func() time.Time CursorSecret string CommunityResolver CommunityResolver diff --git a/internal/db/migrations/052_moderation_media_purges.sql b/internal/db/migrations/052_moderation_media_purges.sql new file mode 100644 index 0000000..4f243d3 --- /dev/null +++ b/internal/db/migrations/052_moderation_media_purges.sql @@ -0,0 +1,21 @@ +-- +goose Up +CREATE TABLE moderation_media_purges ( + owner_did TEXT NOT NULL, + blob_cid TEXT NOT NULL, + state TEXT NOT NULL DEFAULT 'pending' CHECK (state IN ('pending', 'completed')), + attempts INTEGER NOT NULL DEFAULT 0 CHECK (attempts >= 0), + next_attempt_at TIMESTAMPTZ NOT NULL DEFAULT '-infinity', + earliest_completion_at TIMESTAMPTZ, + last_failure_code TEXT, + generation BIGINT NOT NULL DEFAULT 0, + claim BIGINT NOT NULL DEFAULT 0, + created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(), + PRIMARY KEY (owner_did, blob_cid) +); + +CREATE INDEX moderation_media_purges_due ON moderation_media_purges (next_attempt_at) + WHERE state = 'pending'; + +-- +goose Down +DROP TABLE IF EXISTS moderation_media_purges; diff --git a/internal/db/postgres/admission_repo_schema_test.go b/internal/db/postgres/admission_repo_schema_test.go index 474c105..5d3adfa 100644 --- a/internal/db/postgres/admission_repo_schema_test.go +++ b/internal/db/postgres/admission_repo_schema_test.go @@ -322,6 +322,8 @@ func TestMigration034_DownRestoresTheAuthorForeignKeyUnvalidated(t *testing.T) { // all sit on top of 034, so all sixteen have to come off first. Rolling back explicitly, // one asserted step at a time, is what keeps the assertions below pointed at // 034's Down rather than at whatever happens to be newest. + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/internal/db/postgres/community_subscriber_recount_migration_test.go b/internal/db/postgres/community_subscriber_recount_migration_test.go index 91c79a0..d3f0219 100644 --- a/internal/db/postgres/community_subscriber_recount_migration_test.go +++ b/internal/db/postgres/community_subscriber_recount_migration_test.go @@ -22,6 +22,8 @@ func TestMigration045RecountsAndMaintainsCommunitySubscribers(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/internal/db/postgres/credential_reencrypt_migration_test.go b/internal/db/postgres/credential_reencrypt_migration_test.go index 4838b0d..dbd1c81 100644 --- a/internal/db/postgres/credential_reencrypt_migration_test.go +++ b/internal/db/postgres/credential_reencrypt_migration_test.go @@ -27,6 +27,8 @@ func TestMigration046DownRestoresUsableEncryptionKey(t *testing.T) { db := testkit.DB(t) assert.False(t, credentialReencryptKeyTable(t, db).Valid, "migration 046 Up must drop encryption_keys before its Down behavior can be tested") + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), @@ -399,6 +401,8 @@ func TestCredentialReencryptRejectsLegacyRowsAfterEncryptionKeysDropped(t *testi func credentialReencryptVersion45Database(t *testing.T) *sql.DB { t.Helper() db := testkit.DB(t) + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/internal/db/postgres/discover_hot_migration_test.go b/internal/db/postgres/discover_hot_migration_test.go index f6d27fd..6dffd2c 100644 --- a/internal/db/postgres/discover_hot_migration_test.go +++ b/internal/db/postgres/discover_hot_migration_test.go @@ -35,6 +35,8 @@ func TestMigration048DiscoverHotSnapshots(t *testing.T) { _, err := db.ExecContext(ctx, `CREATE TABLE discover_hot_migration_sentinel (id BIGINT PRIMARY KEY)`) require.NoError(t, err) + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/internal/db/postgres/future_comment_created_at_migration_test.go b/internal/db/postgres/future_comment_created_at_migration_test.go index 4d1fac4..51c7c6c 100644 --- a/internal/db/postgres/future_comment_created_at_migration_test.go +++ b/internal/db/postgres/future_comment_created_at_migration_test.go @@ -18,6 +18,8 @@ func TestMigration041_ClampsFutureCommentCreatedAt(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/internal/db/postgres/moderation_concurrency_integration_test.go b/internal/db/postgres/moderation_concurrency_integration_test.go index a86239c..3063fb2 100644 --- a/internal/db/postgres/moderation_concurrency_integration_test.go +++ b/internal/db/postgres/moderation_concurrency_integration_test.go @@ -191,8 +191,8 @@ func (g *gatedMediaReconciler) ReconcileTx(ctx context.Context, tx *sql.Tx, subj return blocks, err } -func (g *gatedMediaReconciler) Purge(blocks []moderation.MediaBlock) { - g.inner.Purge(blocks) +func (g *gatedMediaReconciler) Purge(ctx context.Context, blocks []moderation.MediaBlock) { + g.inner.Purge(ctx, blocks) } // holdRestoreBeforeCommit parks a mutation at its idempotency-record insert, diff --git a/internal/db/postgres/moderation_repo.go b/internal/db/postgres/moderation_repo.go index a400f1f..7551eb2 100644 --- a/internal/db/postgres/moderation_repo.go +++ b/internal/db/postgres/moderation_repo.go @@ -495,6 +495,126 @@ func (t *moderationTransaction) InsertNewMediaBlocks(ctx context.Context, blocks return inserted, nil } +// RecordCDNPurgeTargets runs in the removal or reconciliation transaction. +func (t *moderationTransaction) RecordCDNPurgeTargets(ctx context.Context, blobs []imageproxy.BlockedBlob) error { + for _, blob := range blobs { + _, err := t.tx.ExecContext(ctx, ` + INSERT INTO moderation_media_purges (owner_did, blob_cid) + VALUES ($1, $2) + ON CONFLICT (owner_did, blob_cid) DO UPDATE SET + state = 'pending', attempts = 0, next_attempt_at = '-infinity', + earliest_completion_at = NULL, last_failure_code = NULL, + generation = moderation_media_purges.generation + 1, + claim = moderation_media_purges.claim + 1, updated_at = NOW() + `, blob.OwnerDID, blob.CID) + if err != nil { + return fmt.Errorf("record CDN purge target: %w", err) + } + } + return nil +} + +// ClaimDueCDNPurgeTargets locks pending rows due at claim.DueAt without waiting +// on other workers. A clock sample after the locks anchors completion to a time +// no earlier than the committing transaction's visibility. +func (r *ModerationRepository) ClaimDueCDNPurgeTargets(ctx context.Context, claim moderation.CDNPurgeClaim) ([]moderation.CDNPurgeTarget, error) { + if claim.Now == nil { + claim.Now = time.Now + } + if claim.Limit <= 0 { + return nil, nil + } + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return nil, fmt.Errorf("begin CDN purge claim: %w", err) + } + defer tx.Rollback() + var owners, cids []string + for _, blob := range claim.Blobs { + owners = append(owners, blob.OwnerDID) + cids = append(cids, blob.CID) + } + var excludedOwners, excludedCIDs []string + for _, blob := range claim.ExcludedBlobs() { + excludedOwners = append(excludedOwners, blob.OwnerDID) + excludedCIDs = append(excludedCIDs, blob.CID) + } + rows, err := tx.QueryContext(ctx, ` + SELECT owner_did, blob_cid FROM moderation_media_purges + WHERE state = 'pending' AND next_attempt_at <= $1 + AND (NOT $2 OR (owner_did, blob_cid) IN ( + SELECT owner_did, blob_cid FROM unnest($3::text[], $4::text[]) AS allowed(owner_did, blob_cid))) + AND (owner_did, blob_cid) NOT IN ( + SELECT owner_did, blob_cid FROM unnest($6::text[], $7::text[]) AS excluded(owner_did, blob_cid)) + ORDER BY next_attempt_at, owner_did, blob_cid + LIMIT $5 FOR UPDATE SKIP LOCKED + `, claim.DueAt, claim.Blobs != nil, pq.Array(owners), pq.Array(cids), claim.Limit, + pq.Array(excludedOwners), pq.Array(excludedCIDs)) + if err != nil { + return nil, fmt.Errorf("select due CDN purge targets: %w", err) + } + var targets []moderation.CDNPurgeTarget + for rows.Next() { + var target moderation.CDNPurgeTarget + if err := rows.Scan(&target.Blob.OwnerDID, &target.Blob.CID); err != nil { + rows.Close() + return nil, fmt.Errorf("scan due CDN purge target: %w", err) + } + targets = append(targets, target) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, fmt.Errorf("read due CDN purge targets: %w", err) + } + rows.Close() + if len(targets) > 0 { + lockedNow := claim.Now() + for i := range targets { + var earliest time.Time + err := tx.QueryRowContext(ctx, ` + UPDATE moderation_media_purges SET + next_attempt_at = $3, earliest_completion_at = COALESCE(earliest_completion_at, $4), + claim = claim + 1, updated_at = NOW() + WHERE owner_did = $1 AND blob_cid = $2 + RETURNING attempts, earliest_completion_at, generation, claim + `, targets[i].Blob.OwnerDID, targets[i].Blob.CID, + lockedNow.Add(claim.Lease), lockedNow.Add(claim.WriteTimeout)).Scan( + &targets[i].Attempts, &earliest, &targets[i].Generation, &targets[i].Claim) + if err != nil { + return nil, fmt.Errorf("lease CDN purge target: %w", err) + } + targets[i].EarliestCompletionAt = earliest + } + } + if err := tx.Commit(); err != nil { + return nil, fmt.Errorf("commit CDN purge claim: %w", err) + } + return targets, nil +} + +// CompleteCDNPurgeTarget fences new removals through the generation. +func (r *ModerationRepository) CompleteCDNPurgeTarget(ctx context.Context, target moderation.CDNPurgeTarget) error { + _, err := r.db.ExecContext(ctx, ` + UPDATE moderation_media_purges SET state = 'completed', updated_at = NOW() + WHERE owner_did = $1 AND blob_cid = $2 AND generation = $3 AND state = 'pending' + `, target.Blob.OwnerDID, target.Blob.CID, target.Generation) + return err +} + +// RescheduleCDNPurgeTarget fences stale workers by their claim number. +func (r *ModerationRepository) RescheduleCDNPurgeTarget(ctx context.Context, target moderation.CDNPurgeTarget, nextAttemptAt time.Time, failureCode string) error { + _, err := r.db.ExecContext(ctx, ` + UPDATE moderation_media_purges SET + next_attempt_at = $4, + attempts = attempts + CASE WHEN $5::text <> '' THEN 1 ELSE 0 END, + last_failure_code = CASE WHEN $5::text <> '' THEN $5 ELSE last_failure_code END, + updated_at = NOW() + WHERE owner_did = $1 AND blob_cid = $2 AND generation = $3 + AND claim = $6 AND state = 'pending' + `, target.Blob.OwnerDID, target.Blob.CID, target.Generation, nextAttemptAt, failureCode, target.Claim) + return err +} + func (t *moderationTransaction) DeactivateMediaBlocks(ctx context.Context, actionID string) error { _, err := t.tx.ExecContext(ctx, ` UPDATE moderation_media_blocks SET active = FALSE WHERE action_id = $1 AND active diff --git a/internal/db/postgres/read_path_indexes_migration_test.go b/internal/db/postgres/read_path_indexes_migration_test.go index bdfa061..d838886 100644 --- a/internal/db/postgres/read_path_indexes_migration_test.go +++ b/internal/db/postgres/read_path_indexes_migration_test.go @@ -37,6 +37,8 @@ func TestMigration049ReadPathIndexes(t *testing.T) { } assertReadPathIndexDefinitions(t, db, "after 049 Up") + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/internal/db/postgres/rematerialize_ledger_schema_test.go b/internal/db/postgres/rematerialize_ledger_schema_test.go index 608c32d..bf31b36 100644 --- a/internal/db/postgres/rematerialize_ledger_schema_test.go +++ b/internal/db/postgres/rematerialize_ledger_schema_test.go @@ -196,6 +196,8 @@ func TestRematerializeLedgerMigration_RollsBack(t *testing.T) { // 037 and come off first, one asserted step at a time. Asserting which // migration rolled back is what keeps this pointed at 037's Down rather than // drifting onto a newer one later. + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/internal/db/postgres/vote_drift_recount_migration_test.go b/internal/db/postgres/vote_drift_recount_migration_test.go index ee6a2aa..8403e52 100644 --- a/internal/db/postgres/vote_drift_recount_migration_test.go +++ b/internal/db/postgres/vote_drift_recount_migration_test.go @@ -80,6 +80,8 @@ func TestMigration040_RecountsVoteDriftAndSweepsLegacyOrphans(t *testing.T) { // point of a repair migration and cannot be observed by seeding after it has // run. Asserting the version that came off is the tripwire that keeps this // pointed at 040 when later migrations land. + require.EqualValues(t, 52, testkit.MigrateDownOne(t, db, 52), + "052 (moderation CDN purge targets) must be rolled back before testing earlier migrations") require.EqualValues(t, 51, testkit.MigrateDownOne(t, db, 51), "051 (moderation action log indexes) must be rolled back before testing earlier migrations") require.EqualValues(t, 50, testkit.MigrateDownOne(t, db, 50), diff --git a/tests/testkit/cloudflare_purge.go b/tests/testkit/cloudflare_purge.go index 4501f6b..bd030b1 100644 --- a/tests/testkit/cloudflare_purge.go +++ b/tests/testkit/cloudflare_purge.go @@ -21,6 +21,18 @@ type CloudflarePurgeEndpoint struct { status int body string hook func([]string) + scripted []cloudflarePurgeResponse + holds []cloudflarePurgeHold +} + +type cloudflarePurgeResponse struct { + status int + body string +} + +type cloudflarePurgeHold struct { + entered chan struct{} + release chan struct{} } func NewCloudflarePurgeEndpoint(t *testing.T) *CloudflarePurgeEndpoint { @@ -39,6 +51,27 @@ func (endpoint *CloudflarePurgeEndpoint) SetResponse(status int, body string) { endpoint.status, endpoint.body = status, body } +// QueueResponse scripts the next request, then falls back to SetResponse. +func (endpoint *CloudflarePurgeEndpoint) QueueResponse(status int, body string) { + endpoint.mu.Lock() + defer endpoint.mu.Unlock() + endpoint.scripted = append(endpoint.scripted, cloudflarePurgeResponse{status, body}) +} + +// HoldNextRequest holds one request until release is called. Cleanup also +// releases it, before the endpoint's server is closed. +func (endpoint *CloudflarePurgeEndpoint) HoldNextRequest(t *testing.T) (<-chan struct{}, func()) { + t.Helper() + hold := cloudflarePurgeHold{entered: make(chan struct{}), release: make(chan struct{})} + var once sync.Once + release := func() { once.Do(func() { close(hold.release) }) } + t.Cleanup(release) + endpoint.mu.Lock() + endpoint.holds = append(endpoint.holds, hold) + endpoint.mu.Unlock() + return hold.entered, release +} + func (endpoint *CloudflarePurgeEndpoint) SetHook(hook func([]string)) { endpoint.mu.Lock() defer endpoint.mu.Unlock() @@ -68,10 +101,23 @@ func (endpoint *CloudflarePurgeEndpoint) serve(w http.ResponseWriter, r *http.Re Files: files, DecodeError: decodeError, }) status, body, hook := endpoint.status, endpoint.body, endpoint.hook + if len(endpoint.scripted) > 0 { + status, body = endpoint.scripted[0].status, endpoint.scripted[0].body + endpoint.scripted = endpoint.scripted[1:] + } + var hold cloudflarePurgeHold + if len(endpoint.holds) > 0 { + hold = endpoint.holds[0] + endpoint.holds = endpoint.holds[1:] + } endpoint.mu.Unlock() if hook != nil { hook(append([]string(nil), files...)) } + if hold.entered != nil { + close(hold.entered) + <-hold.release + } w.WriteHeader(status) _, _ = w.Write([]byte(body)) } -- 2.51.2