From dbde6a3492b77abf95b944d689c19ab420219cc3 Mon Sep 17 00:00:00 2001 From: Bretton Date: Thu, 24 Sep 2026 06:22:48 -0700 Subject: [PATCH] feat(moderation): add removal core with comment removeContent/restoreContent Task 03 of the admin moderation PRD: land the moderation mutation core with its first subject type. Comments can be removed and restored by instance admins, and a successful mutation is fully enforced on every comment surface. Post subjects are rejected with InvalidSubject until chunk 04. - Migration 049 adds the moderation state tables: moderation_actions, moderation_subjects, moderation_decisions, moderation_idempotency_keys and moderation_media_blocks (no FKs to posts/comments). - removeContent/restoreContent for comments, with fingerprint-bound idempotency keys scoped to (actor, authority), a subject version token with StateConflict on mismatch, ContentChanged on CID drift, the per-actor live-key cap, and an hourly expiry sweep. - Removed comments render as the overlay-only placeholder (deletionReason: moderator, moderation.state: removed, record absent, zero votes, preserved replyCount) and are excluded from actor.getComments. - Media blocks keyed by (owner DID, blob CID), plus ownerless CID blocks for illegal-content removals; restore deactivates only that action's blocks. - The image proxy enforces blocks against the canonical CID, publishes blocks synchronously through striped locks, and purges cached bytes with a restart-safe purge sweep. - A tolerant embeds.CommentImageCIDs enumerator that also covers legacy cid blobs. - The comment consumer reconciles media blocks when it updates or recreates a subject with an active removal. - T0/T1/T2 contracts for validation, idempotency, concurrency, media enforcement and the end-to-end remove/restore arc. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/server/consumers.go | 3 +- cmd/server/main.go | 3 + .../moderation_idempotency_sweep_job.go | 36 ++ .../moderation_idempotency_sweep_job_test.go | 99 ++++ cmd/server/wiring.go | 44 +- go.mod | 6 +- .../imageproxy/decompression_bomb_test.go | 4 +- internal/api/handlers/imageproxy/handler.go | 42 +- .../api/handlers/imageproxy/handler_test.go | 18 +- .../handlers/imageproxy/moderation_test.go | 159 +++++ .../handlers/imageproxy/proxy_serving_test.go | 17 +- .../moderation/get_subject_state_test.go | 2 + .../api/handlers/moderation/remove_content.go | 190 ++++++ .../moderation/remove_content_test.go | 268 +++++++++ .../handlers/moderation/restore_content.go | 51 ++ .../moderation/restore_content_test.go | 113 ++++ internal/api/routes/moderation.go | 6 + ...ration_comment_removal_integration_test.go | 268 +++++++++ .../moderation_media_integration_test.go | 487 ++++++++++++++++ ...ration_reauthorization_integration_test.go | 138 +++++ ...deration_subject_state_integration_test.go | 2 +- internal/api/routes/registration_test.go | 4 +- .../atproto/jetstream/comment_consumer.go | 45 +- internal/config/config.go | 31 +- .../config/moderation_idempotency_test.go | 72 +++ internal/config/testing.go | 2 +- ...nt_moderation_consumer_integration_test.go | 353 ++++++++++++ .../comment_moderation_integration_test.go | 362 ++++++++++++ .../comments/comment_moderation_view_test.go | 161 ++++++ internal/core/comments/comment_service.go | 26 +- .../core/comments/comment_service_test.go | 15 + internal/core/comments/interfaces.go | 10 + internal/core/comments/view_models.go | 19 + .../core/embeds/comment_image_cids_test.go | 88 +++ internal/core/embeds/view.go | 38 ++ internal/core/imageproxy/cache.go | 81 ++- .../core/imageproxy/cache_moderation_test.go | 120 ++++ internal/core/imageproxy/errors.go | 6 + .../core/imageproxy/fetcher_guard_test.go | 2 +- internal/core/imageproxy/moderation_test.go | 481 ++++++++++++++++ internal/core/imageproxy/service.go | 334 +++++++++-- internal/core/imageproxy/service_test.go | 82 +-- internal/core/imageproxy/validation.go | 17 + internal/core/imageproxy/validation_test.go | 44 ++ internal/core/moderation/fake_store_test.go | 241 ++++++++ internal/core/moderation/idempotency_test.go | 276 +++++++++ internal/core/moderation/interfaces.go | 4 +- internal/core/moderation/media.go | 93 +++ .../core/moderation/mutation_validation.go | 51 ++ internal/core/moderation/remove.go | 163 ++++++ internal/core/moderation/remove_rules_test.go | 260 +++++++++ .../core/moderation/remove_validation_test.go | 119 ++++ internal/core/moderation/restore.go | 162 ++++++ .../core/moderation/restore_rules_test.go | 248 ++++++++ internal/core/moderation/service.go | 54 +- internal/core/moderation/service_test.go | 26 +- internal/core/moderation/store.go | 120 ++++ internal/core/moderation/subject_reader.go | 5 +- .../subject_reader_integration_test.go | 25 +- internal/core/moderation/subject_state.go | 24 + internal/core/moderation/types.go | 38 ++ internal/core/moderation/version.go | 8 +- .../db/migrations/049_moderation_state.sql | 83 +++ .../db/postgres/admission_repo_schema_test.go | 2 + internal/db/postgres/comment_repo.go | 34 ++ ...unity_subscriber_recount_migration_test.go | 2 + .../credential_reencrypt_migration_test.go | 4 + .../postgres/discover_hot_migration_test.go | 2 + ...uture_comment_created_at_migration_test.go | 2 + .../moderation_actor_lock_integration_test.go | 67 +++ ...moderation_concurrency_integration_test.go | 541 ++++++++++++++++++ ...moderation_idempotency_integration_test.go | 219 +++++++ ...oderation_media_blocks_integration_test.go | 62 ++ internal/db/postgres/moderation_repo.go | 415 ++++++++++++++ .../moderation_repo_integration_test.go | 281 +++++++++ .../rematerialize_ledger_schema_test.go | 2 + .../vote_drift_recount_migration_test.go | 2 + tests/e2e/comment_contract_test.go | 28 +- tests/e2e/moderation_contract_test.go | 238 ++++++++ 79 files changed, 8084 insertions(+), 166 deletions(-) create mode 100644 cmd/server/moderation_idempotency_sweep_job.go create mode 100644 cmd/server/moderation_idempotency_sweep_job_test.go create mode 100644 internal/api/handlers/imageproxy/moderation_test.go create mode 100644 internal/api/handlers/moderation/remove_content.go create mode 100644 internal/api/handlers/moderation/remove_content_test.go create mode 100644 internal/api/handlers/moderation/restore_content.go create mode 100644 internal/api/handlers/moderation/restore_content_test.go create mode 100644 internal/api/routes/moderation_comment_removal_integration_test.go create mode 100644 internal/api/routes/moderation_media_integration_test.go create mode 100644 internal/api/routes/moderation_reauthorization_integration_test.go create mode 100644 internal/config/moderation_idempotency_test.go create mode 100644 internal/core/comments/comment_moderation_consumer_integration_test.go create mode 100644 internal/core/comments/comment_moderation_integration_test.go create mode 100644 internal/core/comments/comment_moderation_view_test.go create mode 100644 internal/core/embeds/comment_image_cids_test.go create mode 100644 internal/core/imageproxy/cache_moderation_test.go create mode 100644 internal/core/imageproxy/moderation_test.go create mode 100644 internal/core/moderation/fake_store_test.go create mode 100644 internal/core/moderation/idempotency_test.go create mode 100644 internal/core/moderation/media.go create mode 100644 internal/core/moderation/mutation_validation.go create mode 100644 internal/core/moderation/remove.go create mode 100644 internal/core/moderation/remove_rules_test.go create mode 100644 internal/core/moderation/remove_validation_test.go create mode 100644 internal/core/moderation/restore.go create mode 100644 internal/core/moderation/restore_rules_test.go create mode 100644 internal/core/moderation/store.go create mode 100644 internal/core/moderation/subject_state.go create mode 100644 internal/db/migrations/049_moderation_state.sql create mode 100644 internal/db/postgres/moderation_actor_lock_integration_test.go create mode 100644 internal/db/postgres/moderation_concurrency_integration_test.go create mode 100644 internal/db/postgres/moderation_idempotency_integration_test.go create mode 100644 internal/db/postgres/moderation_media_blocks_integration_test.go create mode 100644 internal/db/postgres/moderation_repo.go create mode 100644 internal/db/postgres/moderation_repo_integration_test.go create mode 100644 tests/e2e/moderation_contract_test.go diff --git a/cmd/server/consumers.go b/cmd/server/consumers.go index e2625ec..fbaa453 100644 --- a/cmd/server/consumers.go +++ b/cmd/server/consumers.go @@ -263,7 +263,8 @@ func (a *application) registerFeedConsumers() ([]feedConsumer, error) { consumers = append(consumers, feedConsumer{ name: jetstream.ConsumerComments, handler: jetstream.NewCommentEventConsumer(a.commentRepo, a.db, - jetstream.WithCommentBridgeTrust(a.bridgeTrust)), + jetstream.WithCommentBridgeTrust(a.bridgeTrust), + jetstream.WithCommentMediaReconciler(a.commentMediaReconciler)), }) return consumers, nil diff --git a/cmd/server/main.go b/cmd/server/main.go index 15febe9..23008e7 100644 --- a/cmd/server/main.go +++ b/cmd/server/main.go @@ -19,6 +19,7 @@ import ( "Coves/internal/config" "Coves/internal/core/users" "Coves/internal/crypto/credentialcipher" + postgresRepo "Coves/internal/db/postgres" "Coves/internal/observability" _ "github.com/lib/pq" @@ -114,6 +115,8 @@ func run() error { startAggregatorTokenRefreshJob(backgroundCtx, &backgroundWG, app.apiKeyService) startDiscoverHotCleanupJob(backgroundCtx, &backgroundWG, app.discoverHotStateCleaner, discoverHotCleanupInterval, discoverHotCleanupDerivedRowBatchSize) + startModerationIdempotencySweepJob(backgroundCtx, &backgroundWG, + postgresRepo.NewModerationRepository(db), moderationIdempotencySweepInterval) // Nil when the driver is disabled, and passed as a typed nil would be a // non-nil interface — so the guard is here rather than inside the job. diff --git a/cmd/server/moderation_idempotency_sweep_job.go b/cmd/server/moderation_idempotency_sweep_job.go new file mode 100644 index 0000000..7df7bcf --- /dev/null +++ b/cmd/server/moderation_idempotency_sweep_job.go @@ -0,0 +1,36 @@ +package main + +import ( + "context" + "errors" + "log/slog" + "sync" + "time" +) + +const moderationIdempotencySweepInterval = time.Hour + +// moderationIdempotencySweeper deletes expired moderation idempotency keys. +type moderationIdempotencySweeper interface { + DeleteExpiredIdempotencyKeys(ctx context.Context, now time.Time) (int64, error) +} + +// startModerationIdempotencySweepJob periodically deletes expired keys. +func startModerationIdempotencySweepJob(ctx context.Context, waitGroup *sync.WaitGroup, sweeper moderationIdempotencySweeper, interval time.Duration) { + if sweeper == nil || interval <= 0 { + slog.Warn("moderation idempotency sweep job not started", "sweeper_present", sweeper != nil, "interval", interval) + return + } + runTicker(ctx, waitGroup, "moderation-idempotency-sweep", interval, func(ctx context.Context) { + deleted, err := sweeper.DeleteExpiredIdempotencyKeys(ctx, time.Now()) + if err != nil { + if !errors.Is(err, context.Canceled) { + slog.Error("moderation idempotency sweep failed", "error", err) + } + return + } + if deleted > 0 { + slog.Info("moderation idempotency sweep completed", "expired_keys_removed", deleted) + } + }) +} diff --git a/cmd/server/moderation_idempotency_sweep_job_test.go b/cmd/server/moderation_idempotency_sweep_job_test.go new file mode 100644 index 0000000..60132b4 --- /dev/null +++ b/cmd/server/moderation_idempotency_sweep_job_test.go @@ -0,0 +1,99 @@ +package main + +import ( + "context" + "errors" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +type moderationIdempotencySweeperFake struct { + called chan time.Time + err error +} + +func (fake *moderationIdempotencySweeperFake) DeleteExpiredIdempotencyKeys(ctx context.Context, now time.Time) (int64, error) { + select { + case fake.called <- now: + case <-ctx.Done(): + return 0, ctx.Err() + } + return 0, fake.err +} + +func TestStartModerationIdempotencySweepJobRunsOnStartupAndTickAndStops(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var waitGroup sync.WaitGroup + sweeper := &moderationIdempotencySweeperFake{called: make(chan time.Time, 4)} + started := time.Now() + startModerationIdempotencySweepJob(ctx, &waitGroup, sweeper, 20*time.Millisecond) + for call := 0; call < 2; call++ { + select { + case sweepTime := <-sweeper.called: + assert.False(t, sweepTime.Before(started), "sweep must use the current time") + assert.WithinDuration(t, time.Now(), sweepTime, time.Second) + case <-time.After(time.Second): + t.Fatalf("moderation idempotency sweep did not run on %s", []string{"startup", "tick"}[call]) + } + } + cancel() + requireModerationIdempotencySweepStops(t, &waitGroup) +} + +func TestStartModerationIdempotencySweepJobContinuesAfterError(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + var waitGroup sync.WaitGroup + sweeper := &moderationIdempotencySweeperFake{called: make(chan time.Time, 4), err: errors.New("database unavailable")} + startModerationIdempotencySweepJob(ctx, &waitGroup, sweeper, 20*time.Millisecond) + for call := 0; call < 2; call++ { + select { + case <-sweeper.called: + case <-time.After(time.Second): + t.Fatal("a failed sweep must not stop the next tick") + } + } + cancel() + requireModerationIdempotencySweepStops(t, &waitGroup) +} + +func TestStartModerationIdempotencySweepJobGuardsInvalidInputs(t *testing.T) { + for _, test := range []struct { + name string + sweeper moderationIdempotencySweeper + interval time.Duration + }{ + {name: "nil sweeper", interval: time.Second}, + {name: "zero interval", sweeper: &moderationIdempotencySweeperFake{called: make(chan time.Time, 1)}}, + {name: "negative interval", sweeper: &moderationIdempotencySweeperFake{called: make(chan time.Time, 1)}, interval: -time.Second}, + } { + t.Run(test.name, func(t *testing.T) { + var waitGroup sync.WaitGroup + startModerationIdempotencySweepJob(t.Context(), &waitGroup, test.sweeper, test.interval) + requireModerationIdempotencySweepStops(t, &waitGroup) + if fake, ok := test.sweeper.(*moderationIdempotencySweeperFake); ok { + assert.Empty(t, fake.called) + } + }) + } +} + +func requireModerationIdempotencySweepStops(t *testing.T, waitGroup *sync.WaitGroup) { + t.Helper() + done := make(chan struct{}) + go func() { + waitGroup.Wait() + close(done) + }() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("moderation idempotency sweep job did not stop") + } +} + +var _ moderationIdempotencySweeper = (*moderationIdempotencySweeperFake)(nil) diff --git a/cmd/server/wiring.go b/cmd/server/wiring.go index 347c0ad..c847f86 100644 --- a/cmd/server/wiring.go +++ b/cmd/server/wiring.go @@ -139,6 +139,7 @@ type application struct { userBlockService userblocks.Service adminReportService adminreports.Service moderationService moderation.Service + commentMediaReconciler jetstream.CommentMediaReconciler communitySuggestionService communitysuggestions.Service feedService communityFeeds.Service timelineService timeline.Service @@ -158,7 +159,8 @@ type application struct { // imageProxyHandler is nil when the image proxy is disabled. imageProxyHandler *imageproxyhandlers.Handler - // stopImageProxyCleanup halts the disk cache eviction job. Never nil. + // stopImageProxyCleanup halts the disk cache eviction job and the blocked + // media purge job. Never nil. stopImageProxyCleanup context.CancelFunc // closeOnce guards Close, which is reached from both serve and run's // deferred cleanup on every shutdown. @@ -205,9 +207,22 @@ func buildApplication( if err = app.buildServices(ctx); err != nil { return nil, err } - if err = app.buildImageProxy(); err != nil { + purger, err := app.buildImageProxy() + if err != nil { return nil, err } + app.moderationService = moderation.NewService( + moderation.NewRepositorySubjectReader(app.postRepo, app.commentRepo), + postgresRepo.NewModerationRepository(app.db), + moderation.Config{ + InstanceDID: app.cfg.Instance.DID, + IdempotencyRetention: app.cfg.Moderation.IdempotencyRetention, + MaxLiveIdempotencyKeys: app.cfg.Moderation.MaxLiveIdempotencyKeys, + Purger: purger, + }, + ) + app.commentMediaReconciler = moderation.NewMediaReconciler( + postgresRepo.NewModerationRepository(app.db), app.cfg.Instance.DID, purger) app.buildJetstreamInfrastructure() if err = app.buildBridgedVotePoller(); err != nil { return nil, err @@ -439,7 +454,6 @@ func (a *application) buildServices(ctx context.Context) error { a.buildDualAuth() a.instanceAdminAuth = buildInstanceAdminMiddleware(a.cfg, a.oauthClient, a.oauthStore, a.serviceAuthValidator) - a.moderationService = moderation.NewService(moderation.NewRepositorySubjectReader(a.postRepo, a.commentRepo)) slog.Info("instance moderation admins configured", "admin_count", len(a.cfg.Moderation.Admins)) // The SSRF hatch is open only in dev, where the links a developer pastes and @@ -838,7 +852,7 @@ func (a *application) authenticateInstanceWithPDS(ctx context.Context) { // is disabled — because the view builders need to know whether to emit proxy // URLs or direct blob URLs. In production, config.Validate has already refused // the disabled path unless the operator opted into it explicitly. -func (a *application) buildImageProxy() error { +func (a *application) buildImageProxy() (moderation.MediaPurger, error) { cfg := a.cfg.Media.ImageProxy // Published on every path, including the disabled one. Set explicitly at @@ -867,24 +881,25 @@ func (a *application) buildImageProxy() error { slog.Warn("[IMAGE-PROXY] disabled: image URLs will address PDS blob endpoints directly", "consequence", "media bypasses any scanning CDN and is blocked by the default Content-Security-Policy", ) - return nil + return nil, nil } if err := cfg.Validate(); err != nil { - return fmt.Errorf("image proxy configuration: %w", err) + return nil, fmt.Errorf("image proxy configuration: %w", err) } cache, err := imageproxy.NewDiskCache(cfg.CachePath, cfg.CacheMaxGB, cfg.CacheTTLDays) if err != nil { - return fmt.Errorf("creating image proxy cache: %w", err) + return nil, fmt.Errorf("creating image proxy cache: %w", err) } a.stopImageProxyCleanup = cache.StartCleanupJob(cfg.CleanupInterval) processor, err := imageproxy.NewProcessor(cfg.MaxSourceMegapixels) if err != nil { - return fmt.Errorf("creating image proxy processor: %w", err) + return nil, fmt.Errorf("creating image proxy processor: %w", err) } + moderationRepository := postgresRepo.NewModerationRepository(a.db) service, err := imageproxy.NewService( cache, processor, @@ -894,10 +909,19 @@ func (a *application) buildImageProxy() error { // that carries no credential. imageproxy.NewPDSFetcher(cfg.FetchTimeout, cfg.MaxSourceSizeMB, imageproxy.PrivateHostOptions(a.allowPrivateHosts())...), + moderationRepository, cfg, ) if err != nil { - return fmt.Errorf("creating image proxy service: %w", err) + return nil, fmt.Errorf("creating image proxy service: %w", err) + } + // The startup sweep finishes any moderation purge a restart interrupted; + // later sweeps retry purges the disk refused. + stopCacheCleanup := a.stopImageProxyCleanup + stopBlockedMediaPurge := service.StartActiveBlockPurgeJob(moderationRepository, cfg.CleanupInterval) + a.stopImageProxyCleanup = func() { + stopCacheCleanup() + stopBlockedMediaPurge() } a.imageProxyHandler = imageproxyhandlers.NewHandler(service, a.identityResolver) @@ -917,7 +941,7 @@ func (a *application) buildImageProxy() error { "process_queue_wait", cfg.ProcessQueueWait, "max_in_flight_requests", cfg.MaxInFlightRequests, ) - return nil + return service, nil } func (a *application) buildJetstreamInfrastructure() { diff --git a/go.mod b/go.mod index 1cbc509..87d880e 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,10 @@ require ( github.com/gorilla/websocket v1.5.3 github.com/hashicorp/go-retryablehttp v0.7.8 github.com/hashicorp/golang-lru/v2 v2.0.7 + github.com/ipfs/go-cid v0.6.2 github.com/lib/pq v1.12.3 + github.com/multiformats/go-multibase v0.3.0 + github.com/multiformats/go-multihash v0.2.3 github.com/pressly/goose/v3 v3.28.0 github.com/rivo/uniseg v0.4.7 github.com/stretchr/testify v1.12.1 @@ -45,7 +48,6 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-block-format v0.2.0 // indirect github.com/ipfs/go-blockservice v0.5.2 // indirect - github.com/ipfs/go-cid v0.6.2 // indirect github.com/ipfs/go-datastore v0.6.0 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect @@ -71,8 +73,6 @@ require ( github.com/mr-tron/base58 v1.3.0 // indirect github.com/multiformats/go-base32 v0.1.0 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect - github.com/multiformats/go-multibase v0.3.0 // indirect - github.com/multiformats/go-multihash v0.2.3 // indirect github.com/multiformats/go-varint v0.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/polydawn/refmt v0.90.0 // indirect diff --git a/internal/api/handlers/imageproxy/decompression_bomb_test.go b/internal/api/handlers/imageproxy/decompression_bomb_test.go index aed8a88..f794d02 100644 --- a/internal/api/handlers/imageproxy/decompression_bomb_test.go +++ b/internal/api/handlers/imageproxy/decompression_bomb_test.go @@ -33,6 +33,8 @@ type missCache struct{} func (missCache) Get(_, _, _ string) ([]byte, bool, error) { return nil, false, nil } func (missCache) Set(_, _, _ string, _ []byte) error { return nil } func (missCache) Delete(_, _, _ string) error { return nil } +func (missCache) DeleteOwner(_, _ string) error { return nil } +func (missCache) DeleteCID(_ string) error { return nil } func (missCache) Cleanup() (int, error) { return 0, nil } // pngChunk frames one PNG chunk: length, type, data, CRC32 over type+data. @@ -111,7 +113,7 @@ func TestHandler_RejectsDecompressionBombBeforeDecoding(t *testing.T) { if err != nil { t.Fatalf("NewProcessor: %v", err) } - service, err := imageproxy.NewService(missCache{}, processor, fetcher, imageproxy.DefaultConfig()) + service, err := imageproxy.NewService(missCache{}, processor, fetcher, allowAllBlockChecker{}, imageproxy.DefaultConfig()) if err != nil { t.Fatalf("NewService: %v", err) } diff --git a/internal/api/handlers/imageproxy/handler.go b/internal/api/handlers/imageproxy/handler.go index 40df65f..7206bea 100644 --- a/internal/api/handlers/imageproxy/handler.go +++ b/internal/api/handlers/imageproxy/handler.go @@ -33,6 +33,8 @@ type Service interface { // cid: the content identifier of the blob // pdsURL: the URL of the user's PDS GetImage(ctx context.Context, preset, did, cid, pdsURL string) ([]byte, error) + // IsBlobBlocked reports whether moderation blocks serving the blob. + IsBlobBlocked(ctx context.Context, did, cid string) (bool, error) } // Handler handles HTTP requests for the image proxy. @@ -80,8 +82,12 @@ func (h *Handler) HandleImage(w http.ResponseWriter, r *http.Request) { return } - // Validate CID format (must be valid base32/base58 CID) - if err := imageproxy.ValidateCID(cid); err != nil { + // Decode the CID and continue with its canonical form. Every multibase + // encoding of one CID names the same blob on the PDS, so the block check, + // the ETag, the cache key and the fetch must all see the same string, or a + // re-encoded CID would bypass a moderation block. + cid, err := imageproxy.CanonicalCID(cid) + if err != nil { writeErrorResponse(w, http.StatusBadRequest, "invalid CID format") return } @@ -91,6 +97,16 @@ func (h *Handler) HandleImage(w http.ResponseWriter, r *http.Request) { // Check If-None-Match header for 304 response if r.Header.Get("If-None-Match") == etag { + blocked, err := h.service.IsBlobBlocked(r.Context(), did, cid) + if err != nil { + logBlockCheckFailure(err, did, cid) + writeErrorResponse(w, http.StatusServiceUnavailable, "image moderation unavailable") + return + } + if blocked { + writeErrorResponse(w, http.StatusNotFound, "blob not found") + return + } w.WriteHeader(http.StatusNotModified) return } @@ -159,15 +175,19 @@ func getPDSEndpoint(doc *identity.DIDDocument) string { // budget refusal is logged at WARN with the blob's identity: it is the // signature of a decompression bomb, so an operator needs to be able to find // the repo and the blob afterwards. A processing failure is logged at ERROR -// with the underlying error because it is our fault. An SSRF refusal is logged +// with the underlying error because it is our fault, and so is a failed +// moderation block lookup. An SSRF refusal is logged // at WARN because the response deliberately hides it. Load shedding is NOT // logged here: the service already logs it once with the counter, and a // second line per shed request would be the flood logging itself. The // remaining branches are ordinary client errors and stay quiet. func handleServiceError(w http.ResponseWriter, err error, preset, did, cid string) { switch { - case errors.Is(err, imageproxy.ErrPDSNotFound): + case errors.Is(err, imageproxy.ErrPDSNotFound), errors.Is(err, imageproxy.ErrBlobBlocked): writeErrorResponse(w, http.StatusNotFound, "blob not found") + case errors.Is(err, imageproxy.ErrBlockCheckFailed): + logBlockCheckFailure(err, did, cid) + writeErrorResponse(w, http.StatusServiceUnavailable, "image moderation unavailable") case errors.Is(err, imageproxy.ErrPDSTimeout): writeErrorResponse(w, http.StatusGatewayTimeout, "request timed out") // ONE BRANCH FOR BOTH, so the status and the body cannot drift apart. A @@ -229,6 +249,20 @@ func handleServiceError(w http.ResponseWriter, err error, preset, did, cid strin } } +// logBlockCheckFailure logs a failed moderation block lookup at ERROR: the +// response is a generic 503, so the log line is the only record of the cause. +// A client that went away is not a failure of ours and is not logged. +func logBlockCheckFailure(err error, did, cid string) { + if errors.Is(err, context.Canceled) { + return + } + slog.Error("[IMAGE-PROXY] media block check failed", + "did", did, + "cid", cid, + "error", err, + ) +} + // writeErrorResponse writes a plain text error response. // For the image proxy, we use simple text responses rather than JSON // since the expected response is binary image data. diff --git a/internal/api/handlers/imageproxy/handler_test.go b/internal/api/handlers/imageproxy/handler_test.go index 425d21b..d0f3787 100644 --- a/internal/api/handlers/imageproxy/handler_test.go +++ b/internal/api/handlers/imageproxy/handler_test.go @@ -24,12 +24,26 @@ const ( // validTestDID is a valid did:plc identifier (24 lowercase base32 chars after did:plc:) validTestDID = "did:plc:z72i7hdynmk6r22z27h6tvur" // validTestCID is a valid CIDv1 base32 identifier - validTestCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + validTestCID = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" ) // mockService implements imageproxy.Service for testing type mockService struct { - getImageFunc func(ctx context.Context, preset, did, cid, pdsURL string) ([]byte, error) + getImageFunc func(ctx context.Context, preset, did, cid, pdsURL string) ([]byte, error) + isBlobBlockedFunc func(ctx context.Context, did, cid string) (bool, error) +} + +type allowAllBlockChecker struct{} + +func (allowAllBlockChecker) IsBlocked(context.Context, string, string) (bool, error) { + return false, nil +} + +func (m *mockService) IsBlobBlocked(ctx context.Context, did, cid string) (bool, error) { + if m.isBlobBlockedFunc != nil { + return m.isBlobBlockedFunc(ctx, did, cid) + } + return false, nil } func (m *mockService) GetImage(ctx context.Context, preset, did, cid, pdsURL string) ([]byte, error) { diff --git a/internal/api/handlers/imageproxy/moderation_test.go b/internal/api/handlers/imageproxy/moderation_test.go new file mode 100644 index 0000000..61ca0c4 --- /dev/null +++ b/internal/api/handlers/imageproxy/moderation_test.go @@ -0,0 +1,159 @@ +package imageproxy + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "Coves/internal/atproto/identity" + "Coves/internal/core/imageproxy" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multibase" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func moderationImageRequest(etag bool) *http.Request { + request := createTestRequest(http.MethodGet, "/img/avatar/plain/"+validTestDID+"/"+validTestCID, map[string]string{ + "preset": "avatar", "did": validTestDID, "cid": validTestCID, + }) + if etag { + request.Header.Set("If-None-Match", `"avatar-`+validTestCID+`"`) + } + return request +} + +func TestHandler_ModerationErrorsDoNotDiscloseBlockedBlob(t *testing.T) { + for _, test := range []struct { + name string + err error + status int + body string + }{ + {"blocked blob", fmt.Errorf("%w: private decision", imageproxy.ErrBlobBlocked), http.StatusNotFound, "blob not found"}, + {"block lookup unavailable", fmt.Errorf("%w: database unavailable", imageproxy.ErrBlockCheckFailed), http.StatusServiceUnavailable, ""}, + } { + t.Run(test.name, func(t *testing.T) { + service := &mockService{getImageFunc: func(context.Context, string, string, string, string) ([]byte, error) { + return nil, test.err + }} + response := httptest.NewRecorder() + NewHandler(service, resolverForPDS("https://pds.example.com")).HandleImage(response, moderationImageRequest(false)) + assert.Equal(t, test.status, response.Code) + if test.body != "" { + assert.Equal(t, test.body, response.Body.String()) + } else { + assert.NotEmpty(t, response.Body.String()) + assert.NotContains(t, response.Body.String(), "database unavailable") + } + assert.Equal(t, "no-store", response.Header().Get("Cache-Control")) + assert.Empty(t, response.Header().Get("ETag")) + }) + } +} + +func TestHandler_ConditionalRequestChecksBlockBeforeReturning304(t *testing.T) { + for _, test := range []struct { + name string + blocked bool + lookupErr error + wantStatus int + wantBody string + }{ + {"blocked", true, nil, http.StatusNotFound, "blob not found"}, + {"unblocked", false, nil, http.StatusNotModified, ""}, + {"lookup failed", false, fmt.Errorf("lookup unavailable"), http.StatusServiceUnavailable, ""}, + } { + t.Run(test.name, func(t *testing.T) { + calls := 0 + service := &mockService{ + isBlobBlockedFunc: func(_ context.Context, did, cid string) (bool, error) { + calls++ + assert.Equal(t, validTestDID, did) + assert.Equal(t, validTestCID, cid) + return test.blocked, test.lookupErr + }, + getImageFunc: func(context.Context, string, string, string, string) ([]byte, error) { + t.Error("conditional request must not fetch image data") + return nil, nil + }, + } + resolver := &mockIdentityResolver{resolveDIDFunc: func(context.Context, string) (*identity.DIDDocument, error) { + t.Error("conditional request must not resolve a PDS") + return nil, nil + }} + response := httptest.NewRecorder() + NewHandler(service, resolver).HandleImage(response, moderationImageRequest(true)) + assert.Equal(t, 1, calls, "If-None-Match must consult the block checker") + assert.Equal(t, test.wantStatus, response.Code) + if test.wantBody != "" || test.wantStatus == http.StatusNotModified { + assert.Equal(t, test.wantBody, response.Body.String()) + } else { + assert.NotEmpty(t, response.Body.String()) + assert.NotContains(t, response.Body.String(), "lookup unavailable") + } + if test.wantStatus != http.StatusNotModified { + require.Equal(t, "no-store", response.Header().Get("Cache-Control")) + } + }) + } +} + +// Every multibase encoding of a CID names the same blob on the PDS, so the +// handler must hand the block check, the ETag and the fetch one canonical form. +func TestHandler_CanonicalizesCIDBeforeBlockCheckAndFetch(t *testing.T) { + parsed, err := cid.Decode(validTestCID) + require.NoError(t, err) + require.Equal(t, validTestCID, parsed.String()) + alias, err := parsed.StringOfBase(multibase.Base58BTC) + require.NoError(t, err) + + t.Run("fetch and ETag use the canonical CID", func(t *testing.T) { + var fetched string + service := &mockService{getImageFunc: func(_ context.Context, _, _, requested, _ string) ([]byte, error) { + fetched = requested + return []byte("image"), nil + }} + response := httptest.NewRecorder() + NewHandler(service, resolverForPDS("https://pds.example.com")).HandleImage(response, createTestRequest(http.MethodGet, + "/img/avatar/plain/"+validTestDID+"/"+alias, map[string]string{"preset": "avatar", "did": validTestDID, "cid": alias})) + require.Equal(t, http.StatusOK, response.Code) + assert.Equal(t, validTestCID, fetched) + assert.Equal(t, `"avatar-`+validTestCID+`"`, response.Header().Get("ETag")) + }) + + t.Run("conditional request checks the canonical CID", func(t *testing.T) { + var checked string + service := &mockService{ + isBlobBlockedFunc: func(_ context.Context, _, requested string) (bool, error) { + checked = requested + return true, nil + }, + getImageFunc: func(context.Context, string, string, string, string) ([]byte, error) { + t.Error("a blocked conditional request must not fetch") + return nil, nil + }, + } + request := createTestRequest(http.MethodGet, "/img/avatar/plain/"+validTestDID+"/"+alias, + map[string]string{"preset": "avatar", "did": validTestDID, "cid": alias}) + request.Header.Set("If-None-Match", `"avatar-`+validTestCID+`"`) + response := httptest.NewRecorder() + NewHandler(service, resolverForPDS("https://pds.example.com")).HandleImage(response, request) + assert.Equal(t, http.StatusNotFound, response.Code) + assert.Equal(t, validTestCID, checked) + }) + + t.Run("a syntax-valid string that is not a CID is a 400 without a fetch", func(t *testing.T) { + service := &mockService{getImageFunc: func(context.Context, string, string, string, string) ([]byte, error) { + t.Error("an undecodable CID must not be fetched") + return nil, nil + }} + response := httptest.NewRecorder() + NewHandler(service, resolverForPDS("https://pds.example.com")).HandleImage(response, createTestRequest(http.MethodGet, + "/img/avatar/plain/"+validTestDID+"/bafynotacid", map[string]string{"preset": "avatar", "did": validTestDID, "cid": "bafynotacid"})) + assert.Equal(t, http.StatusBadRequest, response.Code) + assert.Equal(t, "invalid CID format", response.Body.String()) + }) +} diff --git a/internal/api/handlers/imageproxy/proxy_serving_test.go b/internal/api/handlers/imageproxy/proxy_serving_test.go index 1d8c000..1984636 100644 --- a/internal/api/handlers/imageproxy/proxy_serving_test.go +++ b/internal/api/handlers/imageproxy/proxy_serving_test.go @@ -57,6 +57,12 @@ import ( // and a tight budget here would turn a loaded CI machine into a 502. const defaultFetchTimeout = 30 * time.Second +type allowAllBlockChecker struct{} + +func (allowAllBlockChecker) IsBlocked(context.Context, string, string) (bool, error) { + return false, nil +} + // fixedPDSResolver is an identity.Resolver that sends every DID to one PDS. // // Only ResolveDID is implemented because that is the only method the proxy @@ -140,6 +146,7 @@ func newProxyServerWithCache(t *testing.T, resolver identity.Resolver, fetchTime // core/imageproxy/fetcher_guard_test.go, whose fetchers are built // without this option and assert the listener is never reached. imageproxycore.NewPDSFetcher(fetchTimeout, 10, imageproxycore.WithPrivateHostsAllowed()), + allowAllBlockChecker{}, imageproxycore.Config{ Enabled: true, CachePath: cacheDir, @@ -252,7 +259,7 @@ func assertImageSize(t *testing.T, body []byte, wantWidth, wantHeight int) { func TestImageProxy_ServesProcessedBlob(t *testing.T) { t.Parallel() - const cid = "bafybeimockimagetest123" + const cid = "bafkreicdmtgb7deaakqrghapqrdjva63vqcieyzzi66q2pokcwyotdermy" did := "did:plc:" + testkit.UniqueID(t) upstream := newBlobServer(t, map[string]func(http.ResponseWriter){ @@ -272,7 +279,7 @@ func TestImageProxy_ServesProcessedBlob(t *testing.T) { }) t.Run("a blob the PDS does not hold is a 404", func(t *testing.T) { - resp, _ := fetch(t, proxyURL(server, "avatar", did, "nonexistentcid"), nil) + resp, _ := fetch(t, proxyURL(server, "avatar", did, "bafkreickn4h365ejzx3jx6uvmu6atijrecyeavikim7dyf5wer7tcu4zme"), nil) assert.Equal(t, http.StatusNotFound, resp.StatusCode) }) @@ -290,7 +297,7 @@ func TestImageProxy_UpstreamFailuresAreBadGateway(t *testing.T) { // Well-formed CIDs: these must travel past validation so that the failure // under test is the fetch, not the parse. - const validCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + const validCID = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" did := "did:plc:" + testkit.UniqueID(t) t.Run("the resolved PDS refuses the connection", func(t *testing.T) { @@ -358,8 +365,8 @@ func TestImageProxy_UndecodableUpstreamBytes(t *testing.T) { func TestImageProxy_PresetGeometry(t *testing.T) { t.Parallel() - const cid = "bafybeipresetgeometry123" - const smallCID = "bafybeismallsource123" + const cid = "bafkreieswtoyhdnkf552cgtp3jebltovvu6rc56hzhmuvgzghndwrtbe5i" + const smallCID = "bafkreifh4b76hmqvsksqlq3tuovcrekvsl7buylbztorfnwti3bp3zp4pq" did := "did:plc:" + testkit.UniqueID(t) // 1000x1000 so that both directions are exercised: the cover presets crop diff --git a/internal/api/handlers/moderation/get_subject_state_test.go b/internal/api/handlers/moderation/get_subject_state_test.go index 1183a13..d53edf8 100644 --- a/internal/api/handlers/moderation/get_subject_state_test.go +++ b/internal/api/handlers/moderation/get_subject_state_test.go @@ -21,6 +21,8 @@ import ( ) type subjectStateServiceFake struct { + // The mutations are not exercised by the read handler; calling one panics. + moderation.Service state *moderation.SubjectState err error subjects []string diff --git a/internal/api/handlers/moderation/remove_content.go b/internal/api/handlers/moderation/remove_content.go new file mode 100644 index 0000000..79f548b --- /dev/null +++ b/internal/api/handlers/moderation/remove_content.go @@ -0,0 +1,190 @@ +package moderation + +import ( + "errors" + "log/slog" + "mime" + "net/http" + "strings" + "time" + + "Coves/internal/api/middleware" + "Coves/internal/api/reqbody" + "Coves/internal/api/xrpc" + "Coves/internal/core/moderation" +) + +// RemoveContentHandler serves social.coves.moderation.removeContent. +type RemoveContentHandler struct { + service moderation.Service +} + +// NewRemoveContentHandler builds the removeContent handler. +func NewRemoveContentHandler(service moderation.Service) *RemoveContentHandler { + return &RemoveContentHandler{service: service} +} + +// HandleRemoveContent handles POST /xrpc/social.coves.moderation.removeContent. +func (h *RemoveContentHandler) HandleRemoveContent(w http.ResponseWriter, r *http.Request) { + if !requireMutationPOSTJSON(w, r) { + return + } + var input struct { + Subject strongRefView `json:"subject"` + ExpectedVersion string `json:"expectedVersion"` + IdempotencyKey string `json:"idempotencyKey"` + Reason string `json:"reason"` + PrivateNote string `json:"privateNote"` + } + if !decodeMutationRequest(w, r, &input) { + return + } + actorDID := middleware.GetUserDID(r) + if actorDID == "" { + xrpc.WriteError(w, http.StatusUnauthorized, "AuthRequired", "Authentication required") + return + } + result, err := h.service.RemoveContent(r.Context(), actorDID, moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: input.Subject.URI, CID: input.Subject.CID}, + ExpectedVersion: input.ExpectedVersion, IdempotencyKey: input.IdempotencyKey, + Reason: input.Reason, PrivateNote: input.PrivateNote, + }) + writeMutationResult(w, "removeContent", result, err) +} + +func requireMutationPOSTJSON(w http.ResponseWriter, r *http.Request) bool { + if r.Method != http.MethodPost { + xrpc.WriteError(w, http.StatusMethodNotAllowed, "MethodNotAllowed", "Method not allowed") + return false + } + mediaType, _, err := mime.ParseMediaType(r.Header.Get("Content-Type")) + if err != nil || mediaType != "application/json" { + xrpc.WriteError(w, http.StatusBadRequest, "InvalidRequest", "Content-Type must be application/json") + return false + } + return true +} + +func decodeMutationRequest(w http.ResponseWriter, r *http.Request, input any) bool { + if err := reqbody.DecodeJSON(w, r, reqbody.LimitMedium, input); err != nil { + xrpc.WriteError(w, http.StatusBadRequest, "InvalidRequest", "Invalid request body") + return false + } + return true +} + +func writeMutationResult(w http.ResponseWriter, operation string, result *moderation.MutationResult, err error) { + if err != nil || result == nil { + writeMutationError(w, operation, err) + return + } + state := result.State + view := subjectStateView{ + Subject: state.Subject, Version: state.Version, + Moderation: moderationView{State: state.Moderation.State}, RecordState: state.RecordState, + } + if state.CurrentSubject != nil { + view.CurrentSubject = &strongRefView{URI: state.CurrentSubject.URI, CID: state.CurrentSubject.CID} + } + if state.LocalRemoval != nil { + view.LocalRemoval = &actionRefView{ServiceDID: state.LocalRemoval.ServiceDID, ActionID: state.LocalRemoval.ActionID} + } + for _, label := range state.LocalLabels { + view.LocalLabels = append(view.LocalLabels, localLabelView{ + Value: label.Value, + Action: actionRefView{ServiceDID: label.Action.ServiceDID, ActionID: label.Action.ActionID}, + }) + } + response := struct { + Outcome string `json:"outcome"` + State subjectStateView `json:"state"` + Action *adminActionView `json:"action,omitempty"` + }{Outcome: result.Outcome, State: view} + if result.Action != nil { + action := result.Action + projection := actionView{ + Ref: actionRefView{ServiceDID: action.AuthorityDID, ActionID: action.ID}, + Action: action.Action, AuthorityDID: action.AuthorityDID, + Scope: scopeView{Kind: action.ScopeKind, CommunityDID: action.ScopeCommunityDID}, + CreatedAt: action.CreatedAt.Format(time.RFC3339Nano), Origin: action.Origin, + Reason: action.Reason, Actor: actorView{DID: action.ActorDID}, + } + if action.SubjectURI != "" { + projection.Subject = &subjectRefView{URI: action.SubjectURI, CID: action.ObservedCID} + } + if action.ReversesActionID != "" { + projection.Reverses = &actionRefView{ServiceDID: action.AuthorityDID, ActionID: action.ReversesActionID} + } + response.Action = &adminActionView{Action: projection, ActorDID: action.ActorDID, PrivateNote: action.PrivateNote} + } + xrpc.WriteJSON(w, http.StatusOK, response) +} + +type adminActionView struct { + Action actionView `json:"action"` + ActorDID string `json:"actorDid,omitempty"` + PrivateNote string `json:"privateNote,omitempty"` +} + +type actionView struct { + Ref actionRefView `json:"ref"` + Action string `json:"action"` + AuthorityDID string `json:"authorityDid"` + Scope scopeView `json:"scope"` + CreatedAt string `json:"createdAt"` + Origin string `json:"origin"` + Subject *subjectRefView `json:"subject,omitempty"` + Reason string `json:"reason,omitempty"` + Actor actorView `json:"actor"` + Reverses *actionRefView `json:"reverses,omitempty"` +} + +// subjectRefView is defs#subjectRef: the CID is optional because an action can +// target a subject whose record the AppView never observed. +type subjectRefView struct { + URI string `json:"uri"` + CID string `json:"cid,omitempty"` +} + +type scopeView struct { + Kind string `json:"kind"` + CommunityDID string `json:"communityDid,omitempty"` +} + +type actorView struct { + DID string `json:"did"` +} + +func writeMutationError(w http.ResponseWriter, operation string, err error) { + for _, entry := range []struct { + cause error + code string + }{ + {moderation.ErrInvalidRequest, "InvalidRequest"}, + {moderation.ErrInvalidSubject, "InvalidSubject"}, + {moderation.ErrSubjectNotFound, "SubjectNotFound"}, + {moderation.ErrDecisionNotFound, "DecisionNotFound"}, + {moderation.ErrInvalidDecision, "InvalidDecision"}, + {moderation.ErrContentChanged, "ContentChanged"}, + {moderation.ErrStateConflict, "StateConflict"}, + {moderation.ErrIdempotencyConflict, "IdempotencyConflict"}, + {moderation.ErrUnsupportedReason, "UnsupportedReason"}, + } { + if errors.Is(err, entry.cause) { + // Rule errors carry only fixed text and configured limits, such as + // "post removal is unsupported" or the live-key limit, never request + // payloads, so the detail is safe to show the caller. + xrpc.WriteError(w, http.StatusBadRequest, entry.code, strings.TrimPrefix(err.Error(), "moderation: ")) + return + } + } + if errors.Is(err, moderation.ErrModerationUnavailable) { + // Subject AT-URIs, DIDs and database errors carry no credentials or + // private notes, so the full error is logged; the response stays generic. + slog.Error("moderation mutation unavailable", "operation", operation, "error", err) + xrpc.WriteError(w, http.StatusServiceUnavailable, "ModerationUnavailable", "Moderation service temporarily unavailable") + return + } + slog.Error("unexpected moderation mutation failure", "operation", operation, "error", err) + xrpc.WriteError(w, http.StatusInternalServerError, "InternalServerError", "An internal error occurred") +} diff --git a/internal/api/handlers/moderation/remove_content_test.go b/internal/api/handlers/moderation/remove_content_test.go new file mode 100644 index 0000000..9224c02 --- /dev/null +++ b/internal/api/handlers/moderation/remove_content_test.go @@ -0,0 +1,268 @@ +package moderation + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "Coves/internal/api/middleware" + "Coves/internal/core/moderation" + "Coves/internal/validation" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + mutationActorDID = "did:plc:moderationadmin" + mutationInstanceDID = "did:web:moderation.test" + mutationSubjectURI = "at://did:plc:commentauthor/social.coves.community.comment/3kabc" + mutationSubjectCID = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" + mutationReason = "social.coves.moderation.defs#reasonSpam" + mutationNote = "private-note-secret" + removeBody = `{"subject":{"uri":"` + mutationSubjectURI + `","cid":"` + mutationSubjectCID + `"},"expectedVersion":"v0","idempotencyKey":"remove-key","reason":"` + mutationReason + `","privateNote":"` + mutationNote + `"}` +) + +type mutationServiceFake struct { + moderation.Service + removeCalls []moderation.RemoveContentRequest + restoreCalls []moderation.RestoreContentRequest + actors []string + result *moderation.MutationResult + err error +} + +func (fake *mutationServiceFake) RemoveContent(_ context.Context, actorDID string, request moderation.RemoveContentRequest) (*moderation.MutationResult, error) { + fake.actors = append(fake.actors, actorDID) + fake.removeCalls = append(fake.removeCalls, request) + return fake.result, fake.err +} + +func (fake *mutationServiceFake) RestoreContent(_ context.Context, actorDID string, request moderation.RestoreContentRequest) (*moderation.MutationResult, error) { + fake.actors = append(fake.actors, actorDID) + fake.restoreCalls = append(fake.restoreCalls, request) + return fake.result, fake.err +} + +func mutationRequest(method, path, body, contentType string) *http.Request { + request := httptest.NewRequest(method, path, strings.NewReader(body)) + if contentType != "" { + request.Header.Set("Content-Type", contentType) + } + return request.WithContext(context.WithValue(request.Context(), middleware.UserDIDKey, mutationActorDID)) +} + +func mutationResult(actionKind string, privateNote string) *moderation.MutationResult { + state := moderation.SubjectState{ + Subject: mutationSubjectURI, Version: "v1", RecordState: moderation.RecordStatePresent, + CurrentSubject: &moderation.StrongRef{URI: mutationSubjectURI, CID: mutationSubjectCID}, + } + action := &moderation.Action{ + ID: "action-1", ActorDID: mutationActorDID, AuthorityDID: mutationInstanceDID, + ScopeKind: moderation.ScopeInstance, SubjectURI: mutationSubjectURI, ObservedCID: mutationSubjectCID, + Action: actionKind, Origin: moderation.OriginLocal, Reason: mutationReason, + PrivateNote: privateNote, CreatedAt: time.Date(2026, time.September, 24, 12, 30, 0, 0, time.UTC), + } + if actionKind == moderation.ActionRestore { + state.Moderation.State = moderation.ModerationStateClear + action.ReversesActionID = "action-previous" + } else { + state.Moderation.State = moderation.ModerationStateRemoved + state.LocalRemoval = &moderation.ActionRef{ServiceDID: mutationInstanceDID, ActionID: action.ID} + } + return &moderation.MutationResult{Outcome: moderation.OutcomeApplied, State: state, Action: action} +} + +func assertMutationResponse(t *testing.T, response *httptest.ResponseRecorder, result *moderation.MutationResult) { + t.Helper() + require.Equal(t, http.StatusOK, response.Code) + assert.Equal(t, "application/json", response.Header().Get("Content-Type")) + var body map[string]any + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + state := map[string]any{ + "subject": mutationSubjectURI, "version": result.State.Version, + "moderation": map[string]any{"state": result.State.Moderation.State}, + "recordState": "present", "currentSubject": map[string]any{"uri": mutationSubjectURI, "cid": mutationSubjectCID}, + } + if result.State.LocalRemoval != nil { + state["localRemoval"] = map[string]any{"serviceDid": mutationInstanceDID, "actionId": result.State.LocalRemoval.ActionID} + } + want := map[string]any{"outcome": result.Outcome, "state": state} + if result.Action != nil { + action := result.Action + projection := map[string]any{ + "ref": map[string]any{"serviceDid": action.AuthorityDID, "actionId": action.ID}, + "action": action.Action, "authorityDid": action.AuthorityDID, + "scope": map[string]any{"kind": action.ScopeKind}, "createdAt": action.CreatedAt.Format(time.RFC3339), + "origin": action.Origin, "subject": map[string]any{"uri": action.SubjectURI, "cid": action.ObservedCID}, + "reason": action.Reason, "actor": map[string]any{"did": action.ActorDID}, + } + if action.ReversesActionID != "" { + projection["reverses"] = map[string]any{"serviceDid": action.AuthorityDID, "actionId": action.ReversesActionID} + } + adminAction := map[string]any{"action": projection, "actorDid": action.ActorDID} + if action.PrivateNote != "" { + adminAction["privateNote"] = action.PrivateNote + } + want["action"] = adminAction + } + assert.Equal(t, want, body, "response must omit null optional fields and expose the exact action projection") + catalog := lexicon.NewBaseCatalog() + require.NoError(t, catalog.LoadDirectory("../../../atproto/lexicon")) + decoded, err := atdata.UnmarshalJSON(response.Body.Bytes()) + require.NoError(t, err) + assert.NoError(t, validation.ValidateData(catalog, decoded, "social.coves.moderation.defs#mutationResult", 0)) +} + +func assertMutationError(t *testing.T, response *httptest.ResponseRecorder, status int, code string) { + t.Helper() + assert.Equal(t, status, response.Code) + var body map[string]any + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + assert.Equal(t, code, body["error"]) + assert.NotEmpty(t, body["message"]) + assert.Len(t, body, 2, "errors expose only error and message") + assert.NotContains(t, response.Body.String(), mutationNote) +} + +func TestRemoveContentHandlerPassesContextActorAndExactlyMapsRequest(t *testing.T) { + result := mutationResult(moderation.ActionRemove, mutationNote) + fake := &mutationServiceFake{result: result} + response := httptest.NewRecorder() + request := mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", removeBody, "application/json") + NewRemoveContentHandler(fake).HandleRemoveContent(response, request) + assert.Equal(t, []string{mutationActorDID}, fake.actors) + assert.Equal(t, []moderation.RemoveContentRequest{{ + Subject: moderation.StrongRef{URI: mutationSubjectURI, CID: mutationSubjectCID}, + ExpectedVersion: "v0", IdempotencyKey: "remove-key", Reason: mutationReason, PrivateNote: mutationNote, + }}, fake.removeCalls) + assertMutationResponse(t, response, result) +} + +func TestRemoveContentHandlerOmitsActionForUnchanged(t *testing.T) { + result := mutationResult(moderation.ActionRemove, "") + result.Outcome = moderation.OutcomeUnchanged + result.Action = nil + fake := &mutationServiceFake{result: result} + response := httptest.NewRecorder() + NewRemoveContentHandler(fake).HandleRemoveContent(response, + mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", removeBody, "application/json")) + assertMutationResponse(t, response, result) +} + +func TestRemoveContentHandlerRejectsNonJSONInvalidJSONAndGET(t *testing.T) { + for _, test := range []struct { + name, method, body, contentType string + status int + }{ + {"text/plain", http.MethodPost, removeBody, "text/plain", http.StatusBadRequest}, + {"form", http.MethodPost, removeBody, "application/x-www-form-urlencoded", http.StatusBadRequest}, + {"missing content type", http.MethodPost, removeBody, "", http.StatusBadRequest}, + {"malformed JSON", http.MethodPost, `{"subject":`, "application/json", http.StatusBadRequest}, + {"GET", http.MethodGet, "", "", http.StatusMethodNotAllowed}, + } { + t.Run(test.name, func(t *testing.T) { + fake := &mutationServiceFake{result: mutationResult(moderation.ActionRemove, "")} + response := httptest.NewRecorder() + NewRemoveContentHandler(fake).HandleRemoveContent(response, + mutationRequest(test.method, "/xrpc/social.coves.moderation.removeContent", test.body, test.contentType)) + assert.Equal(t, test.status, response.Code) + if test.status == http.StatusBadRequest { + assertMutationError(t, response, test.status, "InvalidRequest") + } + assert.Empty(t, fake.removeCalls) + }) + } +} + +var mutationErrors = []struct { + name string + err error + status int + code string +}{ + {"invalid request", moderation.ErrInvalidRequest, http.StatusBadRequest, "InvalidRequest"}, + {"invalid subject", moderation.ErrInvalidSubject, http.StatusBadRequest, "InvalidSubject"}, + {"subject not found", moderation.ErrSubjectNotFound, http.StatusBadRequest, "SubjectNotFound"}, + {"decision not found", moderation.ErrDecisionNotFound, http.StatusBadRequest, "DecisionNotFound"}, + {"invalid decision", moderation.ErrInvalidDecision, http.StatusBadRequest, "InvalidDecision"}, + {"content changed", moderation.ErrContentChanged, http.StatusBadRequest, "ContentChanged"}, + {"state conflict", moderation.ErrStateConflict, http.StatusBadRequest, "StateConflict"}, + {"idempotency conflict", moderation.ErrIdempotencyConflict, http.StatusBadRequest, "IdempotencyConflict"}, + {"unsupported reason", moderation.ErrUnsupportedReason, http.StatusBadRequest, "UnsupportedReason"}, + {"unavailable", moderation.ErrModerationUnavailable, http.StatusServiceUnavailable, "ModerationUnavailable"}, + {"unknown", errors.New("unexpected storage failure"), http.StatusInternalServerError, "InternalServerError"}, +} + +func TestRemoveContentHandlerMapsWrappedServiceErrors(t *testing.T) { + for _, test := range mutationErrors { + t.Run(test.name, func(t *testing.T) { + fake := &mutationServiceFake{err: fmt.Errorf("mutation failed: %w", test.err)} + response := httptest.NewRecorder() + NewRemoveContentHandler(fake).HandleRemoveContent(response, + mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", removeBody, "application/json")) + assertMutationError(t, response, test.status, test.code) + assert.Len(t, fake.removeCalls, 1) + }) + } +} + +func mutationErrorMessage(t *testing.T, response *httptest.ResponseRecorder) string { + t.Helper() + var body map[string]any + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + message, ok := body["message"].(string) + require.Truef(t, ok, "message must be a string: %s", response.Body.String()) + return message +} + +func TestRemoveContentHandlerNamesPostRemovalAsUnsupported(t *testing.T) { + service := moderation.NewService(nil, nil, moderation.Config{ + InstanceDID: mutationInstanceDID, IdempotencyRetention: time.Hour, MaxLiveIdempotencyKeys: 1, + }) + postBody := strings.Replace(removeBody, moderation.CommentCollection, moderation.PostV2Collection, 1) + response := httptest.NewRecorder() + NewRemoveContentHandler(service).HandleRemoveContent(response, + mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", postBody, "application/json")) + assertMutationError(t, response, http.StatusBadRequest, "InvalidSubject") + assert.Equal(t, "invalid subject: post removal is unsupported", mutationErrorMessage(t, response)) +} + +func TestMutationHandlersWriteRuleDetailButKeepUnavailableGeneric(t *testing.T) { + for _, test := range []struct { + name string + err error + status int + code string + message string + }{ + {"live key cap", fmt.Errorf("%w: live idempotency key limit %d reached", moderation.ErrInvalidRequest, 3), + http.StatusBadRequest, "InvalidRequest", "invalid request: live idempotency key limit 3 reached"}, + {"bare rule sentinel", moderation.ErrStateConflict, http.StatusBadRequest, "StateConflict", "state conflict"}, + {"unavailable with storage detail", fmt.Errorf("%w: %w", moderation.ErrModerationUnavailable, errors.New("dial tcp 10.0.0.9:5432: connection refused")), + http.StatusServiceUnavailable, "ModerationUnavailable", "Moderation service temporarily unavailable"}, + {"unknown with detail", errors.New("dial tcp 10.0.0.9:5432: connection refused"), + http.StatusInternalServerError, "InternalServerError", "An internal error occurred"}, + } { + t.Run(test.name, func(t *testing.T) { + removeResponse := httptest.NewRecorder() + NewRemoveContentHandler(&mutationServiceFake{err: test.err}).HandleRemoveContent(removeResponse, + mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", removeBody, "application/json")) + assertMutationError(t, removeResponse, test.status, test.code) + assert.Equal(t, test.message, mutationErrorMessage(t, removeResponse)) + restoreResponse := httptest.NewRecorder() + NewRestoreContentHandler(&mutationServiceFake{err: test.err}).HandleRestoreContent(restoreResponse, + mutationRequest(http.MethodPost, restorePath, restoreBody, "application/json")) + assertMutationError(t, restoreResponse, test.status, test.code) + assert.Equal(t, test.message, mutationErrorMessage(t, restoreResponse)) + }) + } +} diff --git a/internal/api/handlers/moderation/restore_content.go b/internal/api/handlers/moderation/restore_content.go new file mode 100644 index 0000000..7f4e02a --- /dev/null +++ b/internal/api/handlers/moderation/restore_content.go @@ -0,0 +1,51 @@ +package moderation + +import ( + "net/http" + + "Coves/internal/api/middleware" + "Coves/internal/api/xrpc" + "Coves/internal/core/moderation" +) + +// RestoreContentHandler serves social.coves.moderation.restoreContent. +type RestoreContentHandler struct { + service moderation.Service +} + +// NewRestoreContentHandler builds the restoreContent handler. +func NewRestoreContentHandler(service moderation.Service) *RestoreContentHandler { + return &RestoreContentHandler{service: service} +} + +// HandleRestoreContent handles POST /xrpc/social.coves.moderation.restoreContent. +func (h *RestoreContentHandler) HandleRestoreContent(w http.ResponseWriter, r *http.Request) { + if !requireMutationPOSTJSON(w, r) { + return + } + var input struct { + ActionID string `json:"actionId"` + ReviewedSubject *strongRefView `json:"reviewedSubject"` + ExpectedVersion string `json:"expectedVersion"` + IdempotencyKey string `json:"idempotencyKey"` + Reason string `json:"reason"` + PrivateNote string `json:"privateNote"` + } + if !decodeMutationRequest(w, r, &input) { + return + } + actorDID := middleware.GetUserDID(r) + if actorDID == "" { + xrpc.WriteError(w, http.StatusUnauthorized, "AuthRequired", "Authentication required") + return + } + request := moderation.RestoreContentRequest{ + ActionID: input.ActionID, ExpectedVersion: input.ExpectedVersion, + IdempotencyKey: input.IdempotencyKey, Reason: input.Reason, PrivateNote: input.PrivateNote, + } + if input.ReviewedSubject != nil { + request.ReviewedSubject = &moderation.StrongRef{URI: input.ReviewedSubject.URI, CID: input.ReviewedSubject.CID} + } + result, err := h.service.RestoreContent(r.Context(), actorDID, request) + writeMutationResult(w, "restoreContent", result, err) +} diff --git a/internal/api/handlers/moderation/restore_content_test.go b/internal/api/handlers/moderation/restore_content_test.go new file mode 100644 index 0000000..3ed5565 --- /dev/null +++ b/internal/api/handlers/moderation/restore_content_test.go @@ -0,0 +1,113 @@ +package moderation + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + "Coves/internal/core/moderation" + "Coves/internal/validation" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + restorePath = "/xrpc/social.coves.moderation.restoreContent" + restoreBody = `{"actionId":"action-previous","reviewedSubject":{"uri":"` + mutationSubjectURI + `","cid":"` + mutationSubjectCID + `"},"expectedVersion":"v0","idempotencyKey":"restore-key","reason":"` + mutationReason + `","privateNote":"` + mutationNote + `"}` +) + +func TestRestoreContentHandlerPassesContextActorAndExactlyMapsRequest(t *testing.T) { + for _, test := range []struct { + name string + body string + reviewed *moderation.StrongRef + note string + }{ + {"reviewed subject and private note", restoreBody, &moderation.StrongRef{URI: mutationSubjectURI, CID: mutationSubjectCID}, mutationNote}, + {"reviewed subject and private note omitted", `{"actionId":"action-previous","expectedVersion":"v0","idempotencyKey":"restore-key","reason":"` + mutationReason + `"}`, nil, ""}, + } { + t.Run(test.name, func(t *testing.T) { + result := mutationResult(moderation.ActionRestore, test.note) + fake := &mutationServiceFake{result: result} + response := httptest.NewRecorder() + NewRestoreContentHandler(fake).HandleRestoreContent(response, + mutationRequest(http.MethodPost, restorePath, test.body, "application/json")) + assert.Equal(t, []string{mutationActorDID}, fake.actors) + assert.Equal(t, []moderation.RestoreContentRequest{{ + ActionID: "action-previous", ReviewedSubject: test.reviewed, ExpectedVersion: "v0", + IdempotencyKey: "restore-key", Reason: mutationReason, PrivateNote: test.note, + }}, fake.restoreCalls) + assertMutationResponse(t, response, result) + }) + } +} + +func TestRestoreContentHandlerRejectsNonJSONInvalidJSONAndGET(t *testing.T) { + for _, test := range []struct { + name, method, body, contentType string + status int + }{ + {"text/plain", http.MethodPost, restoreBody, "text/plain", http.StatusBadRequest}, + {"form", http.MethodPost, restoreBody, "application/x-www-form-urlencoded", http.StatusBadRequest}, + {"missing content type", http.MethodPost, restoreBody, "", http.StatusBadRequest}, + {"malformed JSON", http.MethodPost, `{"actionId":`, "application/json", http.StatusBadRequest}, + {"GET", http.MethodGet, "", "", http.StatusMethodNotAllowed}, + } { + t.Run(test.name, func(t *testing.T) { + fake := &mutationServiceFake{result: mutationResult(moderation.ActionRestore, "")} + response := httptest.NewRecorder() + NewRestoreContentHandler(fake).HandleRestoreContent(response, + mutationRequest(test.method, restorePath, test.body, test.contentType)) + assert.Equal(t, test.status, response.Code) + if test.status == http.StatusBadRequest { + assertMutationError(t, response, test.status, "InvalidRequest") + } + assert.Empty(t, fake.restoreCalls) + }) + } +} + +func TestRestoreContentHandlerMapsWrappedServiceErrors(t *testing.T) { + for _, test := range mutationErrors { + t.Run(test.name, func(t *testing.T) { + fake := &mutationServiceFake{err: fmt.Errorf("mutation failed: %w", test.err)} + response := httptest.NewRecorder() + NewRestoreContentHandler(fake).HandleRestoreContent(response, + mutationRequest(http.MethodPost, restorePath, restoreBody, "application/json")) + assertMutationError(t, response, test.status, test.code) + assert.Len(t, fake.restoreCalls, 1) + }) + } +} + +func TestRestoreContentHandlerOmitsCIDForUnavailableSubject(t *testing.T) { + result := mutationResult(moderation.ActionRestore, "") + result.State.RecordState = moderation.RecordStateUnavailable + result.State.CurrentSubject = nil + result.Action.ObservedCID = "" + response := httptest.NewRecorder() + NewRestoreContentHandler(&mutationServiceFake{result: result}).HandleRestoreContent(response, + mutationRequest(http.MethodPost, restorePath, restoreBody, "application/json")) + require.Equal(t, http.StatusOK, response.Code) + var body map[string]any + require.NoError(t, json.Unmarshal(response.Body.Bytes(), &body)) + adminAction, ok := body["action"].(map[string]any) + require.True(t, ok) + action, ok := adminAction["action"].(map[string]any) + require.True(t, ok) + assert.Equal(t, map[string]any{"uri": mutationSubjectURI}, action["subject"], "an unobserved subject has no CID to report") + state, ok := body["state"].(map[string]any) + require.True(t, ok) + assert.Equal(t, "unavailable", state["recordState"]) + assert.NotContains(t, state, "currentSubject") + catalog := lexicon.NewBaseCatalog() + require.NoError(t, catalog.LoadDirectory("../../../atproto/lexicon")) + decoded, err := atdata.UnmarshalJSON(response.Body.Bytes()) + require.NoError(t, err) + assert.NoError(t, validation.ValidateData(catalog, decoded, "social.coves.moderation.defs#mutationResult", 0)) +} diff --git a/internal/api/routes/moderation.go b/internal/api/routes/moderation.go index cf87ad8..62bbe56 100644 --- a/internal/api/routes/moderation.go +++ b/internal/api/routes/moderation.go @@ -13,6 +13,12 @@ import ( // scopes in cmd/server remain unchanged (see oauth_scopes_test.go). func RegisterModerationRoutes(r chi.Router, service moderation.Service, adminAuth *middleware.InstanceAdminMiddleware) { getSubjectState := handler.NewGetSubjectStateHandler(service) + removeContent := handler.NewRemoveContentHandler(service) + restoreContent := handler.NewRestoreContentHandler(service) r.With(adminAuth.RequireInstanceAdmin).Get( "/xrpc/social.coves.moderation.getSubjectState", getSubjectState.HandleGetSubjectState) + r.With(adminAuth.RequireInstanceAdmin).Post( + "/xrpc/social.coves.moderation.removeContent", removeContent.HandleRemoveContent) + r.With(adminAuth.RequireInstanceAdmin).Post( + "/xrpc/social.coves.moderation.restoreContent", restoreContent.HandleRestoreContent) } diff --git a/internal/api/routes/moderation_comment_removal_integration_test.go b/internal/api/routes/moderation_comment_removal_integration_test.go new file mode 100644 index 0000000..729df57 --- /dev/null +++ b/internal/api/routes/moderation_comment_removal_integration_test.go @@ -0,0 +1,268 @@ +//go:build integration + +package routes_test + +import ( + "bytes" + "database/sql" + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + actorAPI "Coves/internal/api/handlers/actor" + commentsAPI "Coves/internal/api/handlers/comments" + "Coves/internal/api/middleware" + "Coves/internal/api/routes" + "Coves/internal/core/comments" + "Coves/internal/core/moderation" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/internal/validation" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func moderationAcceptanceRequest(t *testing.T, client *http.Client, method, target, token string, body any) subjectStateHTTPResponse { + t.Helper() + var input io.Reader + if body != nil { + encoded, err := json.Marshal(body) + require.NoError(t, err) + input = bytes.NewReader(encoded) + } + request, err := http.NewRequestWithContext(t.Context(), method, target, input) + require.NoError(t, err) + if body != nil { + request.Header.Set("Content-Type", "application/json") + } + if token != "" { + request.Header.Set("Authorization", "Bearer "+token) + } + response, err := client.Do(request) + require.NoError(t, err) + defer response.Body.Close() + data, err := io.ReadAll(response.Body) + require.NoError(t, err) + return subjectStateHTTPResponse{status: response.StatusCode, body: data} +} + +func moderationAcceptanceObject(t *testing.T, value any) map[string]any { + t.Helper() + object, ok := value.(map[string]any) + require.Truef(t, ok, "expected an object, got %#v", value) + return object +} + +func moderationAcceptanceArray(t *testing.T, value any) []any { + t.Helper() + array, ok := value.([]any) + require.Truef(t, ok, "expected an array, got %#v", value) + return array +} + +func moderationAcceptanceBody(t *testing.T, response subjectStateHTTPResponse) map[string]any { + t.Helper() + require.Equalf(t, http.StatusOK, response.status, "response body: %s", response.body) + var body map[string]any + require.NoError(t, json.Unmarshal(response.body, &body)) + return body +} + +func moderationAcceptanceThreadComment(t *testing.T, body map[string]any) (map[string]any, []any) { + t.Helper() + comments := moderationAcceptanceArray(t, body["comments"]) + require.Len(t, comments, 1) + node := moderationAcceptanceObject(t, comments[0]) + return moderationAcceptanceObject(t, node["comment"]), moderationAcceptanceArray(t, node["replies"]) +} + +func moderationAcceptanceActorURIs(t *testing.T, body map[string]any) []string { + t.Helper() + var uris []string + for _, entry := range moderationAcceptanceArray(t, body["comments"]) { + uri, ok := moderationAcceptanceObject(t, entry)["uri"].(string) + require.True(t, ok) + uris = append(uris, uri) + } + return uris +} + +func moderationAcceptanceInsertComment(t *testing.T, db *sql.DB, authorDID, rootURI, rootCID, parentURI, parentCID, cid, content string) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.CommentCollection + "/" + rkey + _, err := db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW()) + `, uri, cid, rkey, authorDID, rootURI, rootCID, parentURI, parentCID, content) + require.NoError(t, err) + return uri +} + +func TestModerationCommentRemovalAcceptance(t *testing.T) { + db := testkit.DB(t) + postRepo := postgres.NewPostRepository(db) + commentRepo := postgres.NewCommentRepository(db) + userRepo := postgres.NewUserRepository(db) + communityRepo := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()) + instanceDID := fixtures.InstanceDID() + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepo, commentRepo), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: instanceDID, IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + commentService := comments.NewCommentService(commentRepo, userRepo, postRepo, communityRepo, nil, nil, nil) + + adminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "modadmin")) + nonAdminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "modother")) + authorName := testkit.UniqueIDWithPrefix(t, "modauthor") + authorDID := fixtures.DID(authorName) + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "modcommunity") + communityDID, err := fixtures.Community(t.Context(), db, communityName, "owner"+communityName) + require.NoError(t, err) + postURI := fixtures.Post(t, db, communityDID, authorDID, "moderation acceptance post", 0, time.Now()) + post, err := postRepo.GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + const commentContent = "unique original moderation acceptance comment text" + const replyContent = "reply remains visible under moderated comment" + const commentCID = "bafyreicommentacceptance" + const replyCID = "bafyreireplyacceptance" + commentURI := moderationAcceptanceInsertComment(t, db, authorDID, postURI, post.CID, postURI, post.CID, commentCID, commentContent) + replyURI := moderationAcceptanceInsertComment(t, db, authorDID, postURI, post.CID, commentURI, commentCID, replyCID, replyContent) + _, err = db.ExecContext(t.Context(), `UPDATE comments SET reply_count = 1 WHERE uri = $1`, commentURI) + require.NoError(t, err) + + const adminToken = "moderation-acceptance-admin-session" + const nonAdminToken = "moderation-acceptance-nonadmin-session" + unsealer := fixtures.NewSessionUnsealer() + oauthStore := fixtures.NewOAuthStore() + unsealer.AddSession(adminToken, adminDID, "moderation-admin-session") + oauthStore.AddSession(adminDID, "moderation-admin-session", "admin-access-token") + unsealer.AddSession(nonAdminToken, nonAdminDID, "moderation-nonadmin-session") + oauthStore.AddSession(nonAdminDID, "moderation-nonadmin-session", "non-admin-access-token") + adminAuth := middleware.NewInstanceAdminMiddleware(unsealer, oauthStore, nil, moderation.NewAllowlistAuthority([]string{adminDID})) + optionalAuth := middleware.NewOAuthAuthMiddleware(unsealer, oauthStore) + mux := chi.NewRouter() + routes.RegisterModerationRoutes(mux, moderationService, adminAuth) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.community.comment.getComments", + commentsAPI.NewGetCommentsHandler(commentsAPI.NewServiceAdapter(commentService), nil).HandleGetComments) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.actor.getComments", + actorAPI.NewGetCommentsHandler(commentService, nil, nil).HandleGetComments) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + client := server.Client() + threadURL := server.URL + "/xrpc/social.coves.community.comment.getComments?" + url.Values{"post": {postURI}, "sort": {"new"}}.Encode() + actorURL := server.URL + "/xrpc/social.coves.actor.getComments?" + url.Values{"actor": {authorDID}}.Encode() + removeURL := server.URL + "/xrpc/social.coves.moderation.removeContent" + + // Verify the fixture reaches both real read paths before exercising the mutation. + initialThread := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil)) + initialComment, initialReplies := moderationAcceptanceThreadComment(t, initialThread) + require.Equal(t, commentURI, initialComment["uri"]) + require.Equal(t, commentContent, moderationAcceptanceObject(t, initialComment["record"])["content"]) + require.Len(t, initialReplies, 1) + require.Equal(t, replyURI, moderationAcceptanceObject(t, moderationAcceptanceObject(t, initialReplies[0])["comment"])["uri"]) + initialActivity := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, actorURL, "", nil)) + require.ElementsMatch(t, []string{commentURI, replyURI}, moderationAcceptanceActorURIs(t, initialActivity)) + + stateResponse := requestSubjectState(t, client, server.URL, commentURI, adminToken) + initialState := moderationAcceptanceObject(t, moderationAcceptanceBody(t, stateResponse)["state"]) + initialVersion, ok := initialState["version"].(string) + require.True(t, ok) + require.Equal(t, "v0", initialVersion) + require.Equal(t, commentCID, moderationAcceptanceObject(t, initialState["currentSubject"])["cid"]) + require.Equal(t, "clear", moderationAcceptanceObject(t, initialState["moderation"])["state"]) + + removeRequest := map[string]any{ + "subject": map[string]string{"uri": commentURI, "cid": commentCID}, + "expectedVersion": initialVersion, "idempotencyKey": "k-remove", + "reason": "social.coves.moderation.defs#reasonSpam", + } + unauthorized := moderationAcceptanceRequest(t, client, http.MethodPost, removeURL, nonAdminToken, removeRequest) + requireXRPCError(t, unauthorized, http.StatusForbidden, "Forbidden") + afterUnauthorized := moderationAcceptanceObject(t, moderationAcceptanceBody(t, requestSubjectState(t, client, server.URL, commentURI, adminToken))["state"]) + require.Equal(t, initialVersion, afterUnauthorized["version"]) + require.Equal(t, "clear", moderationAcceptanceObject(t, afterUnauthorized["moderation"])["state"]) + + removedResponse := moderationAcceptanceRequest(t, client, http.MethodPost, removeURL, adminToken, removeRequest) + removed := moderationAcceptanceBody(t, removedResponse) + catalog := lexicon.NewBaseCatalog() + require.NoError(t, catalog.LoadDirectory("../../atproto/lexicon")) + data, err := atdata.UnmarshalJSON(removedResponse.body) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, data, "social.coves.moderation.defs#mutationResult", 0)) + require.Equal(t, "applied", removed["outcome"]) + removalAction := moderationAcceptanceObject(t, moderationAcceptanceObject(t, removed["action"])["action"]) + require.Equal(t, "remove", removalAction["action"]) + removalID, ok := moderationAcceptanceObject(t, removalAction["ref"])["actionId"].(string) + require.True(t, ok) + require.NotEmpty(t, removalID) + removedState := moderationAcceptanceObject(t, removed["state"]) + removedVersion, ok := removedState["version"].(string) + require.True(t, ok) + require.NotEqual(t, initialVersion, removedVersion) + require.Equal(t, "removed", moderationAcceptanceObject(t, removedState["moderation"])["state"]) + require.Equal(t, removalID, moderationAcceptanceObject(t, removedState["localRemoval"])["actionId"]) + + removedThread := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil)) + placeholder, replies := moderationAcceptanceThreadComment(t, removedThread) + require.Equal(t, commentURI, placeholder["uri"]) + require.Equal(t, true, placeholder["isDeleted"]) + require.Equal(t, "moderator", placeholder["deletionReason"]) + view := moderationAcceptanceObject(t, placeholder["moderation"]) + require.Equal(t, "removed", view["state"]) + sources := moderationAcceptanceArray(t, view["sources"]) + require.NotEmpty(t, sources) + require.Equal(t, instanceDID, moderationAcceptanceObject(t, sources[0])["authorityDid"]) + require.Equal(t, "instance", moderationAcceptanceObject(t, moderationAcceptanceObject(t, sources[0])["scope"])["kind"]) + require.Nil(t, placeholder["record"]) + require.Equal(t, "handle.invalid", moderationAcceptanceObject(t, placeholder["author"])["handle"]) + require.NotContains(t, placeholder, "embed") + placeholderJSON, err := json.Marshal(placeholder) + require.NoError(t, err) + require.NotContains(t, string(placeholderJSON), commentContent) + require.Len(t, replies, 1) + reply := moderationAcceptanceObject(t, moderationAcceptanceObject(t, replies[0])["comment"]) + require.Equal(t, replyURI, reply["uri"]) + require.Equal(t, replyContent, moderationAcceptanceObject(t, reply["record"])["content"]) + + activity := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, actorURL, "", nil)) + activityURIs := moderationAcceptanceActorURIs(t, activity) + assert.NotContains(t, activityURIs, commentURI) + require.Contains(t, activityURIs, replyURI) + + restoreResponse := moderationAcceptanceRequest(t, client, http.MethodPost, server.URL+"/xrpc/social.coves.moderation.restoreContent", adminToken, map[string]any{ + "actionId": removalID, + "reviewedSubject": map[string]string{"uri": commentURI, "cid": commentCID}, + "expectedVersion": removedVersion, "idempotencyKey": "k-restore", + "reason": "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + restored := moderationAcceptanceBody(t, restoreResponse) + data, err = atdata.UnmarshalJSON(restoreResponse.body) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, data, "social.coves.moderation.defs#mutationResult", 0)) + require.Equal(t, "applied", restored["outcome"]) + restoreAction := moderationAcceptanceObject(t, moderationAcceptanceObject(t, restored["action"])["action"]) + require.Equal(t, "restore", restoreAction["action"]) + require.Equal(t, removalID, moderationAcceptanceObject(t, restoreAction["reverses"])["actionId"]) + require.Equal(t, "clear", moderationAcceptanceObject(t, moderationAcceptanceObject(t, restored["state"])["moderation"])["state"]) + + restoredThread := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil)) + restoredComment, restoredReplies := moderationAcceptanceThreadComment(t, restoredThread) + require.Equal(t, commentURI, restoredComment["uri"]) + require.NotContains(t, restoredComment, "isDeleted") + require.Equal(t, commentContent, moderationAcceptanceObject(t, restoredComment["record"])["content"]) + require.Len(t, restoredReplies, 1) + require.Equal(t, replyURI, moderationAcceptanceObject(t, moderationAcceptanceObject(t, restoredReplies[0])["comment"])["uri"]) +} diff --git a/internal/api/routes/moderation_media_integration_test.go b/internal/api/routes/moderation_media_integration_test.go new file mode 100644 index 0000000..c821c71 --- /dev/null +++ b/internal/api/routes/moderation_media_integration_test.go @@ -0,0 +1,487 @@ +//go:build integration + +package routes_test + +import ( + "context" + "database/sql" + "encoding/json" + "fmt" + "io" + "io/fs" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "sync" + "testing" + "time" + + imagehandler "Coves/internal/api/handlers/imageproxy" + "Coves/internal/api/routes" + "Coves/internal/atproto/identity" + "Coves/internal/core/imageproxy" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/go-chi/chi/v5" + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multibase" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// mediaImageCID returns a real, canonical CIDv1 for label. The proxy decodes +// every CID, so a fixture must be a decodable CID rather than a lookalike. +func mediaImageCID(label string) string { + digest, err := multihash.Sum([]byte("moderation media "+label), multihash.SHA2_256, -1) + if err != nil { + panic(err) + } + return cid.NewCidV1(cid.Raw, digest).String() +} + +// base58MediaCID re-encodes a canonical CID in base58btc: the same CID in a +// different multibase string. +func base58MediaCID(t *testing.T, canonical string) string { + t.Helper() + parsed, err := cid.Decode(canonical) + require.NoError(t, err) + encoded, err := parsed.StringOfBase(multibase.Base58BTC) + require.NoError(t, err) + require.NotEqual(t, canonical, encoded) + return encoded +} + +type mediaBlobKey struct{ did, cid string } + +type mediaPDS struct { + mu sync.Mutex + counts map[mediaBlobKey]int + known map[mediaBlobKey]bool +} + +func (p *mediaPDS) serve(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet || r.URL.Path != "/xrpc/com.atproto.sync.getBlob" { + http.NotFound(w, r) + return + } + // Like the reference PDS, look the blob up by its parsed CID, so every + // multibase encoding of one CID names the same blob. + parsed, err := cid.Decode(r.URL.Query().Get("cid")) + if err != nil { + http.Error(w, "invalid cid", http.StatusBadRequest) + return + } + key := mediaBlobKey{r.URL.Query().Get("did"), parsed.String()} + p.mu.Lock() + known := p.known[key] + if known { + p.counts[key]++ + } + p.mu.Unlock() + if !known { + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "image/png") + _, _ = w.Write(testkit.TestPNG(32, 32)) +} + +func (p *mediaPDS) count(did, cid string) int { + p.mu.Lock() + defer p.mu.Unlock() + return p.counts[mediaBlobKey{did, cid}] +} + +type mediaPDSResolver struct{ url string } + +func (r mediaPDSResolver) ResolveDID(_ context.Context, did string) (*identity.DIDDocument, error) { + return &identity.DIDDocument{DID: did, Service: []identity.Service{{ + Type: "AtprotoPersonalDataServer", ServiceEndpoint: r.url, + }}}, nil +} +func (mediaPDSResolver) Resolve(context.Context, string) (*identity.Identity, error) { + return nil, fmt.Errorf("image proxy must resolve the blob owner DID") +} +func (mediaPDSResolver) ResolveHandle(context.Context, string) (string, string, error) { + return "", "", fmt.Errorf("image proxy must resolve the blob owner DID") +} +func (mediaPDSResolver) Purge(context.Context, string) error { return nil } + +type waitingMediaFetcher struct { + upstream imageproxy.Fetcher + entered chan struct{} + release chan struct{} + once sync.Once +} + +func (f *waitingMediaFetcher) Fetch(ctx context.Context, pdsURL, did, cid string) ([]byte, error) { + f.once.Do(func() { close(f.entered) }) + select { + case <-f.release: + return f.upstream.Fetch(ctx, pdsURL, did, cid) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +type moderationMediaHarness struct { + db *sql.DB + cache *imageproxy.DiskCache + cacheDir string + proxy *httptest.Server + pds *mediaPDS + moderation moderation.Service + postURI string + postCID string + ownerA string + ownerB string +} + +func newModerationMediaHarness(t *testing.T, blockFetch bool) (*moderationMediaHarness, *waitingMediaFetcher) { + t.Helper() + db := testkit.DB(t) + ownerName := testkit.UniqueIDWithPrefix(t, "mediaowner") + otherName := testkit.UniqueIDWithPrefix(t, "otherowner") + ownerA, ownerB := fixtures.DID(ownerName), fixtures.DID(otherName) + fixtures.User(t, db, ownerName+".test", ownerA) + fixtures.User(t, db, otherName+".test", ownerB) + communityName := testkit.UniqueIDWithPrefix(t, "mediapost") + communityDID, err := fixtures.Community(t.Context(), db, communityName, "owner"+communityName) + require.NoError(t, err) + postURI := fixtures.Post(t, db, communityDID, ownerA, "moderation media", 0, time.Now()) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + + cacheDir := t.TempDir() + cache, err := imageproxy.NewDiskCache(cacheDir, 1, 0) + require.NoError(t, err) + processor, err := imageproxy.NewProcessor(imageproxy.DefaultMaxSourceMegapixels) + require.NoError(t, err) + pds := &mediaPDS{counts: make(map[mediaBlobKey]int), known: make(map[mediaBlobKey]bool)} + pdsServer := httptest.NewServer(http.HandlerFunc(pds.serve)) + t.Cleanup(pdsServer.Close) + var fetcher imageproxy.Fetcher = imageproxy.NewPDSFetcher(30*time.Second, 10, imageproxy.WithPrivateHostsAllowed()) + var waiting *waitingMediaFetcher + if blockFetch { + waiting = &waitingMediaFetcher{upstream: fetcher, entered: make(chan struct{}), release: make(chan struct{})} + fetcher = waiting + t.Cleanup(func() { + select { + case <-waiting.release: + default: + close(waiting.release) + } + }) + } + store := postgres.NewModerationRepository(db) + proxyService, err := imageproxy.NewService(cache, processor, fetcher, store, imageproxy.DefaultConfig()) + require.NoError(t, err) + service := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + store, moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000, Purger: proxyService}, + ) + router := chi.NewRouter() + routes.RegisterImageProxyRoutes(router, imagehandler.NewHandler(proxyService, mediaPDSResolver{url: pdsServer.URL})) + proxy := httptest.NewServer(router) + t.Cleanup(proxy.Close) + return &moderationMediaHarness{db: db, cache: cache, cacheDir: cacheDir, proxy: proxy, pds: pds, + moderation: service, postURI: postURI, postCID: post.CID, ownerA: ownerA, ownerB: ownerB}, waiting +} + +func (h *moderationMediaHarness) comment(t *testing.T, owner string, imageCIDs ...string) moderation.StrongRef { + t.Helper() + images := make([]any, 0, len(imageCIDs)) + for _, cid := range imageCIDs { + images = append(images, map[string]any{"alt": "test image", "image": map[string]any{ + "$type": "blob", "ref": map[string]any{"$link": cid}, "mimeType": "image/png", "size": 10, + }}) + } + return h.commentWithEmbed(t, owner, map[string]any{"$type": "social.coves.embed.images", "images": images}, imageCIDs...) +} + +// commentWithEmbed indexes a comment carrying embed verbatim, and registers +// servedCIDs as blobs the owner's PDS serves. +func (h *moderationMediaHarness) commentWithEmbed(t *testing.T, owner string, embedValue any, servedCIDs ...string) moderation.StrongRef { + t.Helper() + rkey := testkit.TID() + subject := moderation.StrongRef{URI: "at://" + owner + "/" + moderation.CommentCollection + "/" + rkey, CID: mediaImageCID("comment record")} + for _, cid := range servedCIDs { + h.pds.known[mediaBlobKey{owner, cid}] = true + } + embed, err := json.Marshal(embedValue) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $5, $6, 'moderated comment', $7::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, owner, h.postURI, h.postCID, string(embed)) + require.NoError(t, err) + return subject +} + +func (h *moderationMediaHarness) request(t *testing.T, preset, owner, cid string) int { + t.Helper() + url := h.proxy.URL + "/img/" + preset + "/plain/" + owner + "/" + cid + request, err := http.NewRequestWithContext(t.Context(), http.MethodGet, url, nil) + require.NoError(t, err) + response, err := h.proxy.Client().Do(request) + require.NoError(t, err) + defer response.Body.Close() + body, err := io.ReadAll(response.Body) + require.NoError(t, err) + if response.StatusCode != http.StatusOK && response.StatusCode != http.StatusNotFound { + t.Fatalf("GET %s: %d %s", url, response.StatusCode, body) + } + return response.StatusCode +} + +func (h *moderationMediaHarness) remove(t *testing.T, subject moderation.StrongRef, reason string) *moderation.MutationResult { + t.Helper() + result, err := h.moderation.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-" + testkit.UniqueID(t), Reason: reason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + return result +} + +func (h *moderationMediaHarness) restore(t *testing.T, subject moderation.StrongRef, removed *moderation.MutationResult) { + t.Helper() + result, err := h.moderation.RestoreContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: removed.State.Version, + IdempotencyKey: "restore-" + testkit.UniqueID(t), Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) +} + +func (h *moderationMediaHarness) cachePath(preset, owner, cid string) string { + return filepath.Join(h.cacheDir, preset, strings.ReplaceAll(owner, ":", "_"), cid) +} + +func (h *moderationMediaHarness) assertNoCachedBlob(t *testing.T, cid string, owner string) { + t.Helper() + require.NoError(t, filepath.WalkDir(h.cacheDir, func(path string, entry fs.DirEntry, err error) error { + if err != nil { + return err + } + if !entry.IsDir() && entry.Name() == cid && (owner == "" || filepath.Base(filepath.Dir(path)) == strings.ReplaceAll(owner, ":", "_")) { + assert.Failf(t, "blocked image remains on disk", "path: %s", path) + } + return nil + })) +} + +func TestModerationMediaServing(t *testing.T) { + const spam = "social.coves.moderation.defs#reasonSpam" + const illegal = "social.coves.moderation.defs#reasonIllegalContent" + const preset = "content_preview" + var cid1, cid2, cid5, cid6, cid7 = mediaImageCID("a"), mediaImageCID("b"), mediaImageCID("e"), mediaImageCID("f"), mediaImageCID("g") + + t.Run("spam purges warm images across presets and blocks cold fetches", func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + subject := h.comment(t, h.ownerA, cid1, cid2) + for _, name := range []string{preset, "content_full", "avatar_small"} { + for _, cid := range []string{cid1, cid2} { + require.Equal(t, http.StatusOK, h.request(t, name, h.ownerA, cid)) + _, err := os.Stat(h.cachePath(name, h.ownerA, cid)) + require.NoError(t, err, "image must really be warm on disk before removal") + } + } + before1, before2 := h.pds.count(h.ownerA, cid1), h.pds.count(h.ownerA, cid2) + assert.Equal(t, 3, before1) + assert.Equal(t, 3, before2) + h.remove(t, subject, spam) + for _, cid := range []string{cid1, cid2} { + h.assertNoCachedBlob(t, cid, h.ownerA) + for i := 0; i < 2; i++ { + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, cid)) + } + } + assert.Equal(t, before1, h.pds.count(h.ownerA, cid1), "blocked image must not be refetched") + assert.Equal(t, before2, h.pds.count(h.ownerA, cid2), "blocked image must not be refetched") + }) + + t.Run("same CID under an unremoved owner remains fetchable", func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + subject := h.comment(t, h.ownerA, cid1) + h.comment(t, h.ownerB, cid1) + h.remove(t, subject, spam) + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, cid1)) + require.Equal(t, http.StatusOK, h.request(t, preset, h.ownerB, cid1)) + assert.Equal(t, 1, h.pds.count(h.ownerB, cid1), "another owner's blob must still be fetched") + }) + + t.Run("illegal content purges every owner's cached copy", func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + subject := h.comment(t, h.ownerA, cid5) + h.comment(t, h.ownerB, cid5) + for _, name := range []string{preset, "content_full"} { + for _, owner := range []string{h.ownerA, h.ownerB} { + require.Equal(t, http.StatusOK, h.request(t, name, owner, cid5)) + _, err := os.Stat(h.cachePath(name, owner, cid5)) + require.NoError(t, err) + } + } + h.remove(t, subject, illegal) + h.assertNoCachedBlob(t, cid5, "") + for _, owner := range []string{h.ownerA, h.ownerB} { + require.Equal(t, http.StatusNotFound, h.request(t, preset, owner, cid5)) + assert.Equal(t, 2, h.pds.count(owner, cid5), "a blocked owner must not be fetched again") + } + }) + + t.Run("in flight fetch cannot publish after removal", func(t *testing.T) { + h, waiting := newModerationMediaHarness(t, true) + subject := h.comment(t, h.ownerA, cid6) + result := make(chan int, 1) + go func() { + request, err := http.NewRequestWithContext(context.Background(), http.MethodGet, + h.proxy.URL+"/img/"+preset+"/plain/"+h.ownerA+"/"+cid6, nil) + if err != nil { + result <- 0 + return + } + response, err := h.proxy.Client().Do(request) + if err != nil { + result <- 0 + return + } + defer response.Body.Close() + _, _ = io.Copy(io.Discard, response.Body) + result <- response.StatusCode + }() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + select { + case <-waiting.entered: + return true, nil + default: + return false, nil + } + }) + h.remove(t, subject, spam) + close(waiting.release) + var status int + testkit.WaitFor(t, 10*time.Second, func() (bool, error) { + select { + case status = <-result: + return true, nil + default: + return false, nil + } + }) + require.Equal(t, http.StatusNotFound, status, "an in-flight fetch must not publish blocked bytes") + testkit.Holds(t, 300*time.Millisecond, func() (bool, error) { + _, err := os.Stat(h.cachePath(preset, h.ownerA, cid6)) + return os.IsNotExist(err), nil + }) + assert.Equal(t, 1, h.pds.count(h.ownerA, cid6), "fetch must have reached the PDS before the post-fetch block check") + }) + + t.Run("shared blob stays blocked until last removal is restored then refetches", func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + first := h.comment(t, h.ownerA, cid7) + second := h.comment(t, h.ownerA, cid7) + require.Equal(t, http.StatusOK, h.request(t, preset, h.ownerA, cid7)) + before := h.pds.count(h.ownerA, cid7) + require.Equal(t, 1, before) + removedFirst := h.remove(t, first, spam) + removedSecond := h.remove(t, second, spam) + h.assertNoCachedBlob(t, cid7, h.ownerA) + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, cid7)) + h.restore(t, first, removedFirst) + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, cid7)) + assert.Equal(t, before, h.pds.count(h.ownerA, cid7)) + h.restore(t, second, removedSecond) + require.Equal(t, http.StatusOK, h.request(t, preset, h.ownerA, cid7)) + assert.Equal(t, before+1, h.pds.count(h.ownerA, cid7), "last restore must permit a fresh real PDS fetch") + }) +} + +func TestModerationMediaBlocksEveryServedEncoding(t *testing.T) { + const spam = "social.coves.moderation.defs#reasonSpam" + const preset = "content_preview" + + t.Run("base58btc encoding of a blocked CID is refused without a fetch", func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + blocked := mediaImageCID("re-encoded") + subject := h.comment(t, h.ownerA, blocked) + h.remove(t, subject, spam) + alias := base58MediaCID(t, blocked) + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, alias)) + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, blocked)) + assert.Zero(t, h.pds.count(h.ownerA, blocked), "a re-encoded blocked CID must not reach the PDS") + h.assertNoCachedBlob(t, alias, "") + h.assertNoCachedBlob(t, blocked, "") + }) + + t.Run("legacy cid-encoded image is blocked on removal", func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + legacy := mediaImageCID("legacy blob") + subject := h.commentWithEmbed(t, h.ownerA, map[string]any{ + "$type": "social.coves.embed.images", + "images": []any{map[string]any{"alt": "legacy image", "image": map[string]any{ + "cid": legacy, "mimeType": "image/png", + }}}, + }, legacy) + require.Equal(t, http.StatusOK, h.request(t, preset, h.ownerA, legacy), "the legacy blob must really be servable before removal") + before := h.pds.count(h.ownerA, legacy) + h.remove(t, subject, spam) + h.assertNoCachedBlob(t, legacy, h.ownerA) + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, legacy)) + assert.Equal(t, before, h.pds.count(h.ownerA, legacy), "a blocked legacy blob must not be refetched") + }) + + t.Run("malformed embeds never prevent removal", func(t *testing.T) { + valid := mediaImageCID("valid beside malformed") + for _, test := range []struct { + name string + embed any + blocked []string + }{ + {name: "images is a string", embed: map[string]any{"$type": "social.coves.embed.images", "images": "x"}}, + {name: "image is a string", embed: map[string]any{"$type": "social.coves.embed.images", "images": []any{map[string]any{"image": "x"}}}}, + {name: "ref is a string", embed: map[string]any{"$type": "social.coves.embed.images", "images": []any{map[string]any{"image": map[string]any{"ref": mediaImageCID("string ref")}}}}}, + {name: "type is not a string", embed: map[string]any{"$type": 5, "images": []any{map[string]any{"image": map[string]any{"ref": map[string]any{"$link": mediaImageCID("numeric type")}}}}}}, + {name: "embed is an array", embed: []any{"x"}}, + {name: "undecodable link", embed: map[string]any{"$type": "social.coves.embed.images", "images": []any{map[string]any{"image": map[string]any{"ref": map[string]any{"$link": "bafynotacid"}}}}}}, + { + name: "valid image beside malformed entries", + embed: map[string]any{"$type": "social.coves.embed.images", "images": []any{ + "x", map[string]any{"image": 7}, + map[string]any{"image": map[string]any{"ref": map[string]any{"$link": valid}}}, + }}, + blocked: []string{valid}, + }, + } { + t.Run(test.name, func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + subject := h.commentWithEmbed(t, h.ownerA, test.embed, test.blocked...) + removed := h.remove(t, subject, spam) + var blocks []string + rows, err := h.db.QueryContext(t.Context(), ` + SELECT blob_cid FROM moderation_media_blocks WHERE action_id = $1 AND active ORDER BY blob_cid + `, removed.Action.ID) + require.NoError(t, err) + defer rows.Close() + for rows.Next() { + var blobCID string + require.NoError(t, rows.Scan(&blobCID)) + blocks = append(blocks, blobCID) + } + require.NoError(t, rows.Err()) + assert.Equal(t, test.blocked, blocks) + for _, cid := range test.blocked { + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, cid)) + assert.Zero(t, h.pds.count(h.ownerA, cid)) + } + }) + } + }) +} diff --git a/internal/api/routes/moderation_reauthorization_integration_test.go b/internal/api/routes/moderation_reauthorization_integration_test.go new file mode 100644 index 0000000..383e740 --- /dev/null +++ b/internal/api/routes/moderation_reauthorization_integration_test.go @@ -0,0 +1,138 @@ +//go:build integration + +package routes_test + +import ( + "database/sql" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + "time" + + "Coves/internal/api/middleware" + "Coves/internal/api/routes" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + moderationRouteRemovePath = "/xrpc/social.coves.moderation.removeContent" + moderationRouteReason = "social.coves.moderation.defs#reasonSpam" +) + +// moderationRouteComments indexes count comments under one post and returns +// their URIs and CIDs. +func moderationRouteComments(t *testing.T, db *sql.DB, count int) ([]string, []string) { + t.Helper() + authorName := testkit.UniqueIDWithPrefix(t, "routeauthor") + authorDID := fixtures.DID(authorName) + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "routecommunity") + communityDID, err := fixtures.Community(t.Context(), db, communityName, "owner"+communityName) + require.NoError(t, err) + postURI := fixtures.Post(t, db, communityDID, authorDID, "moderation route post", 0, time.Now()) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + var uris, cids []string + for index := range count { + cid := "bafyreiroutecomment" + string(rune('a'+index)) + uris = append(uris, moderationAcceptanceInsertComment(t, db, authorDID, postURI, post.CID, postURI, post.CID, cid, "route comment")) + cids = append(cids, cid) + } + return uris, cids +} + +func moderationRouteServer(t *testing.T, service moderation.Service, token, adminDID string, allowlist []string) *httptest.Server { + t.Helper() + unsealer := fixtures.NewSessionUnsealer() + oauthStore := fixtures.NewOAuthStore() + unsealer.AddSession(token, adminDID, "moderation-route-session") + oauthStore.AddSession(adminDID, "moderation-route-session", "moderation-route-access-token") + adminAuth := middleware.NewInstanceAdminMiddleware(unsealer, oauthStore, nil, moderation.NewAllowlistAuthority(allowlist)) + router := chi.NewRouter() + routes.RegisterModerationRoutes(router, service, adminAuth) + server := httptest.NewServer(router) + t.Cleanup(server.Close) + return server +} + +func moderationRouteRemoveRequest(uri, cid, key string) map[string]any { + return map[string]any{ + "subject": map[string]string{"uri": uri, "cid": cid}, + "expectedVersion": "v0", "idempotencyKey": key, "reason": moderationRouteReason, + } +} + +func TestModerationRemoveReplayRechecksInstanceAdminAllowlist(t *testing.T) { + db := testkit.DB(t) + service := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + uris, cids := moderationRouteComments(t, db, 1) + adminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "formeradmin")) + const token = "moderation-replay-admin-session" + removeRequest := moderationRouteRemoveRequest(uris[0], cids[0], "k-remove") + + listed := moderationRouteServer(t, service, token, adminDID, []string{adminDID}) + removed := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, listed.Client(), http.MethodPost, listed.URL+moderationRouteRemovePath, token, removeRequest)) + require.Equal(t, "applied", removed["outcome"]) + removalID, ok := moderationAcceptanceObject(t, moderationAcceptanceObject(t, moderationAcceptanceObject(t, removed["action"])["action"])["ref"])["actionId"].(string) + require.True(t, ok) + require.NotEmpty(t, removalID) + removedVersion, ok := moderationAcceptanceObject(t, removed["state"])["version"].(string) + require.True(t, ok) + require.Equal(t, "v1", removedVersion) + + // The operator removes the DID from the allowlist. The same session replays + // the same key and body, which the service would answer from storage. + delisted := moderationRouteServer(t, service, token, adminDID, nil) + replay := moderationAcceptanceRequest(t, delisted.Client(), http.MethodPost, delisted.URL+moderationRouteRemovePath, token, removeRequest) + requireXRPCError(t, replay, http.StatusForbidden, "Forbidden") + var body map[string]any + require.NoError(t, json.Unmarshal(replay.body, &body)) + assert.ElementsMatch(t, []string{"error", "message"}, keysOf(body), "a revoked admin receives only the error envelope") + assert.NotContains(t, string(replay.body), removalID) + assert.NotContains(t, string(replay.body), `"`+removedVersion+`"`) + var removeActions int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_actions WHERE subject_uri = $1`, uris[0]).Scan(&removeActions)) + assert.Equal(t, 1, removeActions) +} + +func TestModerationRemoveLiveKeyCapNamesTheLimit(t *testing.T) { + db := testkit.DB(t) + service := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1}, + ) + uris, cids := moderationRouteComments(t, db, 2) + adminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "capadmin")) + const token = "moderation-cap-admin-session" + server := moderationRouteServer(t, service, token, adminDID, []string{adminDID}) + target := server.URL + moderationRouteRemovePath + + first := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, server.Client(), http.MethodPost, target, token, moderationRouteRemoveRequest(uris[0], cids[0], "cap-first"))) + require.Equal(t, "applied", first["outcome"]) + capped := moderationAcceptanceRequest(t, server.Client(), http.MethodPost, target, token, moderationRouteRemoveRequest(uris[1], cids[1], "cap-second")) + requireXRPCError(t, capped, http.StatusBadRequest, "InvalidRequest") + var body map[string]any + require.NoError(t, json.Unmarshal(capped.body, &body)) + assert.Equal(t, "invalid request: live idempotency key limit 1 reached", body["message"]) +} + +func keysOf(object map[string]any) []string { + keys := make([]string, 0, len(object)) + for key := range object { + keys = append(keys, key) + } + return keys +} diff --git a/internal/api/routes/moderation_subject_state_integration_test.go b/internal/api/routes/moderation_subject_state_integration_test.go index 4edc26c..d2df66f 100644 --- a/internal/api/routes/moderation_subject_state_integration_test.go +++ b/internal/api/routes/moderation_subject_state_integration_test.go @@ -66,7 +66,7 @@ func TestGetSubjectState(t *testing.T) { postRepo := postgres.NewPostRepository(db) commentRepo := postgres.NewCommentRepository(db) reader := moderation.NewRepositorySubjectReader(postRepo, commentRepo) - service := moderation.NewService(reader) + service := moderation.NewService(reader, postgres.NewModerationRepository(db), moderation.Config{InstanceDID: fixtures.InstanceDID()}) adminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "admin")) nonAdminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "nonadmin")) diff --git a/internal/api/routes/registration_test.go b/internal/api/routes/registration_test.go index a928cb7..081dadd 100644 --- a/internal/api/routes/registration_test.go +++ b/internal/api/routes/registration_test.go @@ -247,8 +247,10 @@ var declaredRoutes = []declaredRoute{ // RegisterAdminReportRoutes — social.coves.admin.* {http.MethodPost, "/xrpc/social.coves.admin.submitReport", authRequired, 10, false}, - // RegisterModerationRoutes — instance-admin-only state query. + // RegisterModerationRoutes — instance-admin-only state query and mutations. {http.MethodGet, "/xrpc/social.coves.moderation.getSubjectState", authRequired, 0, false}, + {http.MethodPost, "/xrpc/social.coves.moderation.removeContent", authRequired, 0, false}, + {http.MethodPost, "/xrpc/social.coves.moderation.restoreContent", authRequired, 0, false}, // RegisterCommunitySuggestionRoutes — social.coves.community.suggestion.* {http.MethodGet, "/xrpc/social.coves.community.suggestion.list", authOptional, 0, false}, diff --git a/internal/atproto/jetstream/comment_consumer.go b/internal/atproto/jetstream/comment_consumer.go index f31374c..1174514 100644 --- a/internal/atproto/jetstream/comment_consumer.go +++ b/internal/atproto/jetstream/comment_consumer.go @@ -3,6 +3,7 @@ package jetstream import ( "Coves/internal/atproto/utils" "Coves/internal/core/comments" + "Coves/internal/core/moderation" "Coves/internal/core/posts" "Coves/internal/core/richtext" "context" @@ -37,12 +38,44 @@ type CommentEventConsumer struct { db *sql.DB // Direct DB access for atomic count updates // bridgeTrust gates whether a comment's user repo may assert bridgedStats. // nil means default-deny (bridgedStats are ignored for every comment). - bridgeTrust *BridgeTrust + bridgeTrust *BridgeTrust + mediaReconciler CommentMediaReconciler } // CommentEventConsumerOption configures optional CommentEventConsumer behaviour. type CommentEventConsumerOption func(*CommentEventConsumer) +// CommentMediaReconciler blocks images introduced on a removed comment. +type CommentMediaReconciler interface { + ReconcileTx(ctx context.Context, tx *sql.Tx, subjectURI string) ([]moderation.MediaBlock, error) + Purge(blocks []moderation.MediaBlock) +} + +// WithCommentMediaReconciler reconciles media blocks when a removed comment is rewritten. +func WithCommentMediaReconciler(reconciler CommentMediaReconciler) CommentEventConsumerOption { + return func(c *CommentEventConsumer) { c.mediaReconciler = reconciler } +} + +// commitCommentWrite reconciles the indexed embed within the write transaction; +// cached bytes are purged only after both the comment and its blocks commit. +func (c *CommentEventConsumer) commitCommentWrite(ctx context.Context, tx *sql.Tx, uri string) error { + var blocks []moderation.MediaBlock + if c.mediaReconciler != nil { + var err error + blocks, err = c.mediaReconciler.ReconcileTx(ctx, tx, uri) + if err != nil { + return fmt.Errorf("reconcile comment media: %w", err) + } + } + if err := tx.Commit(); err != nil { + return err + } + if c.mediaReconciler != nil { + c.mediaReconciler.Purge(blocks) + } + return nil +} + // WithCommentBridgeTrust installs the provenance gate that decides which user repos may // assert bridgedStats on their comments. Without it, bridgedStats are default-denied. func WithCommentBridgeTrust(bt *BridgeTrust) CommentEventConsumerOption { @@ -410,7 +443,7 @@ func (c *CommentEventConsumer) updateComment(ctx context.Context, repoDID string return nil } - if err := tx.Commit(); err != nil { + if err := c.commitCommentWrite(ctx, tx, uri); err != nil { return fmt.Errorf("failed to commit comment update transaction: %w", err) } @@ -593,7 +626,7 @@ func (c *CommentEventConsumer) indexCommentAndUpdateCounts(ctx context.Context, } // Parent unchanged and the row was never decounted, so parent counts // are already correct — commit without the increment sections below. - if commitErr := tx.Commit(); commitErr != nil { + if commitErr := c.commitCommentWrite(ctx, tx, comment.URI); commitErr != nil { return fmt.Errorf("failed to commit transaction: %w", commitErr) } return nil @@ -773,7 +806,7 @@ func (c *CommentEventConsumer) indexCommentAndUpdateCounts(ctx context.Context, // Test coverage: TestPostConsumer_CommentCountReconciliation in post_consumer_test.go if isResurrectionWithSameParent { log.Printf("Resurrection with same parent - skipping parent count increment for: %s", comment.URI) - if err := tx.Commit(); err != nil { + if err := c.commitCommentWrite(ctx, tx, comment.URI); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } return nil @@ -855,14 +888,14 @@ func (c *CommentEventConsumer) indexCommentAndUpdateCounts(ctx context.Context, // Unknown or unsupported parent collection // Comment is still indexed, we just don't update parent counts log.Printf("Comment parent has unsupported collection: %s (comment indexed, parent count not updated)", collection) - if commitErr := tx.Commit(); commitErr != nil { + if commitErr := c.commitCommentWrite(ctx, tx, comment.URI); commitErr != nil { return fmt.Errorf("failed to commit transaction: %w", commitErr) } return nil } // Commit transaction - if err := tx.Commit(); err != nil { + if err := c.commitCommentWrite(ctx, tx, comment.URI); err != nil { return fmt.Errorf("failed to commit transaction: %w", err) } diff --git a/internal/config/config.go b/internal/config/config.go index e03e8d5..a4b65d6 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -146,7 +146,9 @@ type Config struct { type ModerationConfig struct { // Admins is the operator-managed allowlist of instance admin DIDs // (MODERATION_ADMINS). An empty list grants nobody admin authority. - Admins []string + Admins []string + IdempotencyRetention time.Duration + MaxLiveIdempotencyKeys int } // DatabaseConfig holds the AppView PostgreSQL connection and pool settings. @@ -527,7 +529,9 @@ func Load() (*Config, error) { if err := cfg.loadSubmissions(); err != nil { return nil, err } - cfg.Moderation.Admins = csvVar("MODERATION_ADMINS") + if err := cfg.loadModeration(); err != nil { + return nil, err + } cfg.PDS = PDSConfig{ URL: stringVar("PDS_URL", "http://localhost:3001"), @@ -1018,6 +1022,29 @@ func (c *Config) loadSubmissions() error { return nil } +func (c *Config) loadModeration() error { + retention, err := durationVar("MODERATION_IDEMPOTENCY_RETENTION", 24*time.Hour) + if err != nil { + return err + } + if retention <= 0 { + return fmt.Errorf("MODERATION_IDEMPOTENCY_RETENTION must be greater than 0 (got %s)", retention) + } + maxKeys, err := intVar("MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS", 1000) + if err != nil { + return err + } + if maxKeys <= 0 { + return fmt.Errorf("MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS must be greater than 0 (got %d)", maxKeys) + } + c.Moderation = ModerationConfig{ + Admins: csvVar("MODERATION_ADMINS"), + IdempotencyRetention: retention, + MaxLiveIdempotencyKeys: maxKeys, + } + return nil +} + // Validate enforces the constraints that Load's defaults cannot express, // notably the ones that differ between dev and production. It returns every // problem at once so a misconfigured deployment can be fixed in a single pass diff --git a/internal/config/moderation_idempotency_test.go b/internal/config/moderation_idempotency_test.go new file mode 100644 index 0000000..9455fd9 --- /dev/null +++ b/internal/config/moderation_idempotency_test.go @@ -0,0 +1,72 @@ +package config + +import ( + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestLoadModerationIdempotencySettings(t *testing.T) { + for _, test := range []struct { + name string + retention string + maxKeys string + wantTime time.Duration + wantKeys int + }{ + {name: "defaults", wantTime: 24 * time.Hour, wantKeys: 1000}, + {name: "configured limits", retention: "2h", maxKeys: "5", wantTime: 2 * time.Hour, wantKeys: 5}, + } { + t.Run(test.name, func(t *testing.T) { + clearEnv(t) + prodEnv(t) + t.Setenv("MODERATION_IDEMPOTENCY_RETENTION", test.retention) + t.Setenv("MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS", test.maxKeys) + cfg, err := Load() + require.NoError(t, err) + assert.Equal(t, test.wantTime, cfg.Moderation.IdempotencyRetention) + assert.Equal(t, test.wantKeys, cfg.Moderation.MaxLiveIdempotencyKeys) + }) + } +} + +func TestLoadModerationIdempotencyRejectsInvalidSettings(t *testing.T) { + for _, test := range []struct { + name string + value string + }{ + {"MODERATION_IDEMPOTENCY_RETENTION", "0s"}, + {"MODERATION_IDEMPOTENCY_RETENTION", "-1h"}, + {"MODERATION_IDEMPOTENCY_RETENTION", "banana"}, + {"MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS", "0"}, + {"MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS", "-1"}, + {"MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS", "x"}, + } { + t.Run(test.name+"="+test.value, func(t *testing.T) { + clearEnv(t) + prodEnv(t) + t.Setenv(test.name, test.value) + _, err := Load() + require.Error(t, err) + assert.ErrorContains(t, err, test.name) + }) + } +} + +func TestClearEnvForTestClearsModerationIdempotencySettings(t *testing.T) { + for _, name := range []string{"MODERATION_IDEMPOTENCY_RETENTION", "MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS"} { + t.Setenv(name, "invalid") + } + ClearEnvForTest(t) + for _, name := range []string{"MODERATION_IDEMPOTENCY_RETENTION", "MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS"} { + assert.Empty(t, os.Getenv(name), "%s must be in loadedEnvVars", name) + } + t.Setenv("IS_DEV_ENV", "true") + cfg, err := Load() + require.NoError(t, err) + assert.Equal(t, 24*time.Hour, cfg.Moderation.IdempotencyRetention) + assert.Equal(t, 1000, cfg.Moderation.MaxLiveIdempotencyKeys) +} diff --git a/internal/config/testing.go b/internal/config/testing.go index a765010..5979023 100644 --- a/internal/config/testing.go +++ b/internal/config/testing.go @@ -18,7 +18,7 @@ var loadedEnvVars = []string{ "OAUTH_CLIENT_PRIVATE_KEY", "OAUTH_CLIENT_KEY_ID", "INSTANCE_DID", "INSTANCE_DOMAIN", "COMMUNITY_CREATORS", "TRUSTED_BRIDGE_PDS_HOSTS", "SKIP_DID_WEB_VERIFICATION", - "MODERATION_ADMINS", + "MODERATION_ADMINS", "MODERATION_IDEMPOTENCY_RETENTION", "MODERATION_IDEMPOTENCY_MAX_LIVE_KEYS", "BRIDGED_VOTE_POLL_INTERVAL", "BRIDGED_VOTE_POLL_LOOKBACK", "BRIDGED_VOTE_POLL_SWEEP_CAP", "PDS_URL", "PDS_INSTANCE_HANDLE", "PDS_INSTANCE_PASSWORD", "PDS_ADMIN_PASSWORD", "JETSTREAM_FEEDS", "REDRIVE_INTERVAL", "IDENTITY_NEGATIVE_CACHE_TTL", diff --git a/internal/core/comments/comment_moderation_consumer_integration_test.go b/internal/core/comments/comment_moderation_consumer_integration_test.go new file mode 100644 index 0000000..75a0efb --- /dev/null +++ b/internal/core/comments/comment_moderation_consumer_integration_test.go @@ -0,0 +1,353 @@ +//go:build integration + +package comments_test + +import ( + "context" + "database/sql" + "testing" + "time" + + "Coves/internal/atproto/jetstream" + "Coves/internal/core/comments" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type consumerPurgeCall struct { + ownerDID string + blobCID string + visible bool + err error +} + +type consumerPurger struct { + db *sql.DB + calls []consumerPurgeCall +} + +func (p *consumerPurger) record(ownerDID, blobCID string) error { + call := consumerPurgeCall{ownerDID: ownerDID, blobCID: blobCID} + // A separate connection cannot see an uncommitted consumer transaction. + call.err = p.db.QueryRowContext(context.Background(), ` + SELECT EXISTS (SELECT 1 FROM moderation_media_blocks + WHERE owner_did IS NOT DISTINCT FROM NULLIF($1, '') + AND blob_cid = $2 AND active) + `, ownerDID, blobCID).Scan(&call.visible) + p.calls = append(p.calls, call) + return call.err +} + +func (p *consumerPurger) PurgeOwnerBlob(ownerDID, blobCID string) error { + return p.record(ownerDID, blobCID) +} + +func (p *consumerPurger) PurgeBlob(blobCID string) error { + return p.record("", blobCID) +} + +func consumerCommentEvent(authorDID, postURI, postCID, rkey, operation, rev, cid, text, imageCID string, eventTime time.Time) *jetstream.JetstreamEvent { + event := &jetstream.JetstreamEvent{ + Did: authorDID, Kind: "commit", TimeUS: eventTime.UnixMicro(), + Commit: &jetstream.CommitEvent{ + Rev: rev, Operation: operation, Collection: moderation.CommentCollection, + RKey: rkey, CID: cid, + }, + } + if operation != "delete" { + record := map[string]interface{}{ + "$type": moderation.CommentCollection, "content": text, + "reply": map[string]interface{}{ + "root": map[string]interface{}{"uri": postURI, "cid": postCID}, + "parent": map[string]interface{}{"uri": postURI, "cid": postCID}, + }, + "createdAt": eventTime.Format(time.RFC3339), + } + if imageCID != "" { + record["embed"] = map[string]interface{}{ + "$type": "social.coves.embed.images", + "images": []interface{}{map[string]interface{}{ + "alt": "indexed image", + "image": map[string]interface{}{ + "$type": "blob", "ref": map[string]interface{}{"$link": imageCID}, + "mimeType": "image/png", "size": 10, + }, + }}, + } + } + event.Commit.Record = record + } + return event +} + +func TestModerationCommentConsumerReconcilesRemovedImages(t *testing.T) { + for _, scenario := range []struct { + name, reason, operation string + initialImage bool + remove bool + }{ + {name: "edit adds image under spam removal", reason: moderationTestReason, operation: "update", initialImage: true, remove: true}, + {name: "illegal content edit adds ownerless block", reason: "social.coves.moderation.defs#reasonIllegalContent", operation: "update", remove: true}, + {name: "duplicate create after removal does not purge", reason: moderationTestReason, operation: "duplicate", initialImage: true, remove: true}, + {name: "author delete then recreate same URI", reason: moderationTestReason, operation: "recreate", remove: true}, + {name: "newer-rev re-create of the active row", reason: moderationTestReason, operation: "recreate-active", initialImage: true, remove: true}, + {name: "author delete then recreate under an unsupported parent", reason: moderationTestReason, operation: "recreate-unsupported-parent", remove: true}, + {name: "purged row recreated by a fresh insert", reason: moderationTestReason, operation: "purge-recreate", initialImage: true, remove: true}, + {name: "ordinary edit does not block image", operation: "update"}, + } { + t.Run(scenario.name, func(t *testing.T) { + db := testkit.DB(t) + ctx := t.Context() + owner := testkit.UniqueIDWithPrefix(t, "mediaowner") + authorDID := fixtures.DID(owner) + fixtures.User(t, db, owner+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "mediathread") + communityDID, err := fixtures.Community(ctx, db, communityName, "owner"+communityName) + require.NoError(t, err) + postURI := fixtures.Post(t, db, communityDID, authorDID, "media reconciliation", 0, time.Now()) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(ctx, postURI) + require.NoError(t, err) + + moderationRepo := postgres.NewModerationRepository(db) + purger := &consumerPurger{db: db} + consumer := jetstream.NewCommentEventConsumer(postgres.NewCommentRepository(db), db, + jetstream.WithCommentMediaReconciler(moderation.NewMediaReconciler(moderationRepo, fixtures.InstanceDID(), purger))) + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + moderationRepo, + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + commentService := setupCommentService(db) + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.CommentCollection + "/" + rkey + initialImageCID := "" + if scenario.initialImage { + initialImageCID = moderatedImageCID + } + started := time.Now().Add(-2 * time.Minute) + create := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "create", "v1", moderatedCommentCID, "original comment", initialImageCID, started) + require.NoError(t, consumer.HandleEvent(ctx, create)) + initial, err := postgres.NewCommentRepository(db).GetByURI(ctx, uri) + require.NoError(t, err) + assert.Equal(t, moderatedCommentCID, initial.CID) + var removal *moderation.MutationResult + if scenario.remove { + removal, err = moderationService.RemoveContent(ctx, fixtures.DID(testkit.UniqueIDWithPrefix(t, "mediaadmin")), moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: uri, CID: moderatedCommentCID}, ExpectedVersion: "v0", + IdempotencyKey: "consumer-remove", Reason: scenario.reason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removal.Outcome) + } + assert.Empty(t, purger.calls, "the fake is installed on the consumer, not the removal service") + + newImageCID := moderatedCommentCID // Distinct from the original image CID. + switch scenario.operation { + case "update": + update := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "update", "v2", moderatedImageCID, "edited comment", newImageCID, started.Add(time.Minute)) + require.NoError(t, consumer.HandleEvent(ctx, update)) + case "duplicate": + require.NoError(t, consumer.HandleEvent(ctx, create)) + case "recreate": + deleted := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "delete", "v2", "", "", "", started.Add(time.Minute)) + require.NoError(t, consumer.HandleEvent(ctx, deleted)) + recreated := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "create", "v3", moderatedImageCID, "recreated comment", newImageCID, started.Add(2*time.Minute)) + require.NoError(t, consumer.HandleEvent(ctx, recreated)) + case "recreate-active": + recreated := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "create", "v2", moderatedImageCID, "recreated comment", newImageCID, started.Add(time.Minute)) + require.NoError(t, consumer.HandleEvent(ctx, recreated)) + case "recreate-unsupported-parent": + deleted := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "delete", "v2", "", "", "", started.Add(time.Minute)) + require.NoError(t, consumer.HandleEvent(ctx, deleted)) + recreated := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "create", "v3", moderatedImageCID, "recreated comment", newImageCID, started.Add(2*time.Minute)) + recreated.Commit.Record["reply"].(map[string]interface{})["parent"] = map[string]interface{}{ + "uri": "at://" + authorDID + "/social.coves.unsupported.collection/" + testkit.TID(), "cid": post.CID, + } + require.NoError(t, consumer.HandleEvent(ctx, recreated)) + case "purge-recreate": + // Account-deletion purge removes the row outright; the next create + // takes the fresh-insert path rather than resurrection. + _, err := db.ExecContext(ctx, `DELETE FROM comments WHERE uri = $1`, uri) + require.NoError(t, err) + recreated := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "create", "v2", moderatedImageCID, "recreated comment", newImageCID, started.Add(time.Minute)) + require.NoError(t, consumer.HandleEvent(ctx, recreated)) + } + + indexed, err := postgres.NewCommentRepository(db).GetByURI(ctx, uri) + require.NoError(t, err) + if scenario.operation == "duplicate" { + assert.Equal(t, "original comment", indexed.Content) + assert.Equal(t, moderatedCommentCID, indexed.CID) + } else { + assert.Equal(t, moderatedImageCID, indexed.CID, "the new record must actually be indexed") + require.NotNil(t, indexed.Embed) + assert.Contains(t, *indexed.Embed, newImageCID) + assert.Nil(t, indexed.DeletedAt, "recreated comment must be present in the index") + } + + state, err := moderationService.GetSubjectState(ctx, uri) + require.NoError(t, err) + require.NotNil(t, state.CurrentSubject) + assert.Equal(t, indexed.CID, state.CurrentSubject.CID) + response, err := commentService.GetComments(ctx, &comments.GetCommentsRequest{PostURI: postURI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + if scenario.operation == "recreate-unsupported-parent" { + require.Empty(t, response.Comments, "a comment under an unsupported parent is not a top-level reply") + } else { + require.Len(t, response.Comments, 1) + } + if scenario.remove { + var activeAction string + require.NoError(t, db.QueryRowContext(ctx, `SELECT active_action_id FROM moderation_decisions WHERE subject_uri = $1 AND authority_did = $2 AND kind = 'removal' AND active`, uri, fixtures.InstanceDID()).Scan(&activeAction)) + assert.Equal(t, removal.Action.ID, activeAction) + assert.Equal(t, removal.State.Version, state.Version, "an author event must not advance the moderation version") + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, activeAction, state.LocalRemoval.ActionID) + } + if scenario.remove && len(response.Comments) == 1 { + view := response.Comments[0].Comment + assert.True(t, view.IsDeleted) + require.NotNil(t, view.Moderation) + assert.Equal(t, moderation.ModerationStateRemoved, view.Moderation.State) + assert.Nil(t, view.Record, "the edited/recreated text must stay hidden") + assert.Nil(t, view.Embed) + } else if !scenario.remove { + view := response.Comments[0].Comment + assert.Equal(t, moderation.ModerationStateClear, state.Moderation.State) + assert.False(t, view.IsDeleted) + assert.Nil(t, view.Moderation) + require.NotNil(t, view.Record) + } + + var blocks int + require.NoError(t, db.QueryRowContext(ctx, `SELECT count(*) FROM moderation_media_blocks WHERE blob_cid = $1 AND active`, newImageCID).Scan(&blocks)) + if scenario.operation == "duplicate" || !scenario.remove { + assert.Zero(t, blocks) + assert.Empty(t, purger.calls, "replays and unremoved comments must never trigger cache purges") + if scenario.initialImage { + blocked, checkErr := moderationRepo.IsBlocked(ctx, authorDID, initialImageCID) + require.NoError(t, checkErr) + assert.True(t, blocked, "the original removal's image block must survive duplicate delivery") + } + return + } + blocked, err := moderationRepo.IsBlocked(ctx, authorDID, newImageCID) + require.NoError(t, err) + assert.True(t, blocked, "new images on a removed comment must be blocked") + require.NotEmpty(t, purger.calls, "new pair must be purged") + assert.Equal(t, consumerPurgeCall{ownerDID: authorDID, blobCID: newImageCID, visible: true}, purger.calls[0], "pair purge must happen after commit") + if scenario.reason == "social.coves.moderation.defs#reasonIllegalContent" { + assert.Equal(t, 2, blocks, "illegal content blocks both the owner pair and every owner") + require.Len(t, purger.calls, 2) + assert.Contains(t, purger.calls, consumerPurgeCall{blobCID: newImageCID, visible: true}, "ownerless purge must happen after commit") + otherOwnerBlocked, checkErr := moderationRepo.IsBlocked(ctx, fixtures.DID("otherimageowner"), newImageCID) + require.NoError(t, checkErr) + assert.True(t, otherOwnerBlocked) + } else { + assert.Equal(t, 1, blocks, "spam blocks only the image owner's pair") + require.Len(t, purger.calls, 1) + otherOwnerBlocked, checkErr := moderationRepo.IsBlocked(ctx, fixtures.DID("otherimageowner"), newImageCID) + require.NoError(t, checkErr) + assert.False(t, otherOwnerBlocked, "a spam block must not affect another owner") + } + if scenario.initialImage { + oldBlocked, checkErr := moderationRepo.IsBlocked(ctx, authorDID, initialImageCID) + require.NoError(t, checkErr) + assert.True(t, oldBlocked, "editing must not release the original image block") + } + }) + } +} + +// An author controls the embed of a removed comment. No shape of it may make +// the consumer's reconciliation fail, and every image blob the comment view can +// serve — including the legacy top-level cid encoding — must be blocked. +func TestModerationCommentConsumerReconcilesMalformedEmbeds(t *testing.T) { + blob := func(cid string) map[string]interface{} { + return map[string]interface{}{"$type": "blob", "ref": map[string]interface{}{"$link": cid}, "mimeType": "image/png", "size": 10} + } + for _, test := range []struct { + name string + embed map[string]interface{} + blocked []string + }{ + {name: "images is a string", embed: map[string]interface{}{"$type": "social.coves.embed.images", "images": "x"}}, + {name: "image is a string", embed: map[string]interface{}{"$type": "social.coves.embed.images", "images": []interface{}{map[string]interface{}{"image": "x"}}}}, + {name: "ref is a string", embed: map[string]interface{}{"$type": "social.coves.embed.images", "images": []interface{}{map[string]interface{}{"image": map[string]interface{}{"ref": moderatedCommentCID}}}}}, + {name: "type is not a string", embed: map[string]interface{}{"$type": 5, "images": []interface{}{map[string]interface{}{"image": blob(moderatedCommentCID)}}}}, + { + name: "legacy cid blob beside malformed entries", + embed: map[string]interface{}{"$type": "social.coves.embed.images", "images": []interface{}{ + "x", map[string]interface{}{"image": 7}, + map[string]interface{}{"image": map[string]interface{}{"cid": moderatedCommentCID, "mimeType": "image/png"}}, + }}, + blocked: []string{moderatedCommentCID}, + }, + } { + t.Run(test.name, func(t *testing.T) { + db := testkit.DB(t) + ctx := t.Context() + owner := testkit.UniqueIDWithPrefix(t, "malformedowner") + authorDID := fixtures.DID(owner) + fixtures.User(t, db, owner+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "malformedthread") + communityDID, err := fixtures.Community(ctx, db, communityName, "owner"+communityName) + require.NoError(t, err) + postURI := fixtures.Post(t, db, communityDID, authorDID, "malformed embed reconciliation", 0, time.Now()) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(ctx, postURI) + require.NoError(t, err) + + moderationRepo := postgres.NewModerationRepository(db) + purger := &consumerPurger{db: db} + consumer := jetstream.NewCommentEventConsumer(postgres.NewCommentRepository(db), db, + jetstream.WithCommentMediaReconciler(moderation.NewMediaReconciler(moderationRepo, fixtures.InstanceDID(), purger))) + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + moderationRepo, + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.CommentCollection + "/" + rkey + started := time.Now().Add(-2 * time.Minute) + create := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "create", "v1", moderatedCommentCID, "original comment", moderatedImageCID, started) + require.NoError(t, consumer.HandleEvent(ctx, create)) + removal, err := moderationService.RemoveContent(ctx, fixtures.DID(testkit.UniqueIDWithPrefix(t, "malformedadmin")), moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: uri, CID: moderatedCommentCID}, ExpectedVersion: "v0", + IdempotencyKey: "malformed-remove", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removal.Outcome) + + update := consumerCommentEvent(authorDID, postURI, post.CID, rkey, "update", "v2", moderatedImageCID, "edited comment", "", started.Add(time.Minute)) + update.Commit.Record["embed"] = test.embed + require.NoError(t, consumer.HandleEvent(ctx, update), "a malformed embed must not fail reconciliation") + indexed, err := postgres.NewCommentRepository(db).GetByURI(ctx, uri) + require.NoError(t, err) + assert.Equal(t, moderatedImageCID, indexed.CID, "the edit must be indexed") + + rows, err := db.QueryContext(ctx, ` + SELECT blob_cid FROM moderation_media_blocks + WHERE action_id = $1 AND active AND blob_cid <> $2 ORDER BY blob_cid + `, removal.Action.ID, moderatedImageCID) + require.NoError(t, err) + defer rows.Close() + var blocked []string + for rows.Next() { + var blobCID string + require.NoError(t, rows.Scan(&blobCID)) + blocked = append(blocked, blobCID) + } + require.NoError(t, rows.Err()) + assert.Equal(t, test.blocked, blocked) + originalBlocked, err := moderationRepo.IsBlocked(ctx, authorDID, moderatedImageCID) + require.NoError(t, err) + assert.True(t, originalBlocked, "the original image block must survive the edit") + }) + } +} diff --git a/internal/core/comments/comment_moderation_integration_test.go b/internal/core/comments/comment_moderation_integration_test.go new file mode 100644 index 0000000..48d24e5 --- /dev/null +++ b/internal/core/comments/comment_moderation_integration_test.go @@ -0,0 +1,362 @@ +//go:build integration + +package comments_test + +import ( + "database/sql" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + "Coves/internal/core/comments" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + moderatedCommentCID = "bafyreigj3fwnwjuzr35k2kuzmb5dixxczrzjhqkr5srlqplsh6gq3bj3si" + moderatedImageCID = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" + moderationTestReason = "social.coves.moderation.defs#reasonSpam" +) + +func moderationThreadFixture(t *testing.T) (*sql.DB, comments.Service, moderation.Service, string, moderation.StrongRef, string) { + t.Helper() + db := testkit.DB(t) + postRepository := postgres.NewPostRepository(db) + commentRepository := postgres.NewCommentRepository(db) + service := setupCommentService(db) + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepository, commentRepository), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + name := testkit.UniqueIDWithPrefix(t, "threadauthor") + authorDID := fixtures.DID(name) + fixtures.User(t, db, name+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "threadcommunity") + communityDID, err := fixtures.Community(t.Context(), db, communityName, "owner"+communityName) + require.NoError(t, err) + postURI := fixtures.Post(t, db, communityDID, authorDID, "moderation thread fixture", 0, time.Now()) + post, err := postRepository.GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + rkey := testkit.TID() + subject := moderation.StrongRef{URI: "at://" + authorDID + "/" + moderation.CommentCollection + "/" + rkey, CID: moderatedCommentCID} + _, err = db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, + content, content_facets, embed, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $5, $6, $7, $8::jsonb, $9::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, authorDID, postURI, post.CID, + "private original moderation text", `[{"index":{"byteStart":0,"byteEnd":7},"features":[{"$type":"app.bsky.richtext.facet#tag","tag":"privateFacetOnly"}]}]`, + fmt.Sprintf(`{"$type":"social.coves.embed.images","images":[{"image":{"$type":"blob","ref":{"$link":"%s"},"mimeType":"image/png","size":10},"alt":"privateEmbedOnly"}]}`, moderatedImageCID)) + require.NoError(t, err) + return db, service, moderationService, postURI, subject, authorDID +} + +func moderationThreadReply(t *testing.T, db *sql.DB, authorDID, postURI, rootCID string, parent moderation.StrongRef, content string, createdAt time.Time) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.CommentCollection + "/" + rkey + _, err := db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) + `, uri, moderatedCommentCID, rkey, authorDID, postURI, rootCID, parent.URI, parent.CID, content, createdAt) + require.NoError(t, err) + result, err := db.ExecContext(t.Context(), `UPDATE comments SET reply_count = reply_count + 1 WHERE uri = $1`, parent.URI) + require.NoError(t, err) + rows, err := result.RowsAffected() + require.NoError(t, err) + require.EqualValues(t, 1, rows) + return uri +} + +func moderationThreadComment(t *testing.T, response *comments.GetCommentsResponse, uri string) *comments.ThreadViewComment { + t.Helper() + for _, node := range response.Comments { + if node.Comment.URI == uri { + return node + } + } + t.Fatalf("comment %s missing from thread", uri) + return nil +} + +func TestModerationCommentThreadPlaceholderPaginatesAndRestoresIndexedActivity(t *testing.T) { + db, commentService, moderationService, postURI, subject, authorDID := moderationThreadFixture(t) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + replyURIs := make(map[string]string) + for i := 0; i < 3; i++ { + content := fmt.Sprintf("visible reply %d", i) + replyURIs[moderationThreadReply(t, db, authorDID, postURI, post.CID, subject, content, time.Now().Add(-time.Duration(4-i)*time.Minute))] = content + } + threadRequest := &comments.GetCommentsRequest{PostURI: postURI, Sort: "new", Depth: 1, Limit: 10} + before, err := commentService.GetComments(t.Context(), threadRequest) + require.NoError(t, err) + require.Len(t, before.Comments, 1) + initialView := moderationThreadComment(t, before, subject.URI).Comment + require.IsType(t, &comments.CommentRecord{}, initialView.Record) + assert.Equal(t, "private original moderation text", initialView.Record.(*comments.CommentRecord).Content) + assert.NotNil(t, initialView.Embed) + initialJSON, err := json.Marshal(initialView) + require.NoError(t, err) + assert.Contains(t, string(initialJSON), "privateFacetOnly", "facet must be served before removal for the leak check to be meaningful") + assert.Contains(t, string(initialJSON), moderatedImageCID, "embed must be served before removal for the leak check to be meaningful") + + adminA := fixtures.DID(testkit.UniqueIDWithPrefix(t, "threadadmina")) + removed, err := moderationService.RemoveContent(t.Context(), adminA, moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "thread-remove", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.NotNil(t, removed) + require.NotNil(t, removed.Action) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + var originalActionRow string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT row_to_json(a)::text FROM moderation_actions a WHERE id = $1`, removed.Action.ID).Scan(&originalActionRow)) + + whileRemoved, err := commentService.GetComments(t.Context(), threadRequest) + require.NoError(t, err) + node := moderationThreadComment(t, whileRemoved, subject.URI) + view := node.Comment + assert.True(t, view.IsDeleted) + require.NotNil(t, view.DeletionReason) + assert.Equal(t, "moderator", *view.DeletionReason) + require.NotNil(t, view.Moderation) + assert.Equal(t, "removed", view.Moderation.State) + assert.Equal(t, []comments.ModerationSourceView{{AuthorityDID: fixtures.InstanceDID(), Scope: comments.ModerationScopeView{Kind: "instance"}}}, view.Moderation.Sources) + assert.Nil(t, view.Record) + assert.Nil(t, view.Embed) + assert.Nil(t, view.Viewer) + assert.Nil(t, view.DeletedAt) + require.NotNil(t, view.Author) + assert.Equal(t, authorDID, view.Author.DID) + assert.Equal(t, "handle.invalid", view.Author.Handle) + require.NotNil(t, view.Stats) + assert.Zero(t, view.Stats.Upvotes) + assert.Zero(t, view.Stats.Downvotes) + assert.Zero(t, view.Stats.Score) + assert.Equal(t, 3, view.Stats.ReplyCount) + require.Len(t, node.Replies, 3) + for _, child := range node.Replies { + assert.Equal(t, replyURIs[child.Comment.URI], child.Comment.Record.(*comments.CommentRecord).Content) + assert.Nil(t, child.Comment.Moderation) + } + serialized, err := json.Marshal(whileRemoved) + require.NoError(t, err) + for _, secret := range []string{"private original moderation text", "privateFacetOnly", moderatedImageCID, "privateEmbedOnly"} { + assert.NotContains(t, string(serialized), secret, "removed comment data must not escape into the served response") + } + var indexedContent, indexedFacets, indexedEmbed string + var deletedAt sql.NullTime + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT content, content_facets::text, embed::text, deleted_at FROM comments WHERE uri = $1`, subject.URI).Scan(&indexedContent, &indexedFacets, &indexedEmbed, &deletedAt)) + assert.Equal(t, "private original moderation text", indexedContent, "moderator removal must remain an overlay") + assert.Contains(t, indexedFacets, "privateFacetOnly") + assert.Contains(t, indexedEmbed, moderatedImageCID) + assert.False(t, deletedAt.Valid) + + // The subtree cursor paginates direct replies, retaining the removed parent on every page. + firstPage, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{ + PostURI: postURI, ParentRkey: strings.TrimPrefix(subject.URI, "at://"+authorDID+"/"+moderation.CommentCollection+"/"), + Sort: "new", Depth: 1, Limit: 2, + }) + require.NoError(t, err) + require.Len(t, firstPage.Comments, 1) + assert.True(t, firstPage.Comments[0].Comment.IsDeleted) + assert.Equal(t, "removed", firstPage.Comments[0].Comment.Moderation.State) + require.Len(t, firstPage.Comments[0].Replies, 2) + require.NotNil(t, firstPage.Cursor) + assert.True(t, firstPage.Comments[0].HasMore) + secondPage, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{ + PostURI: postURI, ParentRkey: strings.TrimPrefix(subject.URI, "at://"+authorDID+"/"+moderation.CommentCollection+"/"), + Sort: "new", Depth: 1, Limit: 2, Cursor: firstPage.Cursor, + }) + require.NoError(t, err) + require.Len(t, secondPage.Comments, 1) + assert.True(t, secondPage.Comments[0].Comment.IsDeleted) + assert.Equal(t, "removed", secondPage.Comments[0].Comment.Moderation.State) + require.Len(t, secondPage.Comments[0].Replies, 1) + assert.Nil(t, secondPage.Cursor) + seen := make(map[string]bool) + for _, page := range []*comments.GetCommentsResponse{firstPage, secondPage} { + for _, child := range page.Comments[0].Replies { + assert.False(t, seen[child.Comment.URI], "a reply must not appear on both pages") + seen[child.Comment.URI] = true + assert.Equal(t, replyURIs[child.Comment.URI], child.Comment.Record.(*comments.CommentRecord).Content) + } + } + assert.Len(t, seen, 3) + + voterDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "threadvoter")) + voteRkey := testkit.TID() + _, err = db.ExecContext(t.Context(), ` + INSERT INTO votes (uri, cid, rkey, voter_did, subject_uri, subject_cid, direction, created_at) + VALUES ($1, $2, $3, $4, $5, $6, 'up', NOW()) + `, "at://"+voterDID+"/social.coves.interaction.vote/"+voteRkey, moderatedCommentCID, voteRkey, voterDID, subject.URI, subject.CID) + require.NoError(t, err) + result, err := db.ExecContext(t.Context(), `UPDATE comments SET upvote_count = upvote_count + 1, score = score + 1 WHERE uri = $1`, subject.URI) + require.NoError(t, err) + rows, err := result.RowsAffected() + require.NoError(t, err) + require.EqualValues(t, 1, rows) + newReply := moderationThreadReply(t, db, authorDID, postURI, post.CID, subject, "reply indexed during removal", time.Now()) + stillRemoved, err := commentService.GetComments(t.Context(), threadRequest) + require.NoError(t, err) + assert.Zero(t, moderationThreadComment(t, stillRemoved, subject.URI).Comment.Stats.Upvotes) + assert.Equal(t, 4, moderationThreadComment(t, stillRemoved, subject.URI).Comment.Stats.ReplyCount) + + adminB := fixtures.DID(testkit.UniqueIDWithPrefix(t, "threadadminb")) + restored, err := moderationService.RestoreContent(t.Context(), adminB, moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: removed.State.Version, + IdempotencyKey: "thread-restore", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.NotNil(t, restored) + require.NotNil(t, restored.Action) + assert.Equal(t, moderation.OutcomeApplied, restored.Outcome) + assert.Equal(t, removed.Action.ID, restored.Action.ReversesActionID) + var reversedID, restoredActor, originalActionAfter string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT reverses_action_id, actor_did FROM moderation_actions WHERE id = $1`, restored.Action.ID).Scan(&reversedID, &restoredActor)) + assert.Equal(t, removed.Action.ID, reversedID) + assert.Equal(t, adminB, restoredActor) + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT row_to_json(a)::text FROM moderation_actions a WHERE id = $1`, removed.Action.ID).Scan(&originalActionAfter)) + assert.Equal(t, originalActionRow, originalActionAfter, "restoring must not rewrite the original action") + + after, err := commentService.GetComments(t.Context(), threadRequest) + require.NoError(t, err) + node = moderationThreadComment(t, after, subject.URI) + view = node.Comment + assert.False(t, view.IsDeleted) + assert.Nil(t, view.Moderation) + require.IsType(t, &comments.CommentRecord{}, view.Record) + assert.Equal(t, "private original moderation text", view.Record.(*comments.CommentRecord).Content) + assert.NotNil(t, view.Embed) + restoredJSON, err := json.Marshal(view) + require.NoError(t, err) + assert.Contains(t, string(restoredJSON), "privateFacetOnly") + assert.Contains(t, string(restoredJSON), moderatedImageCID) + assert.Equal(t, 1, view.Stats.Upvotes) + assert.Equal(t, 1, view.Stats.Score) + assert.Equal(t, 4, view.Stats.ReplyCount) + require.Len(t, node.Replies, 4) + replyFound := false + for _, child := range node.Replies { + if child.Comment.URI == newReply { + replyFound = true + assert.Equal(t, "reply indexed during removal", child.Comment.Record.(*comments.CommentRecord).Content) + } + } + assert.True(t, replyFound, "a reply indexed during removal must be served after restore") +} + +func TestModerationAuthorDeletedCommentRetainsAuthorPlaceholderAfterRemoval(t *testing.T) { + db, commentService, moderationService, postURI, subject, authorDID := moderationThreadFixture(t) + _, err := db.ExecContext(t.Context(), ` + UPDATE comments SET deleted_at = NOW(), deletion_reason = 'author', deleted_by = $1 WHERE uri = $2 + `, authorDID, subject.URI) + require.NoError(t, err) + indexedCID := subject.CID + subject.CID = moderatedImageCID // Q5/R7: the author-deleted record accepts a different supplied CID. + removed, err := moderationService.RemoveContent(t.Context(), fixtures.DID("authordeletedadmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "author-deleted-remove", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.NotNil(t, removed) + require.NotNil(t, removed.Action) + assert.Equal(t, moderation.OutcomeApplied, removed.Outcome) + assert.Equal(t, indexedCID, removed.Action.ObservedCID) + response, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{PostURI: postURI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + require.Len(t, response.Comments, 1) + view := response.Comments[0].Comment + assert.True(t, view.IsDeleted) + require.NotNil(t, view.DeletionReason) + assert.Equal(t, "author", *view.DeletionReason) + require.NotNil(t, view.Moderation) + assert.Equal(t, "removed", view.Moderation.State) + assert.Equal(t, []comments.ModerationSourceView{{AuthorityDID: fixtures.InstanceDID(), Scope: comments.ModerationScopeView{Kind: "instance"}}}, view.Moderation.Sources) + assert.Nil(t, view.Record) + state, err := moderationService.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, moderation.RecordStateDeleted, state.RecordState) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) +} + +func TestModerationActorCommentsOmitsRemovedAndAuthorDeletedAcrossPages(t *testing.T) { + db, commentService, moderationService, postURI, first, authorDID := moderationThreadFixture(t) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + secondURI := moderationThreadReply(t, db, authorDID, postURI, post.CID, first, "removed from profile", time.Now().Add(-time.Minute)) + thirdURI := moderationThreadReply(t, db, authorDID, postURI, post.CID, first, "deleted by author", time.Now().Add(-2*time.Minute)) + fourthURI := moderationThreadReply(t, db, authorDID, postURI, post.CID, first, "restored to profile", time.Now().Add(-3*time.Minute)) + + list := func(limit int, cursor *string) *comments.GetActorCommentsResponse { + t.Helper() + response, err := commentService.GetActorComments(t.Context(), &comments.GetActorCommentsRequest{ + ActorDID: authorDID, Limit: limit, Cursor: cursor, + }) + require.NoError(t, err) + return response + } + before := list(10, nil) + require.Len(t, before.Comments, 4, "all four comments must be visible before moderation") + assert.Equal(t, []string{first.URI, secondURI, thirdURI, fourthURI}, []string{ + before.Comments[0].URI, before.Comments[1].URI, before.Comments[2].URI, before.Comments[3].URI, + }) + + adminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "actoradmin")) + second := moderation.StrongRef{URI: secondURI, CID: moderatedCommentCID} + removed, err := moderationService.RemoveContent(t.Context(), adminDID, moderation.RemoveContentRequest{ + Subject: second, ExpectedVersion: "v0", IdempotencyKey: "actor-remove-second", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + _, err = db.ExecContext(t.Context(), ` + UPDATE comments SET deleted_at = NOW(), deletion_reason = 'author', deleted_by = $1 WHERE uri = $2 + `, authorDID, thirdURI) + require.NoError(t, err) + fourth := moderation.StrongRef{URI: fourthURI, CID: moderatedCommentCID} + removedFourth, err := moderationService.RemoveContent(t.Context(), adminDID, moderation.RemoveContentRequest{ + Subject: fourth, ExpectedVersion: "v0", IdempotencyKey: "actor-remove-fourth", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removedFourth.Outcome) + restored, err := moderationService.RestoreContent(t.Context(), adminDID, moderation.RestoreContentRequest{ + ActionID: removedFourth.Action.ID, ReviewedSubject: &fourth, ExpectedVersion: removedFourth.State.Version, + IdempotencyKey: "actor-restore-fourth", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + + whole := list(10, nil) + var wholeURIs []string + for _, comment := range whole.Comments { + wholeURIs = append(wholeURIs, comment.URI) + } + assert.Equal(t, []string{first.URI, fourthURI}, wholeURIs, "removed and author-deleted comments must be absent from the actor profile") + assert.Nil(t, whole.Cursor) + + var cursor *string + var visited []string + for pageNumber := 0; pageNumber < 3; pageNumber++ { + page := list(1, cursor) + require.Len(t, page.Comments, 1, "each profile page must contain a visible comment") + uri := page.Comments[0].URI + assert.NotEqual(t, secondURI, uri) + assert.NotEqual(t, thirdURI, uri) + assert.NotContains(t, visited, uri, "a visible comment must not appear on two pages") + visited = append(visited, uri) + if page.Cursor == nil { + break + } + cursor = page.Cursor + } + assert.Equal(t, []string{first.URI, fourthURI}, visited, "cursor traversal must match the unpaged profile without gaps or duplicates") +} diff --git a/internal/core/comments/comment_moderation_view_test.go b/internal/core/comments/comment_moderation_view_test.go new file mode 100644 index 0000000..9263578 --- /dev/null +++ b/internal/core/comments/comment_moderation_view_test.go @@ -0,0 +1,161 @@ +package comments + +import ( + "context" + "encoding/json" + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const moderationViewInstanceDID = "did:web:coves.social" + +func moderationViewFixture(t *testing.T) (Service, *mockCommentRepo, *GetCommentsRequest, *Comment, *Comment) { + t.Helper() + postURI := "at://did:plc:postauthor/social.coves.community.postv2/3k7a3dmb5bk2c" + commentURI := "at://did:plc:commentauthor/social.coves.community.comment/3k7a3dmb5bk2d" + commentRepo := newMockCommentRepo() + postRepo := newMockPostRepo() + postRepo.posts[postURI] = createTestPost(postURI, "did:plc:postauthor", "did:plc:community") + userRepo := newMockUserRepo() + userRepo.users["did:plc:commentauthor"] = createTestUser("did:plc:commentauthor", "author.test") + comment := createTestComment(commentURI, "did:plc:commentauthor", "author.test", postURI, postURI, 1) + comment.Content = "secret moderation-view content" + comment.RKey = "3k7a3dmb5bk2d" + comment.UpvoteCount, comment.DownvoteCount, comment.Score = 9, 3, 6 + facets := `[{"index":{"byteStart":0,"byteEnd":6},"features":[{"$type":"app.bsky.richtext.facet#tag","tag":"privateFacetMarker"}]}]` + comment.ContentFacets = &facets + embed := `{"$type":"social.coves.embed.images","images":[{"image":{"$type":"blob","ref":{"$link":"privateImageMarker"},"mimeType":"image/png","size":10},"alt":"privateEmbedMarker"}]}` + comment.Embed = &embed + reply := createTestComment("at://did:plc:replyauthor/social.coves.community.comment/3k7a3dmb5bk2e", "did:plc:replyauthor", "reply.test", postURI, commentURI, 0) + reply.Content = "visible surviving reply" + commentRepo.listByParentWithHotRankFunc = func(_ context.Context, parentURI, _, _ string, _ int, _ *string, _ string) ([]*Comment, *string, error) { + if parentURI == postURI { + return []*Comment{comment}, nil, nil + } + return nil, nil, nil + } + commentRepo.listByParentsBatchFunc = func(_ context.Context, parentURIs []string, _ string, _ int) (map[string][]*Comment, error) { + return map[string][]*Comment{commentURI: {reply}}, nil + } + request := &GetCommentsRequest{PostURI: postURI, Sort: "new", Depth: 1, Limit: 10} + return NewCommentService(commentRepo, userRepo, postRepo, newMockCommunityRepo(), nil, nil, nil), commentRepo, request, comment, reply +} + +func TestCommentServiceGetCommentsOverlaysRemovalWithoutLeakingContent(t *testing.T) { + service, repo, request, comment, reply := moderationViewFixture(t) + repo.activeRemovalsByURI = map[string][]RemovalSource{ + comment.URI: {{AuthorityDID: moderationViewInstanceDID, ScopeKind: "instance"}}, + } + viewerDID := "did:plc:viewer" + request.ViewerDID = &viewerDID + response, err := service.GetComments(t.Context(), request) + require.NoError(t, err) + require.Len(t, response.Comments, 1) + node := response.Comments[0] + require.NotNil(t, node.Comment) + view := node.Comment + assert.True(t, view.IsDeleted) + assert.Equal(t, stringPointer(DeletionReasonModerator), view.DeletionReason) + assert.Equal(t, &ModerationView{ + State: "removed", + Sources: []ModerationSourceView{{AuthorityDID: moderationViewInstanceDID, Scope: ModerationScopeView{Kind: "instance"}}}, + }, view.Moderation) + assert.Nil(t, view.Record) + assert.Nil(t, view.Embed) + assert.Nil(t, view.Viewer) + assert.Nil(t, view.DeletedAt) + require.NotNil(t, view.Author) + assert.Equal(t, comment.CommenterDID, view.Author.DID) + assert.Equal(t, "handle.invalid", view.Author.Handle) + assert.Nil(t, view.Author.DisplayName) + assert.Nil(t, view.Author.Avatar) + require.NotNil(t, view.Stats) + assert.Zero(t, view.Stats.Upvotes) + assert.Zero(t, view.Stats.Downvotes) + assert.Zero(t, view.Stats.Score) + assert.Equal(t, 1, view.Stats.ReplyCount) + require.Len(t, node.Replies, 1) + assert.Equal(t, reply.URI, node.Replies[0].Comment.URI) + assert.Equal(t, reply.Content, node.Replies[0].Comment.Record.(*CommentRecord).Content) + assert.Nil(t, node.Replies[0].Comment.Moderation) + + serialized, err := json.Marshal(view) + require.NoError(t, err) + for _, secret := range []string{comment.Content, "privateFacetMarker", "privateImageMarker", "privateEmbedMarker"} { + assert.NotContains(t, string(serialized), secret) + } + assert.JSONEq(t, `{"state":"removed","sources":[{"authorityDid":"did:web:coves.social","scope":{"kind":"instance"}}]}`, mustMarshalModerationView(t, view.Moderation)) +} + +func TestCommentServiceGetCommentsPreservesAuthorDeletionAlongsideRemoval(t *testing.T) { + service, repo, request, comment, _ := moderationViewFixture(t) + deletedAt := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC) + comment.DeletedAt = &deletedAt + comment.DeletionReason = stringPointer(DeletionReasonAuthor) + repo.activeRemovalsByURI = map[string][]RemovalSource{comment.URI: {{AuthorityDID: moderationViewInstanceDID, ScopeKind: "instance"}}} + response, err := service.GetComments(t.Context(), request) + require.NoError(t, err) + require.Len(t, response.Comments, 1) + view := response.Comments[0].Comment + assert.True(t, view.IsDeleted) + assert.Equal(t, stringPointer(DeletionReasonAuthor), view.DeletionReason) + require.NotNil(t, view.Moderation) + assert.Equal(t, "removed", view.Moderation.State) + assert.Equal(t, []ModerationSourceView{{AuthorityDID: moderationViewInstanceDID, Scope: ModerationScopeView{Kind: "instance"}}}, view.Moderation.Sources) + assert.Nil(t, view.Record) +} + +func TestCommentServiceGetCommentsOverlaysRemovalOnNestedReply(t *testing.T) { + service, repo, request, comment, reply := moderationViewFixture(t) + repo.activeRemovalsByURI = map[string][]RemovalSource{ + reply.URI: {{AuthorityDID: moderationViewInstanceDID, ScopeKind: "instance"}}, + } + response, err := service.GetComments(t.Context(), request) + require.NoError(t, err) + require.Len(t, response.Comments, 1) + assert.Equal(t, comment.Content, response.Comments[0].Comment.Record.(*CommentRecord).Content) + assert.Nil(t, response.Comments[0].Comment.Moderation) + require.Len(t, response.Comments[0].Replies, 1) + nested := response.Comments[0].Replies[0].Comment + assert.True(t, nested.IsDeleted) + assert.Nil(t, nested.Record) + require.NotNil(t, nested.Moderation) + assert.Equal(t, "removed", nested.Moderation.State) +} + +func TestCommentServiceGetCommentsOmitsModerationForClearComment(t *testing.T) { + service, _, request, comment, _ := moderationViewFixture(t) + response, err := service.GetComments(t.Context(), request) + require.NoError(t, err) + require.Len(t, response.Comments, 1) + view := response.Comments[0].Comment + assert.False(t, view.IsDeleted) + assert.Nil(t, view.Moderation) + require.IsType(t, &CommentRecord{}, view.Record) + assert.Equal(t, comment.Content, view.Record.(*CommentRecord).Content) + encoded, err := json.Marshal(view) + require.NoError(t, err) + assert.NotContains(t, string(encoded), `"moderation"`) +} + +func TestCommentServiceGetCommentsFailsClosedOnRemovalLookupError(t *testing.T) { + service, repo, request, _, _ := moderationViewFixture(t) + lookupError := errors.New("removal lookup unavailable") + repo.activeRemovalsErr = lookupError + response, err := service.GetComments(t.Context(), request) + require.ErrorIs(t, err, lookupError) + assert.Nil(t, response, "unchecked content must never be returned") +} + +func stringPointer(value string) *string { return &value } + +func mustMarshalModerationView(t *testing.T, view *ModerationView) string { + t.Helper() + encoded, err := json.Marshal(view) + require.NoError(t, err) + return string(encoded) +} diff --git a/internal/core/comments/comment_service.go b/internal/core/comments/comment_service.go index 4c564de..f2947d4 100644 --- a/internal/core/comments/comment_service.go +++ b/internal/core/comments/comment_service.go @@ -366,6 +366,14 @@ func (s *commentService) buildThreadViews( if len(comments) == 0 { return result, nil } + commentURIs := make([]string, 0, len(comments)) + for _, comment := range comments { + commentURIs = append(commentURIs, comment.URI) + } + removals, err := s.commentRepo.ActiveRemovalsByURIs(ctx, commentURIs) + if err != nil { + return nil, fmt.Errorf("checking active comment removals: %w", err) + } // Batch fetch vote states for all comments at this level (Phase 2B) var voteStates map[string]interface{} @@ -419,11 +427,27 @@ func (s *commentService) buildThreadViews( for _, comment := range comments { var commentView *CommentView + sources := removals[comment.URI] // Build appropriate view based on deletion status - if comment.DeletedAt != nil { + if comment.DeletedAt != nil || len(sources) > 0 { // Deleted comment - build placeholder view to preserve thread structure commentView = s.buildDeletedCommentView(comment) + if len(sources) > 0 { + if comment.DeletedAt == nil || comment.DeletionReason == nil || *comment.DeletionReason != DeletionReasonAuthor { + reason := DeletionReasonModerator + commentView.DeletionReason = &reason + commentView.DeletedAt = nil + } + viewSources := make([]ModerationSourceView, 0, len(sources)) + for _, source := range sources { + viewSources = append(viewSources, ModerationSourceView{ + AuthorityDID: source.AuthorityDID, + Scope: ModerationScopeView{Kind: source.ScopeKind}, + }) + } + commentView.Moderation = &ModerationView{State: "removed", Sources: viewSources} + } } else { // Active comment - build full view with author info and stats commentView = s.buildCommentView(comment, viewerDID, voteStates, usersByDID) diff --git a/internal/core/comments/comment_service_test.go b/internal/core/comments/comment_service_test.go index 4e498d6..210d6bf 100644 --- a/internal/core/comments/comment_service_test.go +++ b/internal/core/comments/comment_service_test.go @@ -21,6 +21,8 @@ import ( // mockCommentRepo is a mock implementation of the comment Repository interface type mockCommentRepo struct { comments map[string]*Comment + activeRemovalsByURI map[string][]RemovalSource + activeRemovalsErr error listByParentWithHotRankFunc func(ctx context.Context, parentURI, sort, timeframe string, limit int, cursor *string, viewerDID string) ([]*Comment, *string, error) listByParentsBatchFunc func(ctx context.Context, parentURIs []string, sort string, limitPerParent int) (map[string][]*Comment, error) getVoteStateForCommentsFunc func(ctx context.Context, viewerDID string, commentURIs []string) (map[string]interface{}, error) @@ -159,6 +161,19 @@ func (m *mockCommentRepo) GetByURIsBatch(ctx context.Context, uris []string) (ma return result, nil } +func (m *mockCommentRepo) ActiveRemovalsByURIs(ctx context.Context, uris []string) (map[string][]RemovalSource, error) { + if m.activeRemovalsErr != nil { + return nil, m.activeRemovalsErr + } + result := make(map[string][]RemovalSource) + for _, uri := range uris { + if sources := m.activeRemovalsByURI[uri]; len(sources) > 0 { + result[uri] = sources + } + } + return result, nil +} + func (m *mockCommentRepo) GetVoteStateForComments(ctx context.Context, viewerDID string, commentURIs []string) (map[string]interface{}, error) { if m.getVoteStateForCommentsFunc != nil { return m.getVoteStateForCommentsFunc(ctx, viewerDID, commentURIs) diff --git a/internal/core/comments/interfaces.go b/internal/core/comments/interfaces.go index 2b86546..794e53d 100644 --- a/internal/core/comments/interfaces.go +++ b/internal/core/comments/interfaces.go @@ -87,6 +87,10 @@ type Repository interface { // Used for hydrating comment threads without N+1 queries GetByURIsBatch(ctx context.Context, uris []string) (map[string]*Comment, error) + // ActiveRemovalsByURIs returns the active moderation removal sources of + // each URI that has at least one. + ActiveRemovalsByURIs(ctx context.Context, uris []string) (map[string][]RemovalSource, error) + // GetVoteStateForComments retrieves the viewer's votes on a batch of comments // Returns map[commentURI]*Vote for efficient lookups // Future: Used when votes table is implemented @@ -117,3 +121,9 @@ type RepositoryTx interface { // deletedByDID: DID of the actor who performed the deletion SoftDeleteWithReasonTx(ctx context.Context, tx *sql.Tx, uri, reason, deletedByDID string) (int64, error) } + +// RemovalSource is the authority and scope of an active removal decision. +type RemovalSource struct { + AuthorityDID string + ScopeKind string +} diff --git a/internal/core/comments/view_models.go b/internal/core/comments/view_models.go index 8a509e1..b02e4f9 100644 --- a/internal/core/comments/view_models.go +++ b/internal/core/comments/view_models.go @@ -23,6 +23,7 @@ type CommentView struct { IsDeleted bool `json:"isDeleted,omitempty"` DeletionReason *string `json:"deletionReason,omitempty"` DeletedAt *string `json:"deletedAt,omitempty"` + Moderation *ModerationView `json:"moderation,omitempty"` } // ThreadViewComment represents a comment with its nested replies @@ -82,3 +83,21 @@ type GetActorCommentsResponse struct { Comments []*CommentView `json:"comments"` Cursor *string `json:"cursor,omitempty"` } + +// ModerationView is a comment's public removal state +// (social.coves.moderation.defs#moderationView). +type ModerationView struct { + State string `json:"state"` + Sources []ModerationSourceView `json:"sources,omitempty"` +} + +// ModerationSourceView attributes a removal (social.coves.moderation.defs#sourceView). +type ModerationSourceView struct { + AuthorityDID string `json:"authorityDid"` + Scope ModerationScopeView `json:"scope"` +} + +// ModerationScopeView is a removal's scope (social.coves.moderation.defs#scopeView). +type ModerationScopeView struct { + Kind string `json:"kind"` +} diff --git a/internal/core/embeds/comment_image_cids_test.go b/internal/core/embeds/comment_image_cids_test.go new file mode 100644 index 0000000..5eda7fa --- /dev/null +++ b/internal/core/embeds/comment_image_cids_test.go @@ -0,0 +1,88 @@ +package embeds + +import ( + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multibase" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const secondTestCID = "bafyreigj3fwnwjuzr35k2kuzmb5dixxczrzjhqkr5srlqplsh6gq3bj3si" + +func imageEntry(image interface{}) map[string]interface{} { + return map[string]interface{}{"alt": "an image", "image": image} +} + +func imagesEmbed(entries ...interface{}) map[string]interface{} { + return map[string]interface{}{"$type": TypeImages, "images": entries} +} + +func TestCommentImageCIDs(t *testing.T) { + parsed, err := cid.Decode(testCID) + require.NoError(t, err) + require.Equal(t, testCID, parsed.String(), "the fixture must already be canonical") + base58Alias, err := parsed.StringOfBase(multibase.Base58BTC) + require.NoError(t, err) + const cidV0 = "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG" + + for _, test := range []struct { + name string + embed map[string]interface{} + want []string + }{ + {name: "nil embed", embed: nil}, + {name: "ref link blobs in order", embed: imagesEmbed(imageEntry(blobRef(testCID)), imageEntry(blobRef(secondTestCID))), want: []string{testCID, secondTestCID}}, + {name: "legacy top-level cid blob", embed: imagesEmbed(imageEntry(map[string]interface{}{"cid": testCID, "mimeType": "image/png"})), want: []string{testCID}}, + {name: "non-canonical encoding is canonicalized", embed: imagesEmbed(imageEntry(blobRef(base58Alias))), want: []string{testCID}}, + {name: "CIDv0 keeps its canonical form", embed: imagesEmbed(imageEntry(blobRef(cidV0))), want: []string{cidV0}}, + {name: "duplicates across encodings collapse", embed: imagesEmbed(imageEntry(blobRef(testCID)), imageEntry(map[string]interface{}{"cid": base58Alias}), imageEntry(blobRef(testCID))), want: []string{testCID}}, + {name: "view type is not a record", embed: map[string]interface{}{"$type": TypeImages + viewSuffix, "images": []interface{}{imageEntry(blobRef(testCID))}}}, + {name: "external embed is outside the comment union", embed: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{"thumb": blobRef(testCID)}}}, + {name: "video embed is outside the comment union", embed: map[string]interface{}{"$type": TypeVideo, "video": blobRef(testCID), "thumbnail": blobRef(secondTestCID)}}, + {name: "missing type", embed: map[string]interface{}{"images": []interface{}{imageEntry(blobRef(testCID))}}}, + {name: "non-string type", embed: map[string]interface{}{"$type": 5, "images": []interface{}{imageEntry(blobRef(testCID))}}}, + {name: "images is a string", embed: map[string]interface{}{"$type": TypeImages, "images": "x"}}, + {name: "images is an object", embed: map[string]interface{}{"$type": TypeImages, "images": map[string]interface{}{"image": blobRef(testCID)}}}, + {name: "empty images", embed: imagesEmbed()}, + { + name: "malformed entries are skipped around a valid one", + embed: imagesEmbed( + "x", 7, nil, + imageEntry("x"), + imageEntry(map[string]interface{}{"ref": testCID}), + imageEntry(map[string]interface{}{"ref": map[string]interface{}{"$link": 5}}), + imageEntry(map[string]interface{}{"ref": map[string]interface{}{}, "cid": testCID}), + imageEntry(blobRef("bafynotacid")), + imageEntry(blobRef("")), + imageEntry(blobRef(secondTestCID)), + ), + want: []string{secondTestCID}, + }, + } { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, CommentImageCIDs(test.embed)) + }) + } +} + +// Every CID the enumerator returns is one HydrateCommentView turns into a +// proxy URL for the same embed, so blocking covers what serving emits. +func TestCommentImageCIDsMatchServedProxyURLs(t *testing.T) { + withProxy(t, "https://img.coves.social") + embed := imagesEmbed(imageEntry(blobRef(testCID)), imageEntry(map[string]interface{}{"cid": secondTestCID})) + cids := CommentImageCIDs(embed) + require.Equal(t, []string{testCID, secondTestCID}, cids) + + HydrateCommentView(embed, testDID, testPDS) + images, ok := embed["images"].([]interface{}) + require.True(t, ok) + require.Len(t, images, len(cids)) + for index, entry := range images { + image, isObject := entry.(map[string]interface{}) + require.True(t, isObject) + assert.Contains(t, image["thumb"], "/"+cids[index]) + assert.Contains(t, image["fullsize"], "/"+cids[index]) + } +} diff --git a/internal/core/embeds/view.go b/internal/core/embeds/view.go index 1d4628e..9f9412d 100644 --- a/internal/core/embeds/view.go +++ b/internal/core/embeds/view.go @@ -28,6 +28,8 @@ package embeds import ( "log/slog" + "github.com/ipfs/go-cid" + "Coves/internal/core/blobs" ) @@ -194,6 +196,42 @@ func HydrateCommentView(embed map[string]interface{}, ownerDID, ownerPDSURL stri HydrateView(embed, ownerDID, ownerPDSURL) } +// CommentImageCIDs returns the canonical CIDs of every image blob that +// HydrateCommentView can serve through the image proxy for a comment embed. +// Moderation blocks exactly these, so blocking and serving cannot drift apart. +// +// The embed is author-controlled and unvalidated, so this never fails: only a +// social.coves.embed.images embed contributes, and an entry that is not an +// object, carries no blob CID in either encoding blobCID accepts, or whose CID +// does not decode is skipped. Each CID is decoded and re-encoded in its +// canonical string form, the form the proxy route normalizes requests to, and +// the result is de-duplicated in first-seen order. +func CommentImageCIDs(embed map[string]interface{}) []string { + if embedType, _ := embed["$type"].(string); embedType != TypeImages { + return nil + } + images, _ := embed["images"].([]interface{}) + var cids []string + seen := make(map[string]bool, len(images)) + for _, entry := range images { + image, isObject := entry.(map[string]interface{}) + if !isObject { + continue + } + parsed, err := cid.Decode(blobCID(image["image"])) + if err != nil { + continue + } + canonical := parsed.String() + if seen[canonical] { + continue + } + seen[canonical] = true + cids = append(cids, canonical) + } + return cids +} + // projectExternal computes the URL-bearing fields of // social.coves.embed.external#viewExternal: the link card thumbnail, plus the // gallery preview images an image-hosting provider can contribute. diff --git a/internal/core/imageproxy/cache.go b/internal/core/imageproxy/cache.go index 89365c3..77859b3 100644 --- a/internal/core/imageproxy/cache.go +++ b/internal/core/imageproxy/cache.go @@ -36,6 +36,10 @@ type Cache interface { // Cleanup runs both LRU eviction and TTL cleanup. // Returns the number of entries removed and any error. Cleanup() (int, error) + // DeleteOwner removes every preset's cached copy of the owner's blob. + DeleteOwner(did, cid string) error + // DeleteCID removes every owner's cached copies of the blob in every preset. + DeleteCID(cid string) error } // DiskCache implements Cache using the filesystem for storage. @@ -202,14 +206,19 @@ func (c *DiskCache) Delete(preset, did, cid string) error { return err } - path := c.cachePath(preset, did, cid) + return removeEntry(c.cachePath(preset, did, cid)) +} - err := os.Remove(path) - if err != nil && !os.IsNotExist(err) { - return err +// removeEntry deletes a cached file and the temporary file an interrupted Set +// may have left beside it. A missing file is not an error. +func removeEntry(path string) error { + var errs []error + for _, candidate := range []string{path, path + ".tmp"} { + if err := os.Remove(candidate); err != nil && !errors.Is(err, fs.ErrNotExist) { + errs = append(errs, err) + } } - - return nil + return errors.Join(errs...) } // cacheEntry represents a cached file with its metadata. @@ -537,3 +546,63 @@ func (c *DiskCache) StartCleanupJob(interval time.Duration) context.CancelFunc { return cancel } + +// DeleteOwner removes every preset's cached copy of the owner's blob. It +// keeps going past a failed preset and returns every failure joined, so one +// bad entry cannot leave the owner's other copies on disk. +func (c *DiskCache) DeleteOwner(did, cid string) error { + if did == "" || cid == "" { + return ErrEmptyParameter + } + presets, err := os.ReadDir(c.basePath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var errs []error + for _, preset := range presets { + if preset.IsDir() { + errs = append(errs, c.Delete(preset.Name(), did, cid)) + } + } + return errors.Join(errs...) +} + +// DeleteCID removes every owner's cached copies of the blob in every preset. +// Like DeleteOwner it keeps going past failures and joins them. A preset +// directory that vanishes mid-walk (the cleanup job removes empty ones) is +// skipped, because nothing in it can still hold the blob. +func (c *DiskCache) DeleteCID(cid string) error { + if cid == "" { + return ErrEmptyParameter + } + presets, err := os.ReadDir(c.basePath) + if errors.Is(err, fs.ErrNotExist) { + return nil + } + if err != nil { + return err + } + var errs []error + for _, preset := range presets { + if !preset.IsDir() { + continue + } + owners, err := os.ReadDir(filepath.Join(c.basePath, preset.Name())) + if errors.Is(err, fs.ErrNotExist) { + continue + } + if err != nil { + errs = append(errs, err) + continue + } + for _, owner := range owners { + if owner.IsDir() { + errs = append(errs, c.Delete(preset.Name(), owner.Name(), cid)) + } + } + } + return errors.Join(errs...) +} diff --git a/internal/core/imageproxy/cache_moderation_test.go b/internal/core/imageproxy/cache_moderation_test.go new file mode 100644 index 0000000..c7c5cae --- /dev/null +++ b/internal/core/imageproxy/cache_moderation_test.go @@ -0,0 +1,120 @@ +package imageproxy + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestDiskCache_PurgeOwnerAndCIDAcrossPresets(t *testing.T) { + cache, err := NewDiskCache(t.TempDir(), 1, 0) + require.NoError(t, err) + const ( + ownerA = "did:plc:cacheownera" + ownerB = "did:plc:cacheownerb" + cid = "bafyreimoderatedimage" + otherCID = "bafyreiunrelatedimage" + ) + for _, entry := range []struct{ preset, did, cid string }{ + {"avatar", ownerA, cid}, {"banner", ownerA, cid}, + {"avatar", ownerB, cid}, {"banner", ownerB, cid}, + {"avatar", ownerA, otherCID}, + } { + require.NoError(t, cache.Set(entry.preset, entry.did, entry.cid, []byte("image"))) + } + assertEntry := func(preset, did, cid string, want bool) { + t.Helper() + _, found, err := cache.Get(preset, did, cid) + require.NoError(t, err) + assert.Equal(t, want, found, "%s %s %s", preset, did, cid) + } + + require.NoError(t, cache.DeleteOwner(ownerA, cid)) + assertEntry("avatar", ownerA, cid, false) + assertEntry("banner", ownerA, cid, false) + assertEntry("avatar", ownerB, cid, true) + assertEntry("banner", ownerB, cid, true) + assertEntry("avatar", ownerA, otherCID, true) + require.NoError(t, cache.DeleteOwner(ownerA, cid), "deletion is idempotent") + + require.NoError(t, cache.Set("banner", ownerA, cid, []byte("reinserted"))) + require.NoError(t, cache.DeleteCID(cid)) + assertEntry("banner", ownerA, cid, false) + assertEntry("avatar", ownerB, cid, false) + assertEntry("banner", ownerB, cid, false) + assertEntry("avatar", ownerA, otherCID, true) + require.NoError(t, cache.DeleteCID(cid), "deletion is idempotent") +} + +// blockPurgeTarget makes the cache entry path a non-empty directory, which +// os.Remove cannot delete, so a purge sees a real filesystem error there. +func blockPurgeTarget(t *testing.T, cache *DiskCache, preset, did, cid string) { + t.Helper() + path := cache.cachePath(preset, did, cid) + require.NoError(t, os.MkdirAll(filepath.Join(path, "occupied"), 0o755)) +} + +func TestDiskCache_DeleteCIDContinuesPastFailedEntries(t *testing.T) { + cache, err := NewDiskCache(t.TempDir(), 1, 0) + require.NoError(t, err) + const ( + ownerA = "did:plc:cacheownera" + ownerB = "did:plc:cacheownerb" + cid = "bafyreimoderatedimage" + ) + // ReadDir sorts entries, so the failing entry is visited first. + blockPurgeTarget(t, cache, "avatar", ownerA, cid) + require.NoError(t, cache.Set("avatar", ownerB, cid, []byte("image"))) + require.NoError(t, cache.Set("banner", ownerA, cid, []byte("image"))) + + err = cache.DeleteCID(cid) + require.Error(t, err, "the entry that could not be removed must be reported") + for _, entry := range []struct{ preset, did string }{{"avatar", ownerB}, {"banner", ownerA}} { + _, found, getErr := cache.Get(entry.preset, entry.did, cid) + require.NoError(t, getErr) + assert.False(t, found, "%s %s must still be purged after an earlier failure", entry.preset, entry.did) + } +} + +func TestDiskCache_DeleteOwnerContinuesPastFailedEntries(t *testing.T) { + cache, err := NewDiskCache(t.TempDir(), 1, 0) + require.NoError(t, err) + const ( + owner = "did:plc:cacheownera" + cid = "bafyreimoderatedimage" + ) + blockPurgeTarget(t, cache, "avatar", owner, cid) + require.NoError(t, cache.Set("banner", owner, cid, []byte("image"))) + + err = cache.DeleteOwner(owner, cid) + require.Error(t, err, "the entry that could not be removed must be reported") + _, found, err := cache.Get("banner", owner, cid) + require.NoError(t, err) + assert.False(t, found, "later presets must still be purged after an earlier failure") +} + +func TestDiskCache_PurgeRemovesInterruptedTemporaryFiles(t *testing.T) { + const ( + owner = "did:plc:cacheownera" + cid = "bafyreimoderatedimage" + ) + for name, purge := range map[string]func(*DiskCache) error{ + "owner": func(cache *DiskCache) error { return cache.DeleteOwner(owner, cid) }, + "cid": func(cache *DiskCache) error { return cache.DeleteCID(cid) }, + } { + t.Run(name, func(t *testing.T) { + cache, err := NewDiskCache(t.TempDir(), 1, 0) + require.NoError(t, err) + require.NoError(t, cache.Set("avatar", owner, cid, []byte("image"))) + temporary := cache.cachePath("avatar", owner, cid) + ".tmp" + require.NoError(t, os.WriteFile(temporary, []byte("partial image"), 0o644)) + + require.NoError(t, purge(cache)) + _, statErr := os.Stat(temporary) + assert.True(t, os.IsNotExist(statErr), "an interrupted write's bytes must be purged too") + }) + } +} diff --git a/internal/core/imageproxy/errors.go b/internal/core/imageproxy/errors.go index 1cc2f7a..d0b6d49 100644 --- a/internal/core/imageproxy/errors.go +++ b/internal/core/imageproxy/errors.go @@ -67,3 +67,9 @@ var ( // ErrNilDependency is returned when a required dependency is nil. ErrNilDependency = errors.New("required dependency is nil") ) + +// ErrBlobBlocked is returned when moderation blocks serving a blob. +var ErrBlobBlocked = errors.New("blob blocked") + +// ErrBlockCheckFailed is returned when the moderation block check fails. +var ErrBlockCheckFailed = errors.New("blob block check failed") diff --git a/internal/core/imageproxy/fetcher_guard_test.go b/internal/core/imageproxy/fetcher_guard_test.go index 7551ad1..4f182b9 100644 --- a/internal/core/imageproxy/fetcher_guard_test.go +++ b/internal/core/imageproxy/fetcher_guard_test.go @@ -540,7 +540,7 @@ func TestImageProxyService_GetImage_RefusesAPrivateAddressOnAColdCache(t *testin "ever calling the fetcher and this whole case would pass without exercising the guard", preset, did, cid) - service, err := NewService(cache, newTestProcessor(t), NewPDSFetcher(5*time.Second, 10), Config{ + service, err := NewService(cache, newTestProcessor(t), NewPDSFetcher(5*time.Second, 10), allowAllBlockChecker(), Config{ Enabled: true, CachePath: t.TempDir(), CacheMaxGB: 1, diff --git a/internal/core/imageproxy/moderation_test.go b/internal/core/imageproxy/moderation_test.go new file mode 100644 index 0000000..042f418 --- /dev/null +++ b/internal/core/imageproxy/moderation_test.go @@ -0,0 +1,481 @@ +package imageproxy + +import ( + "context" + "errors" + "sync" + "sync/atomic" + "testing" + "time" + + "Coves/tests/testkit" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + moderationTestOwner = "did:plc:moderationimageowner" + moderationTestCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" +) + +func TestImageProxyService_BlocksBeforeReadingCache(t *testing.T) { + for _, warm := range []bool{false, true} { + t.Run(map[bool]string{false: "cold", true: "warm"}[warm], func(t *testing.T) { + cache := NewMockCache() + if warm { + cache.SetCacheData("avatar", moderationTestOwner, moderationTestCID, []byte("cached secret")) + } + fetcher := NewMockFetcher([]byte("fetched secret"), nil) + var checks atomic.Int32 + checker := blockCheckFunc(func(_ context.Context, did, cid string) (bool, error) { + assert.Equal(t, moderationTestOwner, did) + assert.Equal(t, moderationTestCID, cid) + checks.Add(1) + return true, nil + }) + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), fetcher, checker, DefaultConfig()) + require.NoError(t, err) + data, err := service.GetImage(t.Context(), "avatar", moderationTestOwner, moderationTestCID, "https://pds.example.com") + assert.ErrorIs(t, err, ErrBlobBlocked) + assert.Empty(t, data) + assert.EqualValues(t, 1, checks.Load()) + assert.Zero(t, cache.GetCalls(), "even a warm entry must not be read") + assert.Zero(t, fetcher.Calls()) + }) + } +} + +func TestImageProxyService_BlockCheckFailureFailsClosed(t *testing.T) { + cache := NewMockCache() + cache.SetCacheData("avatar", moderationTestOwner, moderationTestCID, []byte("cached secret")) + fetcher := NewMockFetcher([]byte("fetched secret"), nil) + lookupError := errors.New("block lookup unavailable") + checker := blockCheckFunc(func(context.Context, string, string) (bool, error) { return false, lookupError }) + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), fetcher, checker, DefaultConfig()) + require.NoError(t, err) + data, err := service.GetImage(t.Context(), "avatar", moderationTestOwner, moderationTestCID, "https://pds.example.com") + assert.ErrorIs(t, err, ErrBlockCheckFailed) + assert.Empty(t, data) + assert.Zero(t, cache.GetCalls()) + assert.Zero(t, fetcher.Calls()) +} + +func TestImageProxyService_BlockArrivesWhileFetching(t *testing.T) { + cache := NewMockCache() + fetcher := newBlockingFetcher([]byte("source image")) + t.Cleanup(fetcher.Release) + var blocked atomic.Bool + var checks atomic.Int32 + checker := blockCheckFunc(func(context.Context, string, string) (bool, error) { + checks.Add(1) + return blocked.Load(), nil + }) + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), fetcher, checker, DefaultConfig()) + require.NoError(t, err) + done := callGetImageAsync(t.Context(), service, moderationTestCID) + fetcher.waitForEntries(t, 1, 5*time.Second) + require.EqualValues(t, 1, checks.Load(), "pre-read check must finish before fetching") + blocked.Store(true) + fetcher.Release() + select { + case result := <-done: + assert.ErrorIs(t, result.err, ErrBlobBlocked) + assert.Empty(t, result.data) + case <-time.After(5 * time.Second): + t.Fatal("GetImage did not finish after the fetch was released") + } + assert.GreaterOrEqual(t, checks.Load(), int32(2), "the post-fetch check must see the new block") + assert.Zero(t, cache.SetCalls(), "a blocked in-flight image must never be cached") +} + +func TestImageProxyService_PurgeSerializesWithCachePublication(t *testing.T) { + cache := NewMockCache() + var checks atomic.Int32 + checkingAfterFetch := make(chan struct{}) + releaseCheck := make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(releaseCheck) }) } + defer release() + checker := blockCheckFunc(func(context.Context, string, string) (bool, error) { + if checks.Add(1) == 2 { + close(checkingAfterFetch) + <-releaseCheck + } + return false, nil + }) + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), NewMockFetcher([]byte("raw"), nil), checker, DefaultConfig()) + require.NoError(t, err) + imageDone := callGetImageAsync(t.Context(), service, moderationTestCID) + select { + case <-checkingAfterFetch: + case <-time.After(5 * time.Second): + t.Fatal("writer never reached the post-fetch block check") + } + purgeStarted := make(chan struct{}) + purgeDone := make(chan error, 1) + go func() { + close(purgeStarted) + purgeDone <- service.PurgeOwnerBlob("did:plc:test123", moderationTestCID) + }() + <-purgeStarted + testkit.Holds(t, 200*time.Millisecond, func() (bool, error) { + select { + case <-purgeDone: + return false, nil + default: + return true, nil + } + }, testkit.WithDescription("purge waiting for the in-flight cache publication")) + release() + select { + case result := <-imageDone: + require.NoError(t, result.err) + assert.Equal(t, []byte("processed"), result.data) + case <-time.After(5 * time.Second): + t.Fatal("writer did not finish after releasing the block check") + } + select { + case err := <-purgeDone: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("purge did not finish after the writer") + } + assert.Equal(t, 1, cache.SetCalls(), "the writer must have cached bytes before purge completed") + data, found, err := cache.Get("avatar", "did:plc:test123", moderationTestCID) + require.NoError(t, err) + assert.False(t, found, "purge must remove the writer's cache entry across the race") + assert.Empty(t, data) +} + +func TestImageProxyService_PurgeBlobClearsAllOwners(t *testing.T) { + cache := NewMockCache() + cache.SetCacheData("avatar", moderationTestOwner, moderationTestCID, []byte("first")) + cache.SetCacheData("banner", "did:plc:otherowner", moderationTestCID, []byte("second")) + cache.SetCacheData("avatar", moderationTestOwner, "bafyreiunrelatedimage", []byte("unrelated")) + service, err := NewService(cache, NewMockProcessor(nil, nil), NewMockFetcher(nil, nil), allowAllBlockChecker(), DefaultConfig()) + require.NoError(t, err) + require.NoError(t, service.PurgeBlob(moderationTestCID)) + for _, entry := range []struct{ preset, did string }{{"avatar", moderationTestOwner}, {"banner", "did:plc:otherowner"}} { + _, found, err := cache.Get(entry.preset, entry.did, moderationTestCID) + require.NoError(t, err) + assert.False(t, found) + } + _, found, err := cache.Get("avatar", moderationTestOwner, "bafyreiunrelatedimage") + require.NoError(t, err) + assert.True(t, found) +} + +func TestImageProxyService_IsBlobBlockedUsesChecker(t *testing.T) { + checker := blockCheckFunc(func(_ context.Context, did, cid string) (bool, error) { + assert.Equal(t, moderationTestOwner, did) + assert.Equal(t, moderationTestCID, cid) + return true, nil + }) + service, err := NewService(NewMockCache(), NewMockProcessor(nil, nil), NewMockFetcher(nil, nil), checker, DefaultConfig()) + require.NoError(t, err) + blocked, err := service.IsBlobBlocked(t.Context(), moderationTestOwner, moderationTestCID) + require.NoError(t, err) + assert.True(t, blocked) +} + +// cidOnOtherStripe returns a CID whose publication lock differs from cid's, +// so a test can show that work on one stripe does not stall the other. +func cidOnOtherStripe(t *testing.T, service *ImageProxyService, cid string) string { + t.Helper() + for _, suffix := range "abcdefghijklmnopqrstuvwxyz" { + candidate := cid[:len(cid)-1] + string(suffix) + if service.publicationLock(candidate) != service.publicationLock(cid) { + return candidate + } + } + t.Fatal("no candidate CID landed on another publication stripe") + return "" +} + +// parkingPurgeCache parks DeleteCID until released, standing in for a purge +// whose directory walk is slow. +type parkingPurgeCache struct { + *MockCache + entered chan struct{} + release chan struct{} +} + +func (c *parkingPurgeCache) DeleteCID(cid string) error { + close(c.entered) + <-c.release + return c.MockCache.DeleteCID(cid) +} + +func receiveImage(t *testing.T, done <-chan getImageResult, what string) getImageResult { + t.Helper() + select { + case result := <-done: + return result + case <-time.After(5 * time.Second): + t.Fatalf("%s did not finish", what) + return getImageResult{} + } +} + +func TestImageProxyService_BlockedRequestPurgesItsCacheEntry(t *testing.T) { + cache := NewMockCache() + cache.SetCacheData("avatar", moderationTestOwner, moderationTestCID, []byte("cached secret")) + cache.SetCacheData("avatar", "did:plc:otherowner", moderationTestCID, []byte("other owner")) + checker := blockCheckFunc(func(_ context.Context, did, _ string) (bool, error) { + return did == moderationTestOwner, nil + }) + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), NewMockFetcher(nil, nil), checker, DefaultConfig()) + require.NoError(t, err) + + _, err = service.GetImage(t.Context(), "avatar", moderationTestOwner, moderationTestCID, "https://pds.example.com") + require.ErrorIs(t, err, ErrBlobBlocked) + assert.Zero(t, cache.GetCalls(), "the blocked entry must never be read") + _, found, err := cache.Get("avatar", moderationTestOwner, moderationTestCID) + require.NoError(t, err) + assert.False(t, found, "a blocked request must remove the cached bytes a lost purge left behind") + _, found, err = cache.Get("avatar", "did:plc:otherowner", moderationTestCID) + require.NoError(t, err) + assert.True(t, found, "another owner's unblocked entry must stay") +} + +func TestImageProxyService_PurgeDiskWorkDoesNotHoldPublicationLock(t *testing.T) { + cache := &parkingPurgeCache{MockCache: NewMockCache(), entered: make(chan struct{}), release: make(chan struct{})} + var releaseOnce sync.Once + releasePurge := func() { releaseOnce.Do(func() { close(cache.release) }) } + t.Cleanup(releasePurge) + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), NewMockFetcher([]byte("raw"), nil), allowAllBlockChecker(), DefaultConfig()) + require.NoError(t, err) + + purgeDone := make(chan error, 1) + go func() { purgeDone <- service.PurgeBlob(moderationTestCID) }() + select { + case <-cache.entered: + case <-time.After(5 * time.Second): + t.Fatal("purge never reached the cache walk") + } + result := receiveImage(t, callGetImageAsync(t.Context(), service, moderationTestCID), + "cache publication on the purged CID's stripe while the purge walks the cache") + require.NoError(t, result.err) + releasePurge() + select { + case err := <-purgeDone: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("purge did not finish after release") + } +} + +func TestImageProxyService_ProcessingSlotFreeDuringPublication(t *testing.T) { + parkedCheck := make(chan struct{}) + releaseCheck := make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(releaseCheck) }) } + t.Cleanup(release) + var parkedCID string + var checksForParked atomic.Int32 + checker := blockCheckFunc(func(_ context.Context, _, cid string) (bool, error) { + if cid == parkedCID && checksForParked.Add(1) == 2 { + close(parkedCheck) + <-releaseCheck + } + return false, nil + }) + cache := NewMockCache() + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), NewMockFetcher([]byte("raw"), nil), checker, semaphoreTestConfig(1)) + require.NoError(t, err) + parkedCID = moderationTestCID + otherCID := cidOnOtherStripe(t, service, parkedCID) + + parkedDone := callGetImageAsync(t.Context(), service, parkedCID) + select { + case <-parkedCheck: + case <-time.After(5 * time.Second): + t.Fatal("first request never reached its post-fetch block check") + } + result := receiveImage(t, callGetImageAsync(t.Context(), service, otherCID), "request on another stripe") + require.NoError(t, result.err, "a request parked in publication must not hold the only processing slot") + release() + require.NoError(t, receiveImage(t, parkedDone, "parked request").err) +} + +// TestImageProxyService_AdmissionHeldDuringPublication: a processed output +// waiting for publication is transient memory the in-flight cap must still +// count, or a slow recheck lets outputs pile up beyond the documented bound. +func TestImageProxyService_AdmissionHeldDuringPublication(t *testing.T) { + parkedCheck := make(chan struct{}) + releaseCheck := make(chan struct{}) + var releaseOnce sync.Once + release := func() { releaseOnce.Do(func() { close(releaseCheck) }) } + t.Cleanup(release) + var checksForParked atomic.Int32 + checker := blockCheckFunc(func(_ context.Context, _, cid string) (bool, error) { + if cid == moderationTestCID && checksForParked.Add(1) == 2 { + close(parkedCheck) + <-releaseCheck + } + return false, nil + }) + fetcher := NewMockFetcher([]byte("raw"), nil) + service, err := NewService(NewMockCache(), NewMockProcessor([]byte("processed"), nil), fetcher, checker, admissionTestConfig(1)) + require.NoError(t, err) + otherCID := cidOnOtherStripe(t, service, moderationTestCID) + + parkedDone := callGetImageAsync(t.Context(), service, moderationTestCID) + select { + case <-parkedCheck: + case <-time.After(5 * time.Second): + t.Fatal("first request never reached its post-fetch block check") + } + refusalsBefore := ProcessorBusyRefusalCount() + result := receiveImage(t, callGetImageAsync(t.Context(), service, otherCID), "request over the in-flight cap") + require.ErrorIs(t, result.err, ErrProcessorBusy, "a request parked in publication must still hold its admission slot") + assert.Equal(t, int64(1), ProcessorBusyRefusalCount()-refusalsBefore) + assert.Equal(t, 1, fetcher.Calls(), "the refused request must never fetch") + + release() + require.NoError(t, receiveImage(t, parkedDone, "parked request").err) + result = receiveImage(t, callGetImageAsync(t.Context(), service, otherCID), "request after publication finished") + require.NoError(t, result.err, "publication must hand the admission slot back when it finishes") +} + +func TestImageProxyService_CallerDisconnectAfterProcessingStillCaches(t *testing.T) { + cache := NewMockCache() + processor := newBlockingProcessor([]byte("processed")) + t.Cleanup(processor.Release) + checker := blockCheckFunc(func(ctx context.Context, _, _ string) (bool, error) { + return false, ctx.Err() + }) + service, err := NewService(cache, processor, NewMockFetcher([]byte("raw"), nil), checker, DefaultConfig()) + require.NoError(t, err) + ctx, cancel := context.WithCancel(t.Context()) + done := callGetImageAsync(ctx, service, moderationTestCID) + processor.waitForEntries(t, 1, 5*time.Second) + cancel() + processor.Release() + receiveImage(t, done, "request whose caller disconnected during processing") + data, found := cache.GetSetData("avatar", "did:plc:test123", moderationTestCID) + assert.True(t, found, "processed bytes must be cached even though the caller went away") + assert.Equal(t, []byte("processed"), data) +} + +func TestImageProxyService_PurgeReturnsWhileFetchIsParked(t *testing.T) { + cache := NewMockCache() + fetcher := newBlockingFetcher([]byte("raw")) + t.Cleanup(fetcher.Release) + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), fetcher, allowAllBlockChecker(), DefaultConfig()) + require.NoError(t, err) + done := callGetImageAsync(t.Context(), service, moderationTestCID) + fetcher.waitForEntries(t, 1, 5*time.Second) + + purged := make(chan error, 2) + go func() { + purged <- service.PurgeBlob(moderationTestCID) + purged <- service.PurgeOwnerBlob("did:plc:test123", moderationTestCID) + }() + for range 2 { + select { + case err := <-purged: + require.NoError(t, err) + case <-time.After(5 * time.Second): + t.Fatal("purge waited on a fetch that holds no publication lock") + } + } + fetcher.Release() + require.NoError(t, receiveImage(t, done, "released fetch").err) +} + +func TestImageProxyService_PublicationWaiterReleasedWhenContextEnds(t *testing.T) { + cache := NewMockCache() + service, err := NewService(cache, NewMockProcessor([]byte("processed"), nil), NewMockFetcher([]byte("raw"), nil), allowAllBlockChecker(), DefaultConfig()) + require.NoError(t, err) + release, err := service.lockPublication(t.Context(), moderationTestCID) + require.NoError(t, err) + t.Cleanup(release) + + t.Run("cancelled waiter", func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + waiterDone := make(chan error, 1) + go func() { + _, err := service.lockPublication(ctx, moderationTestCID) + waiterDone <- err + }() + cancel() + select { + case err := <-waiterDone: + assert.ErrorIs(t, err, context.Canceled) + case <-time.After(5 * time.Second): + t.Fatal("a waiter on the publication stripe ignored its cancelled context") + } + }) + + t.Run("publication gives up at its bound and fails closed", func(t *testing.T) { + service.publicationTimeout = 20 * time.Millisecond + result := receiveImage(t, callGetImageAsync(t.Context(), service, moderationTestCID), "publication waiting on a held stripe") + assert.ErrorIs(t, result.err, ErrBlockCheckFailed) + assert.ErrorIs(t, result.err, context.DeadlineExceeded) + assert.Empty(t, result.data, "bytes that were never rechecked must not be served") + assert.Zero(t, cache.SetCalls()) + }) +} + +type blockedBlobListerFunc func(context.Context) ([]BlockedBlob, error) + +func (f blockedBlobListerFunc) ListActiveBlockedBlobs(ctx context.Context) ([]BlockedBlob, error) { + return f(ctx) +} + +func TestImageProxyService_PurgeActiveBlocksRemovesBlockedEntries(t *testing.T) { + const ( + ownerA = "did:plc:sweepownera" + ownerB = "did:plc:sweepownerb" + ownerBlocked = "bafyreiownerblocked" + everyOwner = "bafyreieveryownerblocked" + unrelated = "bafyreiunrelated" + ) + cache, err := NewDiskCache(t.TempDir(), 1, 0) + require.NoError(t, err) + type entry struct{ preset, did, cid string } + removed := []entry{{"avatar", ownerA, ownerBlocked}, {"banner", ownerA, ownerBlocked}, {"avatar", ownerA, everyOwner}, {"banner", ownerB, everyOwner}} + kept := []entry{{"avatar", ownerB, ownerBlocked}, {"avatar", ownerA, unrelated}} + for _, e := range append(append([]entry{}, removed...), kept...) { + require.NoError(t, cache.Set(e.preset, e.did, e.cid, []byte("image"))) + } + lister := blockedBlobListerFunc(func(context.Context) ([]BlockedBlob, error) { + return []BlockedBlob{{OwnerDID: ownerA, CID: ownerBlocked}, {CID: everyOwner}, {OwnerDID: ownerB, CID: everyOwner}}, nil + }) + service, err := NewService(cache, NewMockProcessor(nil, nil), NewMockFetcher(nil, nil), allowAllBlockChecker(), DefaultConfig()) + require.NoError(t, err) + + require.NoError(t, service.PurgeActiveBlocks(t.Context(), lister)) + for _, e := range removed { + _, found, err := cache.Get(e.preset, e.did, e.cid) + require.NoError(t, err) + assert.False(t, found, "blocked entry %v must be purged", e) + } + for _, e := range kept { + _, found, err := cache.Get(e.preset, e.did, e.cid) + require.NoError(t, err) + assert.True(t, found, "unblocked entry %v must stay", e) + } + + listError := errors.New("block list unavailable") + err = service.PurgeActiveBlocks(t.Context(), blockedBlobListerFunc(func(context.Context) ([]BlockedBlob, error) { return nil, listError })) + assert.ErrorIs(t, err, listError) +} + +func TestImageProxyService_ActiveBlockPurgeJobSweepsAtStartup(t *testing.T) { + cache := NewMockCache() + cache.SetCacheData("avatar", moderationTestOwner, moderationTestCID, []byte("left by a lost purge")) + service, err := NewService(cache, NewMockProcessor(nil, nil), NewMockFetcher(nil, nil), allowAllBlockChecker(), DefaultConfig()) + require.NoError(t, err) + lister := blockedBlobListerFunc(func(context.Context) ([]BlockedBlob, error) { + return []BlockedBlob{{OwnerDID: moderationTestOwner, CID: moderationTestCID}}, nil + }) + + stop := service.StartActiveBlockPurgeJob(lister, 0) + t.Cleanup(stop) + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + _, found, err := cache.Get("avatar", moderationTestOwner, moderationTestCID) + return !found, err + }, testkit.WithDescription("startup sweep removes bytes a lost purge left on disk")) +} diff --git a/internal/core/imageproxy/service.go b/internal/core/imageproxy/service.go index 3227c62..00804ac 100644 --- a/internal/core/imageproxy/service.go +++ b/internal/core/imageproxy/service.go @@ -13,6 +13,8 @@ package imageproxy import ( "context" + "crypto/sha256" + "errors" "fmt" "log/slog" "sync/atomic" @@ -21,11 +23,11 @@ import ( "golang.org/x/sync/semaphore" ) -// cacheWriteErrors tracks the number of async cache write failures. +// cacheWriteErrors tracks the number of cache write failures. // This provides observability for cache write issues until proper metrics are implemented. var cacheWriteErrors atomic.Int64 -// CacheWriteErrorCount returns the total number of async cache write errors. +// CacheWriteErrorCount returns the total number of cache write errors. // This is useful for monitoring and alerting on cache health. func CacheWriteErrorCount() int64 { return cacheWriteErrors.Load() @@ -49,7 +51,7 @@ func ProcessorBusyRefusalCount() int64 { // Service defines the interface for the image proxy service. type Service interface { // GetImage retrieves an image for the given preset, DID, and CID. - // It checks the cache first, then fetches from the PDS if not cached, + // It checks moderation before the cache, then fetches from the PDS if not cached, // waits for a processing slot, processes the image according to the // preset, and stores in cache. GetImage(ctx context.Context, preset, did, cid string, pdsURL string) ([]byte, error) @@ -61,13 +63,23 @@ type ImageProxyService struct { cache Cache processor Processor fetcher Fetcher + blocks BlockChecker config Config + // A fixed number of stripes serializes cache publication with purges for + // the same CID without retaining locks for every blob ever requested. + // Each stripe is a weight-1 semaphore rather than a mutex so a waiter can + // give up when its context ends. + publicationLocks [256]*semaphore.Weighted + // publicationTimeout bounds the wait for the stripe and the moderation + // recheck under it. The cache write that follows is not bounded by it. + publicationTimeout time.Duration // Two bounds together cap the transient memory a burst of cold requests // can demand, which is what a decompression-bomb flood attacks: // // admissionSlots bounds how MANY cache-miss requests are past the cache - // check at once, each holding up to MaxSourceSizeMB of fetched blob; + // check at once, each holding up to MaxSourceSizeMB of fetched blob or, + // once processed, its output until publication finishes; // processSlots bounds how many of those DECODE at once, each costing up // to the pixel budget × ~19 B/px (see the processor's cost model). // @@ -85,7 +97,7 @@ type ImageProxyService struct { // NewService creates a new ImageProxyService with the provided dependencies. // Returns an error if any required dependency is nil or if the processing // budgets the service enforces itself are not positive. -func NewService(cache Cache, processor Processor, fetcher Fetcher, config Config) (*ImageProxyService, error) { +func NewService(cache Cache, processor Processor, fetcher Fetcher, blocks BlockChecker, config Config) (*ImageProxyService, error) { if cache == nil { return nil, fmt.Errorf("%w: cache", ErrNilDependency) } @@ -95,6 +107,9 @@ func NewService(cache Cache, processor Processor, fetcher Fetcher, config Config if fetcher == nil { return nil, fmt.Errorf("%w: fetcher", ErrNilDependency) } + if blocks == nil { + return nil, fmt.Errorf("%w: block checker", ErrNilDependency) + } // A zero-slot semaphore would refuse every request and a zero wait would // refuse any request that did not find a free slot on its first try. if config.MaxConcurrentProcesses <= 0 { @@ -107,27 +122,39 @@ func NewService(cache Cache, processor Processor, fetcher Fetcher, config Config return nil, fmt.Errorf("%w: got %d", ErrInvalidMaxInFlightRequests, config.MaxInFlightRequests) } - return &ImageProxyService{ - cache: cache, - processor: processor, - fetcher: fetcher, - config: config, - admissionSlots: semaphore.NewWeighted(int64(config.MaxInFlightRequests)), - processSlots: semaphore.NewWeighted(int64(config.MaxConcurrentProcesses)), - processQueueWait: config.ProcessQueueWait, - }, nil + service := &ImageProxyService{ + cache: cache, + processor: processor, + fetcher: fetcher, + blocks: blocks, + config: config, + publicationTimeout: defaultPublicationTimeout, + admissionSlots: semaphore.NewWeighted(int64(config.MaxInFlightRequests)), + processSlots: semaphore.NewWeighted(int64(config.MaxConcurrentProcesses)), + processQueueWait: config.ProcessQueueWait, + } + for index := range service.publicationLocks { + service.publicationLocks[index] = semaphore.NewWeighted(1) + } + return service, nil } +// defaultPublicationTimeout bounds the publication stripe wait and the +// post-fetch moderation recheck, which run after the caller may already have +// gone away. +const defaultPublicationTimeout = 10 * time.Second + // GetImage retrieves an image for the given preset, DID, and CID. // The service flow is: // 1. Validate preset exists -// 2. Check cache for (preset, did, cid) - return if hit +// 2. Check moderation, then cache for (preset, did, cid) - return if hit // 3. Acquire an admission slot, waiting at most ProcessQueueWait // 4. Fetch blob from PDS using pdsURL // 5. Acquire a processing slot, waiting at most ProcessQueueWait // 6. Process image with preset -// 7. Store in cache (async, don't block response) -// 8. Return processed image +// 7. Release the processing slot, then under the CID lock recheck +// moderation and store in cache, on a context detached from the caller +// 8. Release the admission slot and return the processed image func (s *ImageProxyService) GetImage(ctx context.Context, presetName, did, cid string, pdsURL string) ([]byte, error) { // Step 1: Validate preset exists preset, err := GetPreset(presetName) @@ -135,7 +162,19 @@ func (s *ImageProxyService) GetImage(ctx context.Context, presetName, did, cid s return nil, err } - // Step 2: Check cache for (preset, did, cid) + // Step 2: Check moderation before reading even a warm cache entry. + blocked, err := s.IsBlobBlocked(ctx, did, cid) + if err != nil { + return nil, err + } + if blocked { + // A purge lost to a crash or a disk error leaves bytes behind; any + // request for them removes this entry. + s.purgeBlockedEntry(ctx, presetName, did, cid) + return nil, ErrBlobBlocked + } + + // Check cache for (preset, did, cid) cachedData, found, err := s.cache.Get(presetName, did, cid) if err != nil { // Log cache read error but continue - cache miss is acceptable @@ -158,8 +197,10 @@ func (s *ImageProxyService) GetImage(ctx context.Context, presetName, did, cid s // Step 3: Acquire an admission slot. This sits AFTER the cache check so a // hit, which costs a file read and holds no blob, is never refused under // load; and BEFORE the fetch so the number of fetched blobs held in memory - // is bounded. The slot spans fetch, the processing-slot wait and decode, - // and the deferred release runs on every exit. + // is bounded. The slot spans fetch, the processing-slot wait, decode and + // publication: a processed output waiting on a slow recheck is transient + // memory too, and content presets bound it only by the source pixel budget. + // The deferred release runs on every exit. err = s.acquireSlot(ctx, s.admissionSlots, "admission", func(waitErr error) error { slog.Warn("[IMAGE-PROXY] in-flight request cap reached, shedding request", "preset", presetName, @@ -177,6 +218,27 @@ func (s *ImageProxyService) GetImage(ctx context.Context, presetName, did, cid s } defer s.admissionSlots.Release(1) + processedData, err := s.fetchAndProcess(ctx, preset, presetName, did, cid, pdsURL) + if err != nil { + return nil, err + } + + // Step 7: The check and the cache publication must be atomic against + // purges; a purge after this write removes the bytes before returning. + if err := s.publish(ctx, presetName, did, cid, processedData); err != nil { + return nil, err + } + + // Step 8: Return processed image + return processedData, nil +} + +// fetchAndProcess runs steps 4 to 6 of GetImage under the caller's admission +// slot. The processing slot is released when it returns, so it is not held +// across the moderation recheck and cache write that follow: those wait on a +// database and a stripe lock, not on CPU, and a decode slot held there would +// refuse unrelated requests as busy. +func (s *ImageProxyService) fetchAndProcess(ctx context.Context, preset Preset, presetName, did, cid, pdsURL string) ([]byte, error) { // Step 4: Fetch blob from PDS rawData, err := s.fetcher.Fetch(ctx, pdsURL, did, cid) if err != nil { @@ -205,36 +267,48 @@ func (s *ImageProxyService) GetImage(ctx context.Context, presetName, did, cid s defer s.processSlots.Release(1) // Step 6: Process image with preset - processedData, err := s.processor.Process(rawData, preset) + return s.processor.Process(rawData, preset) +} + +// publish rechecks moderation and writes the processed image to the cache +// under the CID's publication lock. It runs on a context detached from the +// caller, so a client that hangs up after processing still leaves the bytes +// cached, and bounded by publicationTimeout, so a slow database cannot pin the +// stripe. A stripe that cannot be taken in time fails closed: the recheck has +// not run, so the bytes are not served. +func (s *ImageProxyService) publish(ctx context.Context, presetName, did, cid string, processedData []byte) error { + publishCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), s.publicationTimeout) + defer cancel() + release, err := s.lockPublication(publishCtx, cid) if err != nil { - return nil, err + return fmt.Errorf("%w: waiting for the publication lock: %w", ErrBlockCheckFailed, err) } - - // Step 7: Store in cache (async, don't block response) - go func() { - // Use a background context since the original request context may be cancelled - if cacheErr := s.cache.Set(presetName, did, cid, processedData); cacheErr != nil { - // Increment error counter for monitoring - cacheWriteErrors.Add(1) - slog.Error("[IMAGE-PROXY] async cache write failed", - "preset", presetName, - "did", did, - "cid", cid, - "error", cacheErr, - "total_cache_write_errors", cacheWriteErrors.Load(), - ) - } else { - slog.Debug("[IMAGE-PROXY] cached processed image", - "preset", presetName, - "did", did, - "cid", cid, - "size_bytes", len(processedData), - ) - } - }() - - // Step 8: Return processed image - return processedData, nil + defer release() + blocked, err := s.IsBlobBlocked(publishCtx, did, cid) + if err != nil { + return err + } + if blocked { + return ErrBlobBlocked + } + if cacheErr := s.cache.Set(presetName, did, cid, processedData); cacheErr != nil { + cacheWriteErrors.Add(1) + slog.Error("[IMAGE-PROXY] cache write failed", + "preset", presetName, + "did", did, + "cid", cid, + "error", cacheErr, + "total_cache_write_errors", cacheWriteErrors.Load(), + ) + } else { + slog.Debug("[IMAGE-PROXY] cached processed image", + "preset", presetName, + "did", did, + "cid", cid, + "size_bytes", len(processedData), + ) + } + return nil } // acquireSlot takes one unit of sem on the caller's behalf, waiting at most @@ -267,3 +341,167 @@ func (s *ImageProxyService) acquireSlot(ctx context.Context, sem *semaphore.Weig processorBusyRefusals.Add(1) return onBusy(err) } + +// BlockChecker reports whether moderation blocks serving a blob. +type BlockChecker interface { + IsBlocked(ctx context.Context, ownerDID, cid string) (bool, error) +} + +// IsBlobBlocked reports whether moderation blocks serving the blob. +func (s *ImageProxyService) IsBlobBlocked(ctx context.Context, did, cid string) (bool, error) { + blocked, err := s.blocks.IsBlocked(ctx, did, cid) + if err != nil { + return false, fmt.Errorf("%w: %w", ErrBlockCheckFailed, err) + } + return blocked, nil +} + +// PurgeOwnerBlob removes the owner's cached copies of a blocked blob. The +// caller must have committed the block first. +func (s *ImageProxyService) PurgeOwnerBlob(did, cid string) error { + if err := s.awaitPublications(context.Background(), cid); err != nil { + return err + } + return s.cache.DeleteOwner(did, cid) +} + +// PurgeBlob removes every owner's cached copies of a blocked blob. The caller +// must have committed the block first. +func (s *ImageProxyService) PurgeBlob(cid string) error { + if err := s.awaitPublications(context.Background(), cid); err != nil { + return err + } + return s.cache.DeleteCID(cid) +} + +// awaitPublications waits until every cache publication of cid that holds or +// is queued for the stripe has finished, and does not keep the stripe for the +// disk work that follows. That is enough once the block is committed: a +// publication that took the stripe before this call has finished writing by +// the time it returns, so the caller's deletion sees its file, and one that +// takes the stripe afterwards rechecks moderation after the commit and sees +// the block, so it writes nothing. +func (s *ImageProxyService) awaitPublications(ctx context.Context, cid string) error { + release, err := s.lockPublication(ctx, cid) + if err != nil { + return err + } + release() + return nil +} + +// purgeBlockedEntry removes one blocked cache entry on behalf of a request the +// pre-cache check refused. A failure is logged and otherwise ignored: the +// request is refused either way, and the periodic sweep retries. A caller +// that went away while waiting for the stripe is not a failure worth logging. +func (s *ImageProxyService) purgeBlockedEntry(ctx context.Context, presetName, did, cid string) { + if err := s.awaitPublications(ctx, cid); err != nil { + return + } + if err := s.cache.Delete(presetName, did, cid); err != nil { + slog.Error("[IMAGE-PROXY] failed to purge a blocked cache entry", + "preset", presetName, + "did", did, + "cid", cid, + "error", err, + ) + } +} + +// BlockedBlob identifies a blob with an active moderation block: one owner's +// copy, or every owner's copy when OwnerDID is empty. +type BlockedBlob struct { + OwnerDID string + CID string +} + +// BlockedBlobLister lists every blob with an active moderation block. +type BlockedBlobLister interface { + ListActiveBlockedBlobs(ctx context.Context) ([]BlockedBlob, error) +} + +// PurgeActiveBlocks removes the cached bytes of every blob with an active +// block. The purge a moderation action runs after commit is lost if the +// process exits first or the disk refuses it; this sweep makes that purge +// restart-safe. It keeps going past a failed blob and returns every failure +// joined. +func (s *ImageProxyService) PurgeActiveBlocks(ctx context.Context, lister BlockedBlobLister) error { + blobs, err := lister.ListActiveBlockedBlobs(ctx) + if err != nil { + return fmt.Errorf("listing blocked blobs: %w", err) + } + everyOwner := make(map[string]bool) + for _, blob := range blobs { + if blob.OwnerDID == "" { + everyOwner[blob.CID] = true + } + } + var errs []error + for _, blob := range blobs { + if blob.OwnerDID != "" && everyOwner[blob.CID] { + continue // the every-owner purge below covers this owner + } + if err := s.awaitPublications(ctx, blob.CID); err != nil { + return errors.Join(append(errs, err)...) + } + if blob.OwnerDID == "" { + err = s.cache.DeleteCID(blob.CID) + } else { + err = s.cache.DeleteOwner(blob.OwnerDID, blob.CID) + } + if err != nil { + errs = append(errs, fmt.Errorf("purging blob %s: %w", blob.CID, err)) + } + } + return errors.Join(errs...) +} + +// StartActiveBlockPurgeJob runs PurgeActiveBlocks once straight away, which +// completes any purge a restart interrupted, and then every interval. An +// interval of zero or less runs only the startup sweep. The returned function +// stops the job. +func (s *ImageProxyService) StartActiveBlockPurgeJob(lister BlockedBlobLister, interval time.Duration) context.CancelFunc { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + defer func() { + if r := recover(); r != nil { + slog.Error("[IMAGE-PROXY] CRITICAL: blocked media purge job panicked", "panic", r) + } + }() + s.runActiveBlockPurge(ctx, lister) + if interval <= 0 { + return + } + ticker := time.NewTicker(interval) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + return + case <-ticker.C: + s.runActiveBlockPurge(ctx, lister) + } + } + }() + return cancel +} + +func (s *ImageProxyService) runActiveBlockPurge(ctx context.Context, lister BlockedBlobLister) { + if err := s.PurgeActiveBlocks(ctx, lister); err != nil && ctx.Err() == nil { + slog.Error("[IMAGE-PROXY] blocked media purge failed", "error", err) + } +} + +// lockPublication takes cid's publication stripe, giving up when ctx ends. +func (s *ImageProxyService) lockPublication(ctx context.Context, cid string) (release func(), err error) { + lock := s.publicationLock(cid) + if err := lock.Acquire(ctx, 1); err != nil { + return nil, err + } + return func() { lock.Release(1) }, nil +} + +func (s *ImageProxyService) publicationLock(cid string) *semaphore.Weighted { + digest := sha256.Sum256([]byte(cid)) + return s.publicationLocks[digest[0]] +} diff --git a/internal/core/imageproxy/service_test.go b/internal/core/imageproxy/service_test.go index 8041aa7..1fc02fb 100644 --- a/internal/core/imageproxy/service_test.go +++ b/internal/core/imageproxy/service_test.go @@ -4,13 +4,25 @@ import ( "context" "errors" "fmt" + "strings" "sync" "testing" "time" - "Coves/tests/testkit" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) +type blockCheckFunc func(context.Context, string, string) (bool, error) + +func (f blockCheckFunc) IsBlocked(ctx context.Context, did, cid string) (bool, error) { + return f(ctx, did, cid) +} + +func allowAllBlockChecker() BlockChecker { + return blockCheckFunc(func(context.Context, string, string) (bool, error) { return false, nil }) +} + // MockCache implements Cache for testing type MockCache struct { mu sync.Mutex @@ -58,6 +70,28 @@ func (m *MockCache) Delete(preset, did, cid string) error { return nil } +func (m *MockCache) DeleteOwner(did, cid string) error { + m.mu.Lock() + defer m.mu.Unlock() + for key := range m.data { + if strings.HasSuffix(key, ":"+did+":"+cid) { + delete(m.data, key) + } + } + return nil +} + +func (m *MockCache) DeleteCID(cid string) error { + m.mu.Lock() + defer m.mu.Unlock() + for key := range m.data { + if strings.HasSuffix(key, ":"+cid) { + delete(m.data, key) + } + } + return nil +} + func (m *MockCache) Cleanup() (int, error) { // Mock implementation - no-op for tests return 0, nil @@ -155,7 +189,7 @@ func (m *MockFetcher) Calls() int { // mustNewService is a test helper that creates a service or fails the test func mustNewService(t *testing.T, cache Cache, processor Processor, fetcher Fetcher, config Config) *ImageProxyService { t.Helper() - service, err := NewService(cache, processor, fetcher, config) + service, err := NewService(cache, processor, fetcher, allowAllBlockChecker(), config) if err != nil { t.Fatalf("NewService failed: %v", err) } @@ -223,14 +257,7 @@ func TestImageProxyService_GetImage_CacheMiss(t *testing.T) { t.Errorf("expected processor to be called once, got %d calls", processor.Calls()) } - // The cache write happens on its own goroutine, so wait for the write - // itself rather than for a duration guessed to contain it. - testkit.WaitFor(t, 5*time.Second, func() (bool, error) { - return cache.SetCalls() >= 1, nil - }, testkit.WithDescription("the asynchronous cache write to land"), - testkit.WithDiagnostics(func() string { - return fmt.Sprintf("cache Set calls: %d", cache.SetCalls()) - })) + require.Equal(t, 1, cache.SetCalls(), "cache publication must finish before GetImage returns") // Verify the correct data was cached setData, found := cache.GetSetData("avatar", "did:plc:test123", "bafyreicid123") @@ -287,7 +314,7 @@ func TestImageProxyService_GetImage_ProcessingError(t *testing.T) { } } -func TestImageProxyService_GetImage_CacheWriteIsAsync(t *testing.T) { +func TestImageProxyService_GetImage_CacheWriteCompletesBeforeReturn(t *testing.T) { cache := NewMockCache() rawImageData := []byte("raw image from PDS") processedData := []byte("processed image") @@ -298,10 +325,7 @@ func TestImageProxyService_GetImage_CacheWriteIsAsync(t *testing.T) { service := mustNewService(t, cache, processor, fetcher, config) ctx := context.Background() - // Call GetImage - startTime := time.Now() data, err := service.GetImage(ctx, "avatar", "did:plc:test123", "bafyreicid123", "https://pds.example.com") - elapsed := time.Since(startTime) if err != nil { t.Fatalf("expected no error, got: %v", err) @@ -310,19 +334,7 @@ func TestImageProxyService_GetImage_CacheWriteIsAsync(t *testing.T) { t.Errorf("expected processed data %q, got %q", processedData, data) } - // The response should come back quickly, not blocked by cache write - // (This is a soft assertion - just ensures we're not blocking) - if elapsed > 100*time.Millisecond { - t.Logf("warning: GetImage took %v, expected faster response", elapsed) - } - - // The write still has to happen — asynchronous must not mean dropped. - testkit.WaitFor(t, 5*time.Second, func() (bool, error) { - return cache.SetCalls() >= 1, nil - }, testkit.WithDescription("the cache write to complete after GetImage returned"), - testkit.WithDiagnostics(func() string { - return fmt.Sprintf("cache Set calls: %d", cache.SetCalls()) - })) + require.Equal(t, 1, cache.SetCalls(), "a late cache write could republish bytes after a purge returns") } func TestImageProxyService_GetImage_EmptyPreset(t *testing.T) { @@ -373,28 +385,28 @@ func TestNewService_NilDependencies(t *testing.T) { fetcher := NewMockFetcher(nil, nil) t.Run("nil cache", func(t *testing.T) { - _, err := NewService(nil, processor, fetcher, config) + _, err := NewService(nil, processor, fetcher, allowAllBlockChecker(), config) if !errors.Is(err, ErrNilDependency) { t.Errorf("expected ErrNilDependency, got: %v", err) } }) t.Run("nil processor", func(t *testing.T) { - _, err := NewService(cache, nil, fetcher, config) + _, err := NewService(cache, nil, fetcher, allowAllBlockChecker(), config) if !errors.Is(err, ErrNilDependency) { t.Errorf("expected ErrNilDependency, got: %v", err) } }) t.Run("nil fetcher", func(t *testing.T) { - _, err := NewService(cache, processor, nil, config) + _, err := NewService(cache, processor, nil, allowAllBlockChecker(), config) if !errors.Is(err, ErrNilDependency) { t.Errorf("expected ErrNilDependency, got: %v", err) } }) t.Run("all valid", func(t *testing.T) { - service, err := NewService(cache, processor, fetcher, config) + service, err := NewService(cache, processor, fetcher, allowAllBlockChecker(), config) if err != nil { t.Errorf("expected no error with valid dependencies, got: %v", err) } @@ -402,6 +414,12 @@ func TestNewService_NilDependencies(t *testing.T) { t.Error("expected non-nil service") } }) + + t.Run("nil block checker", func(t *testing.T) { + service, err := NewService(cache, processor, fetcher, nil, config) + assert.ErrorIs(t, err, ErrNilDependency) + assert.Nil(t, service) + }) } // blockingProcessor is a Processor whose Process call parks until released. @@ -566,7 +584,7 @@ func TestNewService_RejectsInvalidProcessingBudgets(t *testing.T) { cfg := DefaultConfig() tt.mutate(&cfg) - service, err := NewService(NewMockCache(), NewMockProcessor(nil, nil), NewMockFetcher(nil, nil), cfg) + service, err := NewService(NewMockCache(), NewMockProcessor(nil, nil), NewMockFetcher(nil, nil), allowAllBlockChecker(), cfg) if err == nil { t.Fatalf("expected NewService to refuse the config with %v, got a service", tt.wantErr) diff --git a/internal/core/imageproxy/validation.go b/internal/core/imageproxy/validation.go index ba7574e..72d4671 100644 --- a/internal/core/imageproxy/validation.go +++ b/internal/core/imageproxy/validation.go @@ -4,6 +4,7 @@ import ( "strings" "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/ipfs/go-cid" ) // ValidateDID validates that a DID string matches expected atproto DID formats. @@ -42,6 +43,22 @@ func ValidateCID(cid string) error { return nil } +// CanonicalCID decodes a CID in any multibase encoding and returns its +// canonical string form. A PDS resolves every encoding of a CID to the same +// blob, so moderation blocks, cache keys and fetches must all use this one +// form; ValidateCID alone is a syntax check and accepts strings that are not +// CIDs. Returns ErrInvalidCID if the value does not decode. +func CanonicalCID(value string) (string, error) { + if err := ValidateCID(value); err != nil { + return "", err + } + parsed, err := cid.Decode(value) + if err != nil { + return "", ErrInvalidCID + } + return parsed.String(), nil +} + // SanitizePathComponent ensures a string is safe to use as a filesystem path component. // It removes or replaces characters that could be used for path traversal attacks. // This is used as an additional safety layer beyond DID/CID validation. diff --git a/internal/core/imageproxy/validation_test.go b/internal/core/imageproxy/validation_test.go index 87deffb..26a52d6 100644 --- a/internal/core/imageproxy/validation_test.go +++ b/internal/core/imageproxy/validation_test.go @@ -3,6 +3,9 @@ package imageproxy import ( "errors" "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multibase" ) func TestValidateDID(t *testing.T) { @@ -340,3 +343,44 @@ func contains(s, substr string) bool { } return false } + +func TestCanonicalCID(t *testing.T) { + const canonical = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" + parsed, err := cid.Decode(canonical) + if err != nil { + t.Fatalf("fixture must decode: %v", err) + } + base58, err := parsed.StringOfBase(multibase.Base58BTC) + if err != nil { + t.Fatalf("re-encode fixture: %v", err) + } + base16, err := parsed.StringOfBase(multibase.Base16) + if err != nil { + t.Fatalf("re-encode fixture: %v", err) + } + tests := []struct { + name string + value string + want string + wantErr error + }{ + {name: "canonical CIDv1 is unchanged", value: canonical, want: canonical}, + {name: "base58btc CIDv1 is canonicalized", value: base58, want: canonical}, + {name: "base16 CIDv1 is canonicalized", value: base16, want: canonical}, + {name: "CIDv0 keeps its canonical form", value: "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG", want: "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG"}, + {name: "syntax-valid string that is not a CID", value: "bafybeimockimagetest123", wantErr: ErrInvalidCID}, + {name: "path traversal", value: "../../../etc/passwd", wantErr: ErrInvalidCID}, + {name: "empty", value: "", wantErr: ErrInvalidCID}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := CanonicalCID(tt.value) + if !errors.Is(err, tt.wantErr) { + t.Fatalf("CanonicalCID(%q) error = %v, want %v", tt.value, err, tt.wantErr) + } + if got != tt.want { + t.Errorf("CanonicalCID(%q) = %q, want %q", tt.value, got, tt.want) + } + }) + } +} diff --git a/internal/core/moderation/fake_store_test.go b/internal/core/moderation/fake_store_test.go new file mode 100644 index 0000000..3b9af32 --- /dev/null +++ b/internal/core/moderation/fake_store_test.go @@ -0,0 +1,241 @@ +package moderation_test + +import ( + "context" + "fmt" + "sync" + "time" + + "Coves/internal/core/moderation" +) + +type inMemoryModerationDecisionKey struct { + authorityDID string + subjectURI string +} + +type inMemoryModerationIdempotencyKey struct { + actorDID string + authorityDID string + key string +} + +type inMemoryModerationState struct { + indexedComments map[string]moderation.IndexedComment + versions map[string]int64 + actions map[string]moderation.Action + activeRemovals map[inMemoryModerationDecisionKey]string + idempotency map[inMemoryModerationIdempotencyKey]moderation.IdempotencyRecord + mediaBlocks map[moderation.MediaBlock]bool + nextActionID int +} + +func (state inMemoryModerationState) copy() inMemoryModerationState { + working := inMemoryModerationState{ + indexedComments: make(map[string]moderation.IndexedComment, len(state.indexedComments)), + versions: make(map[string]int64, len(state.versions)), + actions: make(map[string]moderation.Action, len(state.actions)), + activeRemovals: make(map[inMemoryModerationDecisionKey]string, len(state.activeRemovals)), + idempotency: make(map[inMemoryModerationIdempotencyKey]moderation.IdempotencyRecord, len(state.idempotency)), + mediaBlocks: make(map[moderation.MediaBlock]bool, len(state.mediaBlocks)), + nextActionID: state.nextActionID, + } + for key, comment := range state.indexedComments { + comment.ImageCIDs = append([]string(nil), comment.ImageCIDs...) + working.indexedComments[key] = comment + } + for key, version := range state.versions { + working.versions[key] = version + } + for key, action := range state.actions { + working.actions[key] = action + } + for key, actionID := range state.activeRemovals { + working.activeRemovals[key] = actionID + } + for key, record := range state.idempotency { + working.idempotency[key] = record + } + for block, active := range state.mediaBlocks { + working.mediaBlocks[block] = active + } + return working +} + +// Each transaction holds the store lock and changes a private snapshot. The +// write log includes attempts even if an error rolls the snapshot back. +type inMemoryModerationStore struct { + mu sync.Mutex + state inMemoryModerationState + now time.Time + writeCalls []string + failInsertAction error + failSetRemovalDecision error +} + +func newInMemoryModerationStore(now time.Time) *inMemoryModerationStore { + return &inMemoryModerationStore{ + now: now, + state: inMemoryModerationState{ + indexedComments: make(map[string]moderation.IndexedComment), + versions: make(map[string]int64), + actions: make(map[string]moderation.Action), + activeRemovals: make(map[inMemoryModerationDecisionKey]string), + idempotency: make(map[inMemoryModerationIdempotencyKey]moderation.IdempotencyRecord), + mediaBlocks: make(map[moderation.MediaBlock]bool), + }, + } +} + +func (store *inMemoryModerationStore) InTransaction(ctx context.Context, fn func(context.Context, moderation.Transaction) error) error { + store.mu.Lock() + defer store.mu.Unlock() + working := store.state.copy() + transaction := &inMemoryModerationTransaction{store: store, state: &working} + if err := fn(ctx, transaction); err != nil { + return err + } + store.state = working + return nil +} + +func (store *inMemoryModerationStore) SubjectModeration(_ context.Context, authorityDID, subjectURI string) (*moderation.SubjectModeration, error) { + store.mu.Lock() + defer store.mu.Unlock() + state := &moderation.SubjectModeration{Version: store.state.versions[subjectURI]} + key := inMemoryModerationDecisionKey{authorityDID: authorityDID, subjectURI: subjectURI} + if actionID := store.state.activeRemovals[key]; actionID != "" { + action := store.state.actions[actionID] + state.ActiveRemoval = &action + } + return state, nil +} + +type inMemoryModerationTransaction struct { + store *inMemoryModerationStore + state *inMemoryModerationState +} + +func (*inMemoryModerationTransaction) LockActor(context.Context, string) error { + // InTransaction already serializes writers to the store. + return nil +} + +func (transaction *inMemoryModerationTransaction) LiveIdempotencyRecord(_ context.Context, actorDID, authorityDID, key string, now time.Time) (*moderation.IdempotencyRecord, error) { + record, exists := transaction.state.idempotency[inMemoryModerationIdempotencyKey{actorDID, authorityDID, key}] + if !exists || !now.Before(record.ExpiresAt) { + return nil, nil + } + return &record, nil +} + +func (transaction *inMemoryModerationTransaction) CountLiveIdempotencyKeys(_ context.Context, actorDID string, now time.Time) (int, error) { + count := 0 + for key, record := range transaction.state.idempotency { + if key.actorDID == actorDID && now.Before(record.ExpiresAt) { + count++ + } + } + return count, nil +} + +func (transaction *inMemoryModerationTransaction) SaveIdempotencyRecord(_ context.Context, record moderation.IdempotencyRecord) error { + transaction.store.writeCalls = append(transaction.store.writeCalls, "SaveIdempotencyRecord") + key := inMemoryModerationIdempotencyKey{record.ActorDID, record.AuthorityDID, record.Key} + transaction.state.idempotency[key] = record + return nil +} + +func (transaction *inMemoryModerationTransaction) LockSubject(_ context.Context, subjectURI string) (int64, error) { + version := transaction.state.versions[subjectURI] + transaction.state.versions[subjectURI] = version + return version, nil +} + +func (transaction *inMemoryModerationTransaction) ReadIndexedComment(_ context.Context, subjectURI string) (*moderation.IndexedComment, error) { + comment, exists := transaction.state.indexedComments[subjectURI] + if !exists { + return nil, moderation.ErrSubjectNotIndexed + } + comment.ImageCIDs = append([]string(nil), comment.ImageCIDs...) + return &comment, nil +} + +func (transaction *inMemoryModerationTransaction) GetAction(_ context.Context, actionID string) (*moderation.Action, error) { + action, exists := transaction.state.actions[actionID] + if !exists { + return nil, moderation.ErrDecisionNotFound + } + return &action, nil +} + +func (transaction *inMemoryModerationTransaction) ActiveRemoval(_ context.Context, authorityDID, subjectURI string) (*moderation.Action, error) { + key := inMemoryModerationDecisionKey{authorityDID: authorityDID, subjectURI: subjectURI} + actionID := transaction.state.activeRemovals[key] + if actionID == "" { + return nil, nil + } + action := transaction.state.actions[actionID] + return &action, nil +} + +func (transaction *inMemoryModerationTransaction) InsertAction(_ context.Context, action moderation.Action) (*moderation.Action, error) { + transaction.store.writeCalls = append(transaction.store.writeCalls, "InsertAction") + if transaction.store.failInsertAction != nil { + return nil, transaction.store.failInsertAction + } + for { + transaction.state.nextActionID++ + action.ID = fmt.Sprintf("action-%d", transaction.state.nextActionID) + if _, exists := transaction.state.actions[action.ID]; !exists { + break + } + } + action.CreatedAt = transaction.store.now + transaction.state.actions[action.ID] = action + return &action, nil +} + +func (transaction *inMemoryModerationTransaction) SetRemovalDecision(_ context.Context, authorityDID, subjectURI, actionID string, active bool) error { + transaction.store.writeCalls = append(transaction.store.writeCalls, "SetRemovalDecision") + if transaction.store.failSetRemovalDecision != nil { + return transaction.store.failSetRemovalDecision + } + key := inMemoryModerationDecisionKey{authorityDID: authorityDID, subjectURI: subjectURI} + if active { + if _, exists := transaction.state.actions[actionID]; !exists { + return moderation.ErrDecisionNotFound + } + transaction.state.activeRemovals[key] = actionID + } else { + delete(transaction.state.activeRemovals, key) + } + return nil +} + +func (transaction *inMemoryModerationTransaction) SetSubjectVersion(_ context.Context, subjectURI string, version int64) error { + transaction.store.writeCalls = append(transaction.store.writeCalls, "SetSubjectVersion") + transaction.state.versions[subjectURI] = version + return nil +} + +func (transaction *inMemoryModerationTransaction) InsertMediaBlocks(_ context.Context, blocks []moderation.MediaBlock) error { + transaction.store.writeCalls = append(transaction.store.writeCalls, "InsertMediaBlocks") + for _, block := range blocks { + transaction.state.mediaBlocks[block] = true + } + return nil +} + +func (transaction *inMemoryModerationTransaction) DeactivateMediaBlocks(_ context.Context, actionID string) error { + transaction.store.writeCalls = append(transaction.store.writeCalls, "DeactivateMediaBlocks") + for block := range transaction.state.mediaBlocks { + if block.ActionID == actionID { + transaction.state.mediaBlocks[block] = false + } + } + return nil +} + +var _ moderation.Store = (*inMemoryModerationStore)(nil) +var _ moderation.Transaction = (*inMemoryModerationTransaction)(nil) diff --git a/internal/core/moderation/idempotency_test.go b/internal/core/moderation/idempotency_test.go new file mode 100644 index 0000000..79c84fc --- /dev/null +++ b/internal/core/moderation/idempotency_test.go @@ -0,0 +1,276 @@ +package moderation_test + +import ( + "testing" + "time" + + "Coves/internal/core/moderation" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func seedIdempotencyComment(scenario *removeRulesScenario, recordKey string) moderation.RemoveContentRequest { + request := scenario.request + request.Subject.URI = "at://" + removeRulesAuthorDID + "/" + moderation.CommentCollection + "/" + recordKey + scenario.store.state.indexedComments[request.Subject.URI] = moderation.IndexedComment{ + URI: request.Subject.URI, CID: request.Subject.CID, OwnerDID: removeRulesAuthorDID, + CommunityDID: removeRulesCommunityDID, + } + return request +} + +func assertIdempotencyNoMutation(t *testing.T, scenario removeRulesScenario, call func() (*moderation.MutationResult, error)) (*moderation.MutationResult, error) { + t.Helper() + before := scenario.store.state.copy() + scenario.store.writeCalls = nil + ownerPurges := append([]removeRulesOwnerPurge(nil), scenario.purger.ownerPurges...) + blobPurges := append([]string(nil), scenario.purger.blobPurges...) + result, err := call() + assert.Equal(t, before, scenario.store.state, "replay or rejection must not commit a mutation") + assert.Empty(t, scenario.store.writeCalls) + assert.Equal(t, ownerPurges, scenario.purger.ownerPurges) + assert.Equal(t, blobPurges, scenario.purger.blobPurges) + return result, err +} + +func TestModerationIdempotencyReplaysOriginalResultAfterSubjectChanges(t *testing.T) { + t.Run("remove replay after restore", func(t *testing.T) { + scenario := newRemoveRulesScenario() + original, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, original) + require.NotNil(t, original.Action) + restored, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, moderation.RestoreContentRequest{ + ActionID: original.Action.ID, ReviewedSubject: &scenario.request.Subject, + ExpectedVersion: original.State.Version, IdempotencyKey: "different-restore-key", Reason: removeRulesSpam, + }) + require.NoError(t, err) + require.NotNil(t, restored) + require.NotEqual(t, original.State.Version, restored.State.Version) + replayed, err := assertIdempotencyNoMutation(t, scenario, func() (*moderation.MutationResult, error) { + return scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + }) + require.NoError(t, err) + assert.Equal(t, original, replayed, "replay must return the original action and old version verbatim") + assert.Empty(t, scenario.store.state.activeRemovals) + assert.Len(t, scenario.store.state.actions, 2) + }) + + t.Run("restore replay after a new removal", func(t *testing.T) { + scenario := newRestoreRulesScenario(t) + original, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, original) + require.NotNil(t, original.Action) + next := scenario.removeRulesScenario.request + next.ExpectedVersion = original.State.Version + next.IdempotencyKey = "new-removal-key" + newRemoval, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, next) + require.NoError(t, err) + require.NotNil(t, newRemoval) + require.NotNil(t, newRemoval.Action) + require.NotEqual(t, original.State.Version, newRemoval.State.Version) + replayed, err := assertIdempotencyNoMutation(t, scenario.removeRulesScenario, func() (*moderation.MutationResult, error) { + return scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, scenario.request) + }) + require.NoError(t, err) + assert.Equal(t, original, replayed) + assert.Equal(t, newRemoval.Action.ID, scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, removeRulesURI}]) + assert.Len(t, scenario.store.state.actions, 3) + }) +} + +func TestModerationIdempotencyRejectsDifferentFingerprintWithoutWrites(t *testing.T) { + for _, test := range []struct { + name string + change func(*moderation.RemoveContentRequest) + }{ + {"different reason", func(request *moderation.RemoveContentRequest) { request.Reason = removeRulesIllegal }}, + {"different private note", func(request *moderation.RemoveContentRequest) { request.PrivateNote = "different" }}, + {"different subject CID", func(request *moderation.RemoveContentRequest) { request.Subject.CID = "bafyreidifferentcomment" }}, + } { + t.Run("remove: "+test.name, func(t *testing.T) { + scenario := newRemoveRulesScenario() + comment := scenario.store.state.indexedComments[removeRulesURI] + comment.AuthorDeleted = true + scenario.store.state.indexedComments[removeRulesURI] = comment + seed := scenario.request + seed.IdempotencyKey = "seed-removal" + removed, err := scenario.service.RemoveContent(t.Context(), restoreRulesAdminDID, seed) + require.NoError(t, err) + require.NotNil(t, removed) + scenario.request.ExpectedVersion = removed.State.Version + original, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, original) + require.Equal(t, moderation.OutcomeUnchanged, original.Outcome) + request := scenario.request + test.change(&request) + result, err := assertIdempotencyNoMutation(t, scenario, func() (*moderation.MutationResult, error) { + return scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, request) + }) + require.ErrorIs(t, err, moderation.ErrIdempotencyConflict) + assert.Nil(t, result) + }) + } + + t.Run("restore: different reviewed CID", func(t *testing.T) { + scenario := newRestoreRulesScenario(t) + original, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, original) + request := scenario.request + request.ReviewedSubject = &moderation.StrongRef{URI: removeRulesURI, CID: "bafyreidifferentcomment"} + result, err := assertIdempotencyNoMutation(t, scenario.removeRulesScenario, func() (*moderation.MutationResult, error) { + return scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, request) + }) + require.ErrorIs(t, err, moderation.ErrIdempotencyConflict) + assert.Nil(t, result) + }) +} + +func TestModerationIdempotencyKeyIsScopedToActor(t *testing.T) { + scenario := newRemoveRulesScenario() + first := scenario.request + first.IdempotencyKey = "k1" + second := seedIdempotencyComment(&scenario, "3ksecond") + second.IdempotencyKey = "k1" + firstResult, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, first) + require.NoError(t, err) + require.NotNil(t, firstResult) + secondResult, err := scenario.service.RemoveContent(t.Context(), restoreRulesAdminDID, second) + require.NoError(t, err) + require.NotNil(t, secondResult) + require.NotNil(t, firstResult.Action) + require.NotNil(t, secondResult.Action) + assert.Equal(t, moderation.OutcomeApplied, firstResult.Outcome) + assert.Equal(t, moderation.OutcomeApplied, secondResult.Outcome) + assert.NotEqual(t, firstResult.Action.ID, secondResult.Action.ID) + assert.Len(t, scenario.store.state.actions, 2) + assert.Len(t, scenario.store.state.idempotency, 2) + assert.Contains(t, scenario.store.state.idempotency, inMemoryModerationIdempotencyKey{removeRulesAdminDID, removeRulesInstanceDID, "k1"}) + assert.Contains(t, scenario.store.state.idempotency, inMemoryModerationIdempotencyKey{restoreRulesAdminDID, removeRulesInstanceDID, "k1"}) +} + +func TestModerationIdempotencyLiveKeyCapRejectsOnlyNewKeysForActor(t *testing.T) { + scenario := newRemoveRulesScenario() + scenario.service = moderation.NewService(scenario.reader, scenario.store, moderation.Config{ + InstanceDID: removeRulesInstanceDID, IdempotencyRetention: 24 * time.Hour, + MaxLiveIdempotencyKeys: 2, Now: func() time.Time { return scenario.store.now }, Purger: scenario.purger, + }) + first := scenario.request + first.IdempotencyKey = "k1" + second := seedIdempotencyComment(&scenario, "3ksecond") + second.IdempotencyKey = "k2" + third := seedIdempotencyComment(&scenario, "3kthird") + third.IdempotencyKey = "k3" + firstResult, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, first) + require.NoError(t, err) + require.NotNil(t, firstResult) + secondResult, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, second) + require.NoError(t, err) + require.NotNil(t, secondResult) + result, err := assertIdempotencyNoMutation(t, scenario, func() (*moderation.MutationResult, error) { + return scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, third) + }) + require.ErrorIs(t, err, moderation.ErrInvalidRequest) + assert.ErrorContains(t, err, "2") + assert.Nil(t, result) + assert.Len(t, scenario.store.state.idempotency, 2) + replayed, err := assertIdempotencyNoMutation(t, scenario, func() (*moderation.MutationResult, error) { + return scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, first) + }) + require.NoError(t, err) + assert.Equal(t, firstResult, replayed) + otherActor, err := scenario.service.RemoveContent(t.Context(), restoreRulesAdminDID, third) + require.NoError(t, err) + require.NotNil(t, otherActor) + assert.Equal(t, moderation.OutcomeApplied, otherActor.Outcome) + assert.Len(t, scenario.store.state.idempotency, 3) +} + +func TestModerationIdempotencyExpiredKeyIsReplaced(t *testing.T) { + scenario := newRemoveRulesScenario() + retention := 2 * time.Hour + scenario.service = moderation.NewService(scenario.reader, scenario.store, moderation.Config{ + InstanceDID: removeRulesInstanceDID, IdempotencyRetention: retention, + MaxLiveIdempotencyKeys: 2, Now: func() time.Time { return scenario.store.now }, Purger: scenario.purger, + }) + first, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, first) + require.NotNil(t, first.Action) + restored, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, moderation.RestoreContentRequest{ + ActionID: first.Action.ID, ReviewedSubject: &scenario.request.Subject, + ExpectedVersion: first.State.Version, IdempotencyKey: "restore-before-expiry", Reason: removeRulesSpam, + }) + require.NoError(t, err) + require.NotNil(t, restored) + key := inMemoryModerationIdempotencyKey{removeRulesAdminDID, removeRulesInstanceDID, scenario.request.IdempotencyKey} + expired := scenario.store.state.idempotency[key] + expired.ExpiresAt = scenario.store.now.Add(-time.Second) + scenario.store.state.idempotency[key] = expired + request := scenario.request + request.ExpectedVersion = restored.State.Version + request.Reason = removeRulesIllegal + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, request) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.Action) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + assert.NotEqual(t, first.Action.ID, result.Action.ID) + assert.Equal(t, "v2", restored.State.Version) + assert.Equal(t, "v3", result.State.Version) + replacement := scenario.store.state.idempotency[key] + assert.NotEqual(t, expired.Fingerprint, replacement.Fingerprint) + assert.Equal(t, *result, replacement.Result) + assert.Equal(t, scenario.store.now, replacement.CreatedAt) + assert.Equal(t, scenario.store.now.Add(retention), replacement.ExpiresAt) + assert.Len(t, scenario.store.state.idempotency, 2) +} + +func TestModerationIdempotencySavesAppliedAndUnchangedButNotRejected(t *testing.T) { + scenario := newRemoveRulesScenario() + retention := 90 * time.Minute + scenario.service = moderation.NewService(scenario.reader, scenario.store, moderation.Config{ + InstanceDID: removeRulesInstanceDID, IdempotencyRetention: retention, + MaxLiveIdempotencyKeys: 2, Now: func() time.Time { return scenario.store.now }, Purger: scenario.purger, + }) + applied, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, applied) + require.Equal(t, moderation.OutcomeApplied, applied.Outcome) + appliedKey := inMemoryModerationIdempotencyKey{removeRulesAdminDID, removeRulesInstanceDID, scenario.request.IdempotencyKey} + require.Len(t, scenario.store.state.idempotency, 1) + record := scenario.store.state.idempotency[appliedKey] + assert.Equal(t, removeRulesAdminDID, record.ActorDID) + assert.Equal(t, removeRulesInstanceDID, record.AuthorityDID) + assert.Equal(t, scenario.request.IdempotencyKey, record.Key) + assert.Equal(t, scenario.store.now, record.CreatedAt) + assert.Equal(t, scenario.store.now.Add(retention), record.ExpiresAt) + assert.Equal(t, *applied, record.Result) + assert.NotEmpty(t, record.Fingerprint) + + unchangedRequest := scenario.request + unchangedRequest.ExpectedVersion = applied.State.Version + unchangedRequest.IdempotencyKey = "redundant-removal" + unchanged, err := scenario.service.RemoveContent(t.Context(), restoreRulesAdminDID, unchangedRequest) + require.NoError(t, err) + require.NotNil(t, unchanged) + assert.Equal(t, moderation.OutcomeUnchanged, unchanged.Outcome) + assert.Nil(t, unchanged.Action) + unchangedKey := inMemoryModerationIdempotencyKey{restoreRulesAdminDID, removeRulesInstanceDID, unchangedRequest.IdempotencyKey} + require.Len(t, scenario.store.state.idempotency, 2) + assert.Equal(t, *unchanged, scenario.store.state.idempotency[unchangedKey].Result) + assert.Equal(t, scenario.store.now.Add(retention), scenario.store.state.idempotency[unchangedKey].ExpiresAt) + + rejectedRequest := scenario.request + rejectedRequest.IdempotencyKey = "stale-removal" + result, err := assertIdempotencyNoMutation(t, scenario, func() (*moderation.MutationResult, error) { + return scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, rejectedRequest) + }) + require.ErrorIs(t, err, moderation.ErrStateConflict) + assert.Nil(t, result) + assert.NotContains(t, scenario.store.state.idempotency, inMemoryModerationIdempotencyKey{removeRulesAdminDID, removeRulesInstanceDID, rejectedRequest.IdempotencyKey}) +} diff --git a/internal/core/moderation/interfaces.go b/internal/core/moderation/interfaces.go index 919ea8b..fc620aa 100644 --- a/internal/core/moderation/interfaces.go +++ b/internal/core/moderation/interfaces.go @@ -28,7 +28,9 @@ type CommentReader interface { GetByURI(ctx context.Context, uri string) (*comments.Comment, error) } -// Service is the moderation domain's read surface. +// Service is the moderation domain's read and mutation surface. type Service interface { GetSubjectState(ctx context.Context, subject string) (*SubjectState, error) + RemoveContent(ctx context.Context, actorDID string, request RemoveContentRequest) (*MutationResult, error) + RestoreContent(ctx context.Context, actorDID string, request RestoreContentRequest) (*MutationResult, error) } diff --git a/internal/core/moderation/media.go b/internal/core/moderation/media.go new file mode 100644 index 0000000..d2cc529 --- /dev/null +++ b/internal/core/moderation/media.go @@ -0,0 +1,93 @@ +package moderation + +import ( + "context" + "database/sql" + "log/slog" +) + +// MediaTransaction provides the operations needed to reconcile a comment's images. +type MediaTransaction interface { + ActiveRemoval(ctx context.Context, authorityDID, subjectURI string) (*Action, error) + ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + // InsertNewMediaBlocks inserts only blocks not already active for the action + // and returns the blocks it inserted. + InsertNewMediaBlocks(ctx context.Context, blocks []MediaBlock) ([]MediaBlock, error) +} + +// TransactionBinder binds media operations to a caller's transaction. +type TransactionBinder interface { + BindTransaction(tx *sql.Tx) MediaTransaction +} + +// MediaReconciler keeps media blocks in step with a removed subject's +// indexed images when a consumer rewrites the subject. +type MediaReconciler struct { + binder TransactionBinder + instanceDID string + purger MediaPurger +} + +// NewMediaReconciler builds a MediaReconciler. +func NewMediaReconciler(binder TransactionBinder, instanceDID string, purger MediaPurger) *MediaReconciler { + return &MediaReconciler{binder: binder, instanceDID: instanceDID, purger: purger} +} + +// ReconcileTx blocks images newly present on a subject with an active +// removal, inside the caller's transaction, and returns the new blocks. +func (r *MediaReconciler) ReconcileTx(ctx context.Context, tx *sql.Tx, subjectURI string) ([]MediaBlock, error) { + bound := r.binder.BindTransaction(tx) + action, err := bound.ActiveRemoval(ctx, r.instanceDID, subjectURI) + if err != nil || action == nil { + return nil, err + } + comment, err := bound.ReadIndexedComment(ctx, subjectURI) + if err != nil { + return nil, err + } + return bound.InsertNewMediaBlocks(ctx, imageMediaBlocks(comment, action)) +} + +// Purge removes cached bytes of newly blocked blobs after commit. +func (r *MediaReconciler) Purge(blocks []MediaBlock) { + purgeMediaBlocks(r.purger, blocks) +} + +func purgeMediaBlocks(purger MediaPurger, blocks []MediaBlock) { + if purger == nil { + return + } + for _, block := range blocks { + var err error + if block.OwnerDID == "" { + err = purger.PurgeBlob(block.BlobCID) + } else { + err = purger.PurgeOwnerBlob(block.OwnerDID, block.BlobCID) + } + // The block is committed and serving is refused, but the bytes stay on + // disk until the image proxy's blocked media sweep retries the purge. + if err != nil { + if block.OwnerDID == "" { + slog.Error("moderation media cache purge failed", "action_id", block.ActionID, "cid", block.BlobCID, "error", err) + } else { + slog.Error("moderation media cache purge failed", "action_id", block.ActionID, "did", block.OwnerDID, "cid", block.BlobCID, "error", err) + } + } + } +} + +func imageMediaBlocks(comment *IndexedComment, action *Action) []MediaBlock { + var blocks []MediaBlock + seen := make(map[string]bool) + for _, cid := range comment.ImageCIDs { + if seen[cid] { + continue + } + seen[cid] = true + blocks = append(blocks, MediaBlock{OwnerDID: comment.OwnerDID, BlobCID: cid, ActionID: action.ID}) + if action.Reason == illegalContentReason { + blocks = append(blocks, MediaBlock{BlobCID: cid, ActionID: action.ID}) + } + } + return blocks +} diff --git a/internal/core/moderation/mutation_validation.go b/internal/core/moderation/mutation_validation.go new file mode 100644 index 0000000..4202090 --- /dev/null +++ b/internal/core/moderation/mutation_validation.go @@ -0,0 +1,51 @@ +package moderation + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "unicode/utf8" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/rivo/uniseg" +) + +func validOpaqueField(value string) bool { + return len(value) > 0 && len(value) <= 128 && utf8.ValidString(value) +} + +func validateMutationFields(key, expectedVersion, reason, privateNote string) error { + if !validOpaqueField(key) || !validOpaqueField(expectedVersion) { + return fmt.Errorf("%w: idempotencyKey and expectedVersion must be 1-128 UTF-8 bytes", ErrInvalidRequest) + } + if !utf8.ValidString(privateNote) || len(privateNote) > 10000 || uniseg.GraphemeClusterCount(privateNote) > 1000 { + return fmt.Errorf("%w: privateNote exceeds its length limit", ErrInvalidRequest) + } + if reason == "" || len(reason) > 640 || !utf8.ValidString(reason) { + return fmt.Errorf("%w: reason must be 1-640 UTF-8 bytes", ErrInvalidRequest) + } + if _, ok := removeReasons[reason]; !ok { + return fmt.Errorf("%w: unsupported moderation reason", ErrUnsupportedReason) + } + return nil +} + +func validCommentStrongRef(ref StrongRef) bool { + uri, err := syntax.ParseATURI(ref.URI) + if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" || uri.Collection().String() != CommentCollection { + return false + } + _, err = syntax.ParseCID(ref.CID) + return err == nil +} + +// A fixed ordered tuple makes request fingerprints unambiguous even if a +// caller includes delimiters in opaque fields or private notes. +func mutationFingerprint(operation, actionID, subjectURI, subjectCID, expectedVersion, reason, privateNote string) string { + payload, _ := json.Marshal([8]string{ + operation, actionID, subjectURI, subjectCID, expectedVersion, reason, "", privateNote, + }) + hash := sha256.Sum256(payload) + return hex.EncodeToString(hash[:]) +} diff --git a/internal/core/moderation/remove.go b/internal/core/moderation/remove.go new file mode 100644 index 0000000..55c0d77 --- /dev/null +++ b/internal/core/moderation/remove.go @@ -0,0 +1,163 @@ +package moderation + +import ( + "context" + "errors" + "fmt" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +const illegalContentReason = "social.coves.moderation.defs#reasonIllegalContent" + +var removeReasons = map[string]struct{}{ + "social.coves.moderation.defs#reasonSpam": {}, + "social.coves.moderation.defs#reasonHarassment": {}, + "social.coves.moderation.defs#reasonDoxing": {}, + illegalContentReason: {}, + "social.coves.moderation.defs#reasonRuleViolation": {}, + "social.coves.moderation.defs#reasonModeratorDiscretion": {}, +} + +func validateRemoveRequest(request RemoveContentRequest) error { + uri, err := syntax.ParseATURI(request.Subject.URI) + if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" { + return fmt.Errorf("%w: expected a comment record URI with a DID authority", ErrInvalidSubject) + } + switch uri.Collection().String() { + case PostV2Collection, LegacyPostCollection: + return fmt.Errorf("%w: post removal is unsupported", ErrInvalidSubject) + case CommentCollection: + default: + return fmt.Errorf("%w: unsupported subject collection", ErrInvalidSubject) + } + if _, err := syntax.ParseCID(request.Subject.CID); err != nil { + return fmt.Errorf("%w: invalid subject CID", ErrInvalidRequest) + } + return validateMutationFields(request.IdempotencyKey, request.ExpectedVersion, request.Reason, request.PrivateNote) +} + +func (s *service) removeContent(ctx context.Context, actorDID string, request RemoveContentRequest) (*MutationResult, error) { + if err := validateRemoveRequest(request); err != nil { + return nil, err + } + now := time.Now() + if s.config.Now != nil { + now = s.config.Now() + } + fingerprint := mutationFingerprint(ActionRemove, "", request.Subject.URI, request.Subject.CID, request.ExpectedVersion, request.Reason, request.PrivateNote) + var result *MutationResult + var newlyBlocked []MediaBlock + var ruleError error + err := s.store.InTransaction(ctx, func(ctx context.Context, tx Transaction) error { + fail := func(err error) error { + ruleError = err + return err + } + unavailable := func(err error) error { return fmt.Errorf("%w: %w", ErrModerationUnavailable, err) } + if err := tx.LockActor(ctx, actorDID); err != nil { + return unavailable(err) + } + stored, err := tx.LiveIdempotencyRecord(ctx, actorDID, s.config.InstanceDID, request.IdempotencyKey, now) + if err != nil { + return unavailable(err) + } + if stored != nil { + if stored.Fingerprint != fingerprint { + return fail(ErrIdempotencyConflict) + } + copy := stored.Result + result = © + return nil + } + count, err := tx.CountLiveIdempotencyKeys(ctx, actorDID, now) + if err != nil { + return unavailable(err) + } + if count >= s.config.MaxLiveIdempotencyKeys { + return fail(fmt.Errorf("%w: live idempotency key limit %d reached", ErrInvalidRequest, s.config.MaxLiveIdempotencyKeys)) + } + version, err := tx.LockSubject(ctx, request.Subject.URI) + if err != nil { + return unavailable(err) + } + if request.ExpectedVersion != versionToken(version) { + return fail(ErrStateConflict) + } + comment, err := tx.ReadIndexedComment(ctx, request.Subject.URI) + if errors.Is(err, ErrSubjectNotIndexed) { + return fail(ErrSubjectNotFound) + } + if err != nil { + return unavailable(err) + } + if comment == nil { + return unavailable(errors.New("indexed comment missing")) + } + if !comment.AuthorDeleted && comment.CID != request.Subject.CID { + return fail(ErrContentChanged) + } + active, err := tx.ActiveRemoval(ctx, s.config.InstanceDID, request.Subject.URI) + if err != nil { + return unavailable(err) + } + recordState := RecordStatePresent + var current *StrongRef + if comment.AuthorDeleted { + recordState = RecordStateDeleted + } else { + current = &StrongRef{URI: comment.URI, CID: comment.CID} + } + if active != nil { + state, err := newSubjectState(request.Subject.URI, version, recordState, current, active, s.config.InstanceDID) + if err != nil { + return err + } + result = &MutationResult{Outcome: OutcomeUnchanged, State: state} + } else { + action, err := tx.InsertAction(ctx, Action{ + ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, + ScopeKind: ScopeInstance, SubjectURI: request.Subject.URI, + SubjectCollection: CommentCollection, SubjectCommunityDID: comment.CommunityDID, + ObservedCID: comment.CID, Action: ActionRemove, Reason: request.Reason, + PrivateNote: request.PrivateNote, Origin: OriginLocal, CreatedAt: now, + }) + if err != nil { + return unavailable(err) + } + if action == nil || action.ID == "" { + return unavailable(errors.New("action insert returned no identifier")) + } + if err := tx.SetRemovalDecision(ctx, s.config.InstanceDID, request.Subject.URI, action.ID, true); err != nil { + return unavailable(err) + } + if err := tx.SetSubjectVersion(ctx, request.Subject.URI, version+1); err != nil { + return unavailable(err) + } + newlyBlocked = imageMediaBlocks(comment, action) + if len(newlyBlocked) > 0 { + if err := tx.InsertMediaBlocks(ctx, newlyBlocked); err != nil { + return unavailable(err) + } + } + state, err := newSubjectState(request.Subject.URI, version+1, recordState, current, action, s.config.InstanceDID) + if err != nil { + return err + } + result = &MutationResult{Outcome: OutcomeApplied, State: state, Action: action} + } + return tx.SaveIdempotencyRecord(ctx, IdempotencyRecord{ + ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, Key: request.IdempotencyKey, + Fingerprint: fingerprint, Result: *result, CreatedAt: now, ExpiresAt: now.Add(s.config.IdempotencyRetention), + }) + }) + if err != nil { + if ruleError != nil && errors.Is(err, ruleError) { + return nil, ruleError + } + return nil, fmt.Errorf("%w: %w", ErrModerationUnavailable, err) + } + purgeMediaBlocks(s.config.Purger, newlyBlocked) + return result, nil +} diff --git a/internal/core/moderation/remove_rules_test.go b/internal/core/moderation/remove_rules_test.go new file mode 100644 index 0000000..93abeaa --- /dev/null +++ b/internal/core/moderation/remove_rules_test.go @@ -0,0 +1,260 @@ +package moderation_test + +import ( + "errors" + "strconv" + "testing" + "time" + + "Coves/internal/core/moderation" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + removeRulesAuthorDID = "did:plc:commentauthor" + removeRulesAdminDID = "did:plc:firstadmin" + removeRulesInstanceDID = "did:web:moderation.test" + removeRulesCommunityDID = "did:plc:community" + removeRulesURI = "at://did:plc:commentauthor/social.coves.community.comment/3kabc" + removeRulesCID = "bafyreicommentversion" + removeRulesFirstImage = "bafyreifirstimage" + removeRulesSecondImage = "bafyreisecondimage" + removeRulesSpam = "social.coves.moderation.defs#reasonSpam" + removeRulesIllegal = "social.coves.moderation.defs#reasonIllegalContent" +) + +type removeRulesOwnerPurge struct { + ownerDID string + blobCID string +} + +type removeRulesPurger struct { + ownerPurges []removeRulesOwnerPurge + blobPurges []string +} + +func (purger *removeRulesPurger) PurgeOwnerBlob(ownerDID, blobCID string) error { + purger.ownerPurges = append(purger.ownerPurges, removeRulesOwnerPurge{ownerDID, blobCID}) + return nil +} + +func (purger *removeRulesPurger) PurgeBlob(blobCID string) error { + purger.blobPurges = append(purger.blobPurges, blobCID) + return nil +} + +type removeRulesScenario struct { + store *inMemoryModerationStore + reader *fakeSubjectReader + purger *removeRulesPurger + service moderation.Service + request moderation.RemoveContentRequest +} + +func newRemoveRulesScenario() removeRulesScenario { + clock := time.Date(2026, time.September, 24, 12, 0, 0, 0, time.UTC) + store := newInMemoryModerationStore(clock) + store.state.indexedComments[removeRulesURI] = moderation.IndexedComment{ + URI: removeRulesURI, CID: removeRulesCID, OwnerDID: removeRulesAuthorDID, + CommunityDID: removeRulesCommunityDID, ImageCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + } + reader := &fakeSubjectReader{record: &moderation.IndexedRecord{URI: removeRulesURI, CID: removeRulesCID}} + purger := &removeRulesPurger{} + service := moderation.NewService(reader, store, moderation.Config{ + InstanceDID: removeRulesInstanceDID, IdempotencyRetention: 24 * time.Hour, + MaxLiveIdempotencyKeys: 1000, Now: func() time.Time { return clock }, Purger: purger, + }) + return removeRulesScenario{ + store: store, reader: reader, purger: purger, service: service, + request: moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: removeRulesURI, CID: removeRulesCID}, + ExpectedVersion: "v0", IdempotencyKey: "first-removal", Reason: removeRulesSpam, PrivateNote: "n", + }, + } +} + +func assertRemoveRulesNoWrites(t *testing.T, scenario removeRulesScenario) { + t.Helper() + assert.Empty(t, scenario.store.writeCalls) + assert.Empty(t, scenario.store.state.actions) + assert.Empty(t, scenario.store.state.activeRemovals) + assert.Empty(t, scenario.store.state.versions) + assert.Empty(t, scenario.store.state.mediaBlocks) + assert.Empty(t, scenario.store.state.idempotency) + assert.Empty(t, scenario.purger.ownerPurges) + assert.Empty(t, scenario.purger.blobPurges) +} + +func TestRemoveContentRules(t *testing.T) { + t.Run("applied removal records action, decision, version, two owned blocks and purges", func(t *testing.T) { + scenario := newRemoveRulesScenario() + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + action := result.Action + require.NotEmpty(t, action.ID) + assert.Equal(t, scenario.store.now, action.CreatedAt) + assert.Equal(t, moderation.ActionRemove, action.Action) + assert.Equal(t, removeRulesAdminDID, action.ActorDID) + assert.Equal(t, removeRulesInstanceDID, action.AuthorityDID) + assert.Equal(t, moderation.ScopeInstance, action.ScopeKind) + assert.Equal(t, removeRulesURI, action.SubjectURI) + assert.Equal(t, moderation.CommentCollection, action.SubjectCollection) + assert.Equal(t, removeRulesCommunityDID, action.SubjectCommunityDID) + assert.Equal(t, removeRulesCID, action.ObservedCID) + assert.Equal(t, removeRulesSpam, action.Reason) + assert.Equal(t, "n", action.PrivateNote) + assert.Equal(t, moderation.OriginLocal, action.Origin) + assert.NotEqual(t, "v0", result.State.Version) + assert.Equal(t, moderation.ModerationStateRemoved, result.State.Moderation.State) + assert.Equal(t, &moderation.ActionRef{ServiceDID: removeRulesInstanceDID, ActionID: action.ID}, result.State.LocalRemoval) + + require.Len(t, scenario.store.state.actions, 1) + assert.Equal(t, *action, scenario.store.state.actions[action.ID]) + require.Len(t, scenario.store.state.activeRemovals, 1) + assert.Equal(t, action.ID, scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, removeRulesURI}]) + assert.Equal(t, []string{"InsertAction", "SetRemovalDecision", "SetSubjectVersion", "InsertMediaBlocks", "SaveIdempotencyRecord"}, scenario.store.writeCalls) + stored, err := scenario.store.SubjectModeration(t.Context(), removeRulesInstanceDID, removeRulesURI) + require.NoError(t, err) + assert.Equal(t, "v"+strconv.FormatInt(stored.Version, 10), result.State.Version) + assert.Equal(t, action.ID, stored.ActiveRemoval.ID) + assert.Equal(t, map[moderation.MediaBlock]bool{ + {OwnerDID: removeRulesAuthorDID, BlobCID: removeRulesFirstImage, ActionID: action.ID}: true, + {OwnerDID: removeRulesAuthorDID, BlobCID: removeRulesSecondImage, ActionID: action.ID}: true, + }, scenario.store.state.mediaBlocks) + assert.Equal(t, []removeRulesOwnerPurge{ + {removeRulesAuthorDID, removeRulesFirstImage}, {removeRulesAuthorDID, removeRulesSecondImage}, + }, scenario.purger.ownerPurges) + assert.Empty(t, scenario.purger.blobPurges) + + following, err := scenario.service.GetSubjectState(t.Context(), removeRulesURI) + require.NoError(t, err) + require.NotNil(t, following) + assert.Equal(t, result.State.Version, following.Version, "the next state read must reflect the committed version") + assert.Equal(t, moderation.ModerationStateRemoved, following.Moderation.State) + assert.Equal(t, result.State.LocalRemoval, following.LocalRemoval) + }) + + t.Run("illegal content adds one owner-less block and purge per image", func(t *testing.T) { + scenario := newRemoveRulesScenario() + scenario.request.Reason = removeRulesIllegal + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + require.NotNil(t, result.Action) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + assert.Equal(t, map[moderation.MediaBlock]bool{ + {OwnerDID: removeRulesAuthorDID, BlobCID: removeRulesFirstImage, ActionID: result.Action.ID}: true, + {OwnerDID: removeRulesAuthorDID, BlobCID: removeRulesSecondImage, ActionID: result.Action.ID}: true, + {BlobCID: removeRulesFirstImage, ActionID: result.Action.ID}: true, + {BlobCID: removeRulesSecondImage, ActionID: result.Action.ID}: true, + }, scenario.store.state.mediaBlocks) + assert.Equal(t, []removeRulesOwnerPurge{ + {removeRulesAuthorDID, removeRulesFirstImage}, {removeRulesAuthorDID, removeRulesSecondImage}, + }, scenario.purger.ownerPurges) + assert.Equal(t, []string{removeRulesFirstImage, removeRulesSecondImage}, scenario.purger.blobPurges) + }) + + for _, test := range []struct { + name string + change func(*removeRulesScenario) + want error + }{ + {name: "stale version v7", change: func(s *removeRulesScenario) { s.request.ExpectedVersion = "v7" }, want: moderation.ErrStateConflict}, + {name: "unparseable version banana", change: func(s *removeRulesScenario) { s.request.ExpectedVersion = "banana" }, want: moderation.ErrStateConflict}, + {name: "changed indexed CID", change: func(s *removeRulesScenario) { s.request.Subject.CID = "bafyreistalecommentcid" }, want: moderation.ErrContentChanged}, + {name: "never indexed subject", change: func(s *removeRulesScenario) { + delete(s.store.state.indexedComments, removeRulesURI) + s.reader.record = nil + s.reader.err = moderation.ErrSubjectNotIndexed + }, want: moderation.ErrSubjectNotFound}, + } { + t.Run(test.name+" changes nothing and never purges", func(t *testing.T) { + scenario := newRemoveRulesScenario() + test.change(&scenario) + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.ErrorIs(t, err, test.want) + assert.Nil(t, result) + assertRemoveRulesNoWrites(t, scenario) + }) + } + + t.Run("author-deleted comment removes despite changed strongRef CID", func(t *testing.T) { + scenario := newRemoveRulesScenario() + comment := scenario.store.state.indexedComments[removeRulesURI] + comment.AuthorDeleted = true + scenario.store.state.indexedComments[removeRulesURI] = comment + scenario.reader.record.Deleted = true + scenario.request.Subject.CID = "bafyreidifferentcomment" + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + assert.Equal(t, removeRulesCID, result.Action.ObservedCID) + assert.Equal(t, moderation.RecordStateDeleted, result.State.RecordState) + assert.Nil(t, result.State.CurrentSubject) + require.Len(t, scenario.store.state.actions, 1) + }) + + t.Run("redundant removal by another admin keeps action and version", func(t *testing.T) { + scenario := newRemoveRulesScenario() + prior := moderation.Action{ + ID: "earlier-removal", Action: moderation.ActionRemove, AuthorityDID: removeRulesInstanceDID, + SubjectURI: removeRulesURI, SubjectCollection: moderation.CommentCollection, + } + scenario.store.state.actions[prior.ID] = prior + scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, removeRulesURI}] = prior.ID + scenario.store.state.versions[removeRulesURI] = 1 + scenario.request.ExpectedVersion = "v1" + result, err := scenario.service.RemoveContent(t.Context(), "did:plc:secondadmin", scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, moderation.OutcomeUnchanged, result.Outcome) + assert.Nil(t, result.Action) + assert.Equal(t, "v1", result.State.Version) + assert.Equal(t, moderation.ModerationStateRemoved, result.State.Moderation.State) + assert.Equal(t, &moderation.ActionRef{ServiceDID: removeRulesInstanceDID, ActionID: prior.ID}, result.State.LocalRemoval) + assert.Equal(t, map[string]moderation.Action{prior.ID: prior}, scenario.store.state.actions) + assert.Equal(t, map[string]int64{removeRulesURI: 1}, scenario.store.state.versions) + assert.Equal(t, []string{"SaveIdempotencyRecord"}, scenario.store.writeCalls) + assert.Empty(t, scenario.store.state.mediaBlocks) + assert.Empty(t, scenario.purger.ownerPurges) + assert.Empty(t, scenario.purger.blobPurges) + }) + + for _, test := range []struct { + name string + inject func(*inMemoryModerationStore, error) + wantCalls []string + }{ + {name: "InsertAction failure rolls back", inject: func(store *inMemoryModerationStore, err error) { + store.failInsertAction = err + }, wantCalls: []string{"InsertAction"}}, + {name: "SetRemovalDecision failure rolls back inserted action", inject: func(store *inMemoryModerationStore, err error) { + store.failSetRemovalDecision = err + }, wantCalls: []string{"InsertAction", "SetRemovalDecision"}}, + } { + t.Run(test.name+" and does not purge", func(t *testing.T) { + scenario := newRemoveRulesScenario() + storageError := errors.New("injected store failure") + test.inject(scenario.store, storageError) + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.ErrorIs(t, err, moderation.ErrModerationUnavailable) + assert.ErrorIs(t, err, storageError) + assert.Nil(t, result) + assert.Equal(t, test.wantCalls, scenario.store.writeCalls) + assert.Empty(t, scenario.store.state.actions) + assert.Empty(t, scenario.store.state.activeRemovals) + assert.Empty(t, scenario.store.state.versions) + assert.Empty(t, scenario.store.state.idempotency) + assert.Empty(t, scenario.store.state.mediaBlocks) + assert.Empty(t, scenario.purger.ownerPurges) + assert.Empty(t, scenario.purger.blobPurges) + }) + } +} diff --git a/internal/core/moderation/remove_validation_test.go b/internal/core/moderation/remove_validation_test.go new file mode 100644 index 0000000..ec83006 --- /dev/null +++ b/internal/core/moderation/remove_validation_test.go @@ -0,0 +1,119 @@ +package moderation_test + +import ( + "strings" + "testing" + "time" + + "Coves/internal/core/moderation" + + "github.com/rivo/uniseg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestRemoveContentRejectsUnsupportedSubjectsReasonsAndOversizedInputs(t *testing.T) { + const ( + actorDID = "did:plc:moderationadmin" + authorDID = "did:plc:commentauthor" + instanceDID = "did:web:moderation.test" + commentCID = "bafyreicommentvalidation" + validReason = "social.coves.moderation.defs#reasonSpam" + ) + clock := time.Date(2026, time.September, 24, 12, 0, 0, 0, time.UTC) + tooManyGraphemes := strings.Repeat("👍🏽", 1001) + tooManyBytes := strings.Repeat("👨‍👩‍👧‍👦", 400) + "x" + require.Equal(t, 1001, uniseg.GraphemeClusterCount(tooManyGraphemes)) + require.LessOrEqual(t, len(tooManyGraphemes), 10000) + require.Equal(t, 10001, len(tooManyBytes)) + require.LessOrEqual(t, uniseg.GraphemeClusterCount(tooManyBytes), 1000) + + for _, test := range []struct { + name string + collection string + change func(*moderation.RemoveContentRequest) + want error + }{ + {name: "author-owned post removal is unsupported", collection: moderation.PostV2Collection, want: moderation.ErrInvalidSubject}, + {name: "legacy post removal is unsupported", collection: moderation.LegacyPostCollection, want: moderation.ErrInvalidSubject}, + {name: "unrecognized reason token", change: func(request *moderation.RemoveContentRequest) { + request.Reason = "social.coves.moderation.defs#reasonCsam" + }, want: moderation.ErrUnsupportedReason}, + {name: "unqualified reason", change: func(request *moderation.RemoveContentRequest) { + request.Reason = "spam" + }, want: moderation.ErrUnsupportedReason}, + {name: "private note exceeds grapheme limit", change: func(request *moderation.RemoveContentRequest) { + request.PrivateNote = tooManyGraphemes + }, want: moderation.ErrInvalidRequest}, + {name: "private note exceeds byte limit", change: func(request *moderation.RemoveContentRequest) { + request.PrivateNote = tooManyBytes + }, want: moderation.ErrInvalidRequest}, + {name: "idempotency key exceeds 128 bytes", change: func(request *moderation.RemoveContentRequest) { + request.IdempotencyKey = strings.Repeat("k", 129) + }, want: moderation.ErrInvalidRequest}, + {name: "expected version is required", change: func(request *moderation.RemoveContentRequest) { + request.ExpectedVersion = "" + }, want: moderation.ErrInvalidRequest}, + } { + t.Run(test.name, func(t *testing.T) { + collection := test.collection + if collection == "" { + collection = moderation.CommentCollection + } + uri := "at://" + authorDID + "/" + collection + "/3kabc" + reader := &fakeSubjectReader{record: &moderation.IndexedRecord{URI: uri, CID: commentCID}} + store := newInMemoryModerationStore(clock) + store.state.indexedComments[uri] = moderation.IndexedComment{ + URI: uri, CID: commentCID, OwnerDID: authorDID, CommunityDID: "did:plc:community", + } + service := moderation.NewService(reader, store, moderation.Config{ + InstanceDID: instanceDID, IdempotencyRetention: 24 * time.Hour, + MaxLiveIdempotencyKeys: 1000, Now: func() time.Time { return clock }, + }) + request := moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: uri, CID: commentCID}, + ExpectedVersion: "v0", IdempotencyKey: "remove-validation", Reason: validReason, + } + if test.change != nil { + test.change(&request) + } + + result, err := service.RemoveContent(t.Context(), actorDID, request) + require.ErrorIs(t, err, test.want) + assert.Nil(t, result) + if test.want == moderation.ErrInvalidSubject { + assert.Contains(t, err.Error(), "post", "post removal must be explicitly unsupported") + } + assert.Empty(t, store.writeCalls, "validation must not attempt a mutation") + assert.Empty(t, store.state.actions) + assert.Empty(t, store.state.activeRemovals) + assert.Empty(t, store.state.versions) + assert.Empty(t, store.state.idempotency) + }) + } + + t.Run("the exact note and key limits allow a seeded comment removal", func(t *testing.T) { + uri := "at://" + authorDID + "/" + moderation.CommentCollection + "/3kabc" + reader := &fakeSubjectReader{record: &moderation.IndexedRecord{URI: uri, CID: commentCID}} + store := newInMemoryModerationStore(clock) + store.state.indexedComments[uri] = moderation.IndexedComment{ + URI: uri, CID: commentCID, OwnerDID: authorDID, CommunityDID: "did:plc:community", + } + service := moderation.NewService(reader, store, moderation.Config{ + InstanceDID: instanceDID, IdempotencyRetention: 24 * time.Hour, + MaxLiveIdempotencyKeys: 1000, Now: func() time.Time { return clock }, + }) + note := strings.Repeat("👍🏽", 1000) + key := strings.Repeat("k", 128) + require.Equal(t, 1000, uniseg.GraphemeClusterCount(note)) + require.Len(t, []byte(key), 128) + + result, err := service.RemoveContent(t.Context(), actorDID, moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: uri, CID: commentCID}, + ExpectedVersion: "v0", IdempotencyKey: key, Reason: validReason, PrivateNote: note, + }) + require.NotErrorIs(t, err, moderation.ErrInvalidRequest) + require.NoError(t, err) + require.NotNil(t, result, "a valid seeded request must reach the mutation path") + }) +} diff --git a/internal/core/moderation/restore.go b/internal/core/moderation/restore.go new file mode 100644 index 0000000..a86e204 --- /dev/null +++ b/internal/core/moderation/restore.go @@ -0,0 +1,162 @@ +package moderation + +import ( + "context" + "errors" + "fmt" + "time" +) + +func validateRestoreRequest(request RestoreContentRequest) error { + if !validOpaqueField(request.ActionID) { + return fmt.Errorf("%w: actionId must be 1-128 UTF-8 bytes", ErrInvalidRequest) + } + if err := validateMutationFields(request.IdempotencyKey, request.ExpectedVersion, request.Reason, request.PrivateNote); err != nil { + return err + } + if request.ReviewedSubject != nil && !validCommentStrongRef(*request.ReviewedSubject) { + return fmt.Errorf("%w: reviewedSubject must be a comment strongRef", ErrInvalidRequest) + } + return nil +} + +func (s *service) restoreContent(ctx context.Context, actorDID string, request RestoreContentRequest) (*MutationResult, error) { + if err := validateRestoreRequest(request); err != nil { + return nil, err + } + now := time.Now() + if s.config.Now != nil { + now = s.config.Now() + } + var reviewedURI, reviewedCID string + if request.ReviewedSubject != nil { + reviewedURI = request.ReviewedSubject.URI + reviewedCID = request.ReviewedSubject.CID + } + fingerprint := mutationFingerprint(ActionRestore, request.ActionID, reviewedURI, reviewedCID, request.ExpectedVersion, request.Reason, request.PrivateNote) + var result *MutationResult + var ruleError error + err := s.store.InTransaction(ctx, func(ctx context.Context, tx Transaction) error { + fail := func(err error) error { + ruleError = err + return err + } + unavailable := func(err error) error { return fmt.Errorf("%w: %w", ErrModerationUnavailable, err) } + if err := tx.LockActor(ctx, actorDID); err != nil { + return unavailable(err) + } + stored, err := tx.LiveIdempotencyRecord(ctx, actorDID, s.config.InstanceDID, request.IdempotencyKey, now) + if err != nil { + return unavailable(err) + } + if stored != nil { + if stored.Fingerprint != fingerprint { + return fail(ErrIdempotencyConflict) + } + copy := stored.Result + result = © + return nil + } + count, err := tx.CountLiveIdempotencyKeys(ctx, actorDID, now) + if err != nil { + return unavailable(err) + } + if count >= s.config.MaxLiveIdempotencyKeys { + return fail(fmt.Errorf("%w: live idempotency key limit %d reached", ErrInvalidRequest, s.config.MaxLiveIdempotencyKeys)) + } + removal, err := tx.GetAction(ctx, request.ActionID) + if errors.Is(err, ErrDecisionNotFound) { + return fail(ErrDecisionNotFound) + } + if err != nil { + return unavailable(err) + } + if removal == nil { + return unavailable(errors.New("action lookup returned no action")) + } + if removal.Action != ActionRemove || removal.AuthorityDID != s.config.InstanceDID || removal.ScopeKind != ScopeInstance { + return fail(ErrInvalidDecision) + } + version, err := tx.LockSubject(ctx, removal.SubjectURI) + if err != nil { + return unavailable(err) + } + active, err := tx.ActiveRemoval(ctx, s.config.InstanceDID, removal.SubjectURI) + if err != nil { + return unavailable(err) + } + if active == nil || active.ID != removal.ID { + return fail(ErrInvalidDecision) + } + if request.ExpectedVersion != versionToken(version) { + return fail(ErrStateConflict) + } + if request.ReviewedSubject != nil && reviewedURI != removal.SubjectURI { + return fail(fmt.Errorf("%w: reviewedSubject URI differs from removal subject", ErrInvalidRequest)) + } + comment, err := tx.ReadIndexedComment(ctx, removal.SubjectURI) + if err != nil && !errors.Is(err, ErrSubjectNotIndexed) { + return unavailable(err) + } + recordState := RecordStateUnavailable + var current *StrongRef + var observedCID string + if err == nil { + if comment == nil { + return unavailable(errors.New("indexed comment lookup returned no comment")) + } + observedCID = comment.CID + if comment.AuthorDeleted { + recordState = RecordStateDeleted + } else { + recordState = RecordStatePresent + current = &StrongRef{URI: comment.URI, CID: comment.CID} + if request.ReviewedSubject == nil { + return fail(fmt.Errorf("%w: reviewedSubject is required for a present comment", ErrInvalidRequest)) + } + if reviewedCID != comment.CID { + return fail(ErrContentChanged) + } + } + } + action, err := tx.InsertAction(ctx, Action{ + ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, + ScopeKind: ScopeInstance, SubjectURI: removal.SubjectURI, + SubjectCollection: removal.SubjectCollection, SubjectCommunityDID: removal.SubjectCommunityDID, + ObservedCID: observedCID, Action: ActionRestore, Reason: request.Reason, + PrivateNote: request.PrivateNote, ReversesActionID: removal.ID, + Origin: OriginLocal, CreatedAt: now, + }) + if err != nil { + return unavailable(err) + } + if action == nil || action.ID == "" { + return unavailable(errors.New("action insert returned no identifier")) + } + if err := tx.SetRemovalDecision(ctx, s.config.InstanceDID, removal.SubjectURI, removal.ID, false); err != nil { + return unavailable(err) + } + if err := tx.SetSubjectVersion(ctx, removal.SubjectURI, version+1); err != nil { + return unavailable(err) + } + if err := tx.DeactivateMediaBlocks(ctx, removal.ID); err != nil { + return unavailable(err) + } + state, err := newSubjectState(removal.SubjectURI, version+1, recordState, current, nil, s.config.InstanceDID) + if err != nil { + return err + } + result = &MutationResult{Outcome: OutcomeApplied, State: state, Action: action} + return tx.SaveIdempotencyRecord(ctx, IdempotencyRecord{ + ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, Key: request.IdempotencyKey, + Fingerprint: fingerprint, Result: *result, CreatedAt: now, ExpiresAt: now.Add(s.config.IdempotencyRetention), + }) + }) + if err != nil { + if ruleError != nil && errors.Is(err, ruleError) { + return nil, ruleError + } + return nil, fmt.Errorf("%w: %w", ErrModerationUnavailable, err) + } + return result, nil +} diff --git a/internal/core/moderation/restore_rules_test.go b/internal/core/moderation/restore_rules_test.go new file mode 100644 index 0000000..30c3981 --- /dev/null +++ b/internal/core/moderation/restore_rules_test.go @@ -0,0 +1,248 @@ +package moderation_test + +import ( + "strings" + "testing" + + "Coves/internal/core/moderation" + + "github.com/rivo/uniseg" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const restoreRulesAdminDID = "did:plc:secondadmin" + +type restoreRulesScenario struct { + removeRulesScenario + removal moderation.Action + request moderation.RestoreContentRequest +} + +func newRestoreRulesScenario(t *testing.T) restoreRulesScenario { + t.Helper() + scenario := newRemoveRulesScenario() + removed, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, removed) + require.NotNil(t, removed.Action) + scenario.store.writeCalls = nil + return restoreRulesScenario{ + removeRulesScenario: scenario, + removal: *removed.Action, + request: moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &moderation.StrongRef{URI: removeRulesURI, CID: removeRulesCID}, + ExpectedVersion: removed.State.Version, IdempotencyKey: "first-restore", + Reason: removeRulesSpam, PrivateNote: "reviewed", + }, + } +} + +func assertRestoreRulesRejected(t *testing.T, scenario restoreRulesScenario, want error) { + t.Helper() + before := scenario.store.state.copy() + ownerPurges := append([]removeRulesOwnerPurge(nil), scenario.purger.ownerPurges...) + blobPurges := append([]string(nil), scenario.purger.blobPurges...) + result, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, scenario.request) + require.ErrorIs(t, err, want) + assert.Nil(t, result) + assert.Equal(t, before, scenario.store.state, "rejected restore must not commit any change") + assert.Equal(t, ownerPurges, scenario.purger.ownerPurges) + assert.Equal(t, blobPurges, scenario.purger.blobPurges) +} + +func TestRestoreContentValidatesRequestWithoutChangingActiveRemoval(t *testing.T) { + tooManyGraphemes := strings.Repeat("👍🏽", 1001) + tooManyBytes := strings.Repeat("👨‍👩‍👧‍👦", 400) + "x" + require.Equal(t, 1001, uniseg.GraphemeClusterCount(tooManyGraphemes)) + require.Equal(t, 10001, len(tooManyBytes)) + for _, test := range []struct { + name string + change func(*moderation.RestoreContentRequest) + want error + }{ + {"missing action ID", func(r *moderation.RestoreContentRequest) { r.ActionID = "" }, moderation.ErrInvalidRequest}, + {"long action ID", func(r *moderation.RestoreContentRequest) { r.ActionID = strings.Repeat("a", 129) }, moderation.ErrInvalidRequest}, + {"missing key", func(r *moderation.RestoreContentRequest) { r.IdempotencyKey = "" }, moderation.ErrInvalidRequest}, + {"long key", func(r *moderation.RestoreContentRequest) { r.IdempotencyKey = strings.Repeat("k", 129) }, moderation.ErrInvalidRequest}, + {"missing version", func(r *moderation.RestoreContentRequest) { r.ExpectedVersion = "" }, moderation.ErrInvalidRequest}, + {"long version", func(r *moderation.RestoreContentRequest) { r.ExpectedVersion = strings.Repeat("v", 129) }, moderation.ErrInvalidRequest}, + {"missing reason", func(r *moderation.RestoreContentRequest) { r.Reason = "" }, moderation.ErrInvalidRequest}, + {"long reason", func(r *moderation.RestoreContentRequest) { r.Reason = strings.Repeat("r", 641) }, moderation.ErrInvalidRequest}, + {"unknown reason", func(r *moderation.RestoreContentRequest) { r.Reason = "social.coves.moderation.defs#reasonCsam" }, moderation.ErrUnsupportedReason}, + {"too many note graphemes", func(r *moderation.RestoreContentRequest) { r.PrivateNote = tooManyGraphemes }, moderation.ErrInvalidRequest}, + {"too many note bytes", func(r *moderation.RestoreContentRequest) { r.PrivateNote = tooManyBytes }, moderation.ErrInvalidRequest}, + {"malformed reviewed URI", func(r *moderation.RestoreContentRequest) { r.ReviewedSubject.URI = "not an at URI" }, moderation.ErrInvalidRequest}, + {"malformed reviewed CID", func(r *moderation.RestoreContentRequest) { r.ReviewedSubject.CID = "not a CID" }, moderation.ErrInvalidRequest}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newRestoreRulesScenario(t) + test.change(&scenario.request) + assertRestoreRulesRejected(t, scenario, test.want) + }) + } +} + +func TestRestoreContentRejectsUnknownWrongAuthorityOrInactiveAction(t *testing.T) { + for _, test := range []struct { + name string + change func(*restoreRulesScenario) + want error + }{ + {"unknown action", func(s *restoreRulesScenario) { s.request.ActionID = "missing-action" }, moderation.ErrDecisionNotFound}, + {"label action from instance", func(s *restoreRulesScenario) { + label := s.removal + label.ID = "label-action" + label.Action = "label" + s.store.state.actions[label.ID] = label + s.request.ActionID = label.ID + }, moderation.ErrInvalidDecision}, + {"foreign authority removal", func(s *restoreRulesScenario) { + foreign := s.removal + foreign.ID = "foreign-removal" + foreign.AuthorityDID = "did:web:other.example" + s.store.state.actions[foreign.ID] = foreign + s.store.state.activeRemovals[inMemoryModerationDecisionKey{foreign.AuthorityDID, removeRulesURI}] = foreign.ID + s.request.ActionID = foreign.ID + }, moderation.ErrInvalidDecision}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newRestoreRulesScenario(t) + test.change(&scenario) + assertRestoreRulesRejected(t, scenario, test.want) + }) + } + + t.Run("earlier removal cannot reverse a newer active removal", func(t *testing.T) { + scenario := newRestoreRulesScenario(t) + first := scenario.removal + restored, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, restored) + scenario.store.writeCalls = nil + next := scenario.removeRulesScenario.request + next.ExpectedVersion = restored.State.Version + next.IdempotencyKey = "second-removal" + removedAgain, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, next) + require.NoError(t, err) + require.NotNil(t, removedAgain) + require.NotNil(t, removedAgain.Action) + scenario.store.writeCalls = nil + scenario.request.ActionID = first.ID + scenario.request.ExpectedVersion = removedAgain.State.Version + scenario.request.IdempotencyKey = "restore-earlier-removal" + assertRestoreRulesRejected(t, scenario, moderation.ErrInvalidDecision) + assert.Equal(t, removedAgain.Action.ID, scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, removeRulesURI}]) + }) +} + +func TestRestoreContentRequiresReviewOfPresentIndexedComment(t *testing.T) { + for _, test := range []struct { + name string + change func(*restoreRulesScenario) + want error + }{ + {"missing reviewed subject", func(s *restoreRulesScenario) { s.request.ReviewedSubject = nil }, moderation.ErrInvalidRequest}, + {"reviewed URI differs from decision subject", func(s *restoreRulesScenario) { + s.request.ReviewedSubject.URI = "at://did:plc:commentauthor/social.coves.community.comment/3kother" + }, moderation.ErrInvalidRequest}, + {"indexed CID changed after review", func(s *restoreRulesScenario) { + comment := s.store.state.indexedComments[removeRulesURI] + comment.CID = "bafyreinewcommentversion" + s.store.state.indexedComments[removeRulesURI] = comment + s.reader.record.CID = comment.CID + }, moderation.ErrContentChanged}, + {"stale version", func(s *restoreRulesScenario) { s.request.ExpectedVersion = "v0" }, moderation.ErrStateConflict}, + {"unparseable version", func(s *restoreRulesScenario) { s.request.ExpectedVersion = "banana" }, moderation.ErrStateConflict}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newRestoreRulesScenario(t) + test.change(&scenario) + assertRestoreRulesRejected(t, scenario, test.want) + }) + } +} + +func TestRestoreContentRejectsReviewedURIForAuthorDeletedComment(t *testing.T) { + scenario := newRestoreRulesScenario(t) + comment := scenario.store.state.indexedComments[removeRulesURI] + comment.AuthorDeleted = true + scenario.store.state.indexedComments[removeRulesURI] = comment + scenario.reader.record.Deleted = true + scenario.request.ReviewedSubject.URI = "at://did:plc:commentauthor/social.coves.community.comment/3kother" + assertRestoreRulesRejected(t, scenario, moderation.ErrInvalidRequest) +} + +func TestRestoreContentAllowsMissingReviewForDeletedOrPurgedComment(t *testing.T) { + for _, test := range []struct { + name string + change func(*restoreRulesScenario) + want moderation.RecordState + }{ + {"author-deleted", func(s *restoreRulesScenario) { + comment := s.store.state.indexedComments[removeRulesURI] + comment.AuthorDeleted = true + s.store.state.indexedComments[removeRulesURI] = comment + s.reader.record.Deleted = true + }, moderation.RecordStateDeleted}, + {"purged indexed row", func(s *restoreRulesScenario) { + delete(s.store.state.indexedComments, removeRulesURI) + s.reader.record = nil + s.reader.err = moderation.ErrSubjectNotIndexed + }, moderation.RecordStateUnavailable}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newRestoreRulesScenario(t) + test.change(&scenario) + scenario.request.ReviewedSubject = nil + result, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + assert.Equal(t, test.want, result.State.RecordState) + assert.Nil(t, result.State.CurrentSubject) + assert.Equal(t, moderation.ModerationStateClear, result.State.Moderation.State) + }) + } +} + +func TestRestoreContentAppliesInverseWithoutChangingOriginalAction(t *testing.T) { + scenario := newRestoreRulesScenario(t) + otherBlock := moderation.MediaBlock{OwnerDID: "did:plc:other", BlobCID: removeRulesFirstImage, ActionID: "unrelated-action"} + scenario.store.state.mediaBlocks[otherBlock] = true + priorVersion := scenario.store.state.versions[removeRulesURI] + result, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + action := result.Action + assert.NotEqual(t, scenario.removal.ID, action.ID) + assert.Equal(t, moderation.ActionRestore, action.Action) + assert.Equal(t, restoreRulesAdminDID, action.ActorDID) + assert.Equal(t, removeRulesInstanceDID, action.AuthorityDID) + assert.Equal(t, scenario.removal.ID, action.ReversesActionID) + assert.Equal(t, removeRulesSpam, action.Reason) + assert.Equal(t, "reviewed", action.PrivateNote) + assert.Equal(t, moderation.OriginLocal, action.Origin) + assert.Equal(t, moderation.ScopeInstance, action.ScopeKind) + assert.Equal(t, removeRulesURI, action.SubjectURI) + assert.Equal(t, moderation.CommentCollection, action.SubjectCollection) + assert.Equal(t, scenario.removal, scenario.store.state.actions[scenario.removal.ID], "the removal log row is immutable") + assert.Equal(t, *action, scenario.store.state.actions[action.ID]) + assert.Empty(t, scenario.store.state.activeRemovals) + assert.Equal(t, priorVersion+1, scenario.store.state.versions[removeRulesURI]) + assert.Equal(t, "v2", result.State.Version) + assert.Equal(t, moderation.ModerationStateClear, result.State.Moderation.State) + assert.Nil(t, result.State.LocalRemoval) + assert.Equal(t, []string{"InsertAction", "SetRemovalDecision", "SetSubjectVersion", "DeactivateMediaBlocks", "SaveIdempotencyRecord"}, scenario.store.writeCalls) + require.Len(t, scenario.store.state.mediaBlocks, 3) + for block, active := range scenario.store.state.mediaBlocks { + assert.Equal(t, block.ActionID != scenario.removal.ID, active, "only the reversed action's blocks deactivate") + } + assert.True(t, scenario.store.state.mediaBlocks[otherBlock]) + following, err := scenario.service.GetSubjectState(t.Context(), removeRulesURI) + require.NoError(t, err) + require.NotNil(t, following) + assert.Equal(t, result.State, *following) +} diff --git a/internal/core/moderation/service.go b/internal/core/moderation/service.go index 4113e69..91481b2 100644 --- a/internal/core/moderation/service.go +++ b/internal/core/moderation/service.go @@ -8,13 +8,23 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" ) -// NewService builds the moderation read service over a SubjectReader. -func NewService(reader SubjectReader) Service { - return &service{reader: reader} +// NewService builds the moderation service. +func NewService(reader SubjectReader, store Store, config Config) Service { + return &service{reader: reader, store: store, config: config} } type service struct { reader SubjectReader + store Store + config Config +} + +func (s *service) RemoveContent(ctx context.Context, actorDID string, request RemoveContentRequest) (*MutationResult, error) { + return s.removeContent(ctx, actorDID, request) +} + +func (s *service) RestoreContent(ctx context.Context, actorDID string, request RestoreContentRequest) (*MutationResult, error) { + return s.restoreContent(ctx, actorDID, request) } func (s *service) GetSubjectState(ctx context.Context, subject string) (*SubjectState, error) { @@ -23,27 +33,33 @@ func (s *service) GetSubjectState(ctx context.Context, subject string) (*Subject return nil, fmt.Errorf("%w: expected a record URI with a DID authority and supported collection", ErrInvalidSubject) } - state := &SubjectState{ - Subject: subject, - Version: InitialVersion, - Moderation: ModerationView{State: ModerationStateClear}, - } record, err := s.reader.ReadSubject(ctx, subject) - if errors.Is(err, ErrSubjectNotIndexed) { - state.RecordState = RecordStateUnavailable - return state, nil + if err != nil && !errors.Is(err, ErrSubjectNotIndexed) { + return nil, fmt.Errorf("%w: %w", ErrModerationUnavailable, err) } + if err == nil && record == nil { + return nil, fmt.Errorf("%w: subject reader returned no record", ErrModerationUnavailable) + } + stored, err := s.store.SubjectModeration(ctx, s.config.InstanceDID, subject) if err != nil { return nil, fmt.Errorf("%w: %w", ErrModerationUnavailable, err) } - if record == nil { - return nil, fmt.Errorf("%w: subject reader returned no record", ErrModerationUnavailable) + if stored == nil { + return nil, fmt.Errorf("%w: subject store returned no state", ErrModerationUnavailable) } - if record.Deleted { - state.RecordState = RecordStateDeleted - } else { - state.RecordState = RecordStatePresent - state.CurrentSubject = &StrongRef{URI: record.URI, CID: record.CID} + recordState := RecordStateUnavailable + var current *StrongRef + if record != nil { + if record.Deleted { + recordState = RecordStateDeleted + } else { + recordState = RecordStatePresent + current = &StrongRef{URI: record.URI, CID: record.CID} + } + } + state, err := newSubjectState(subject, stored.Version, recordState, current, stored.ActiveRemoval, s.config.InstanceDID) + if err != nil { + return nil, err } - return state, nil + return &state, nil } diff --git a/internal/core/moderation/service_test.go b/internal/core/moderation/service_test.go index e7e61e3..5f73bff 100644 --- a/internal/core/moderation/service_test.go +++ b/internal/core/moderation/service_test.go @@ -16,6 +16,20 @@ type fakeSubjectReader struct { err error } +type fakeSubjectStore struct{} + +func (fakeSubjectStore) InTransaction(_ context.Context, _ func(context.Context, moderation.Transaction) error) error { + return nil +} + +func (fakeSubjectStore) SubjectModeration(_ context.Context, _, _ string) (*moderation.SubjectModeration, error) { + return &moderation.SubjectModeration{Version: 0}, nil +} + +func newSubjectStateTestService(reader moderation.SubjectReader) moderation.Service { + return moderation.NewService(reader, fakeSubjectStore{}, moderation.Config{InstanceDID: "did:web:test.coves.social"}) +} + func (reader *fakeSubjectReader) ReadSubject(_ context.Context, uri string) (*moderation.IndexedRecord, error) { reader.calls = append(reader.calls, uri) return reader.record, reader.err @@ -45,7 +59,7 @@ func TestGetSubjectStateRejectsInvalidSubjectsBeforeReading(t *testing.T) { } { t.Run(subject, func(t *testing.T) { reader := &fakeSubjectReader{} - state, err := moderation.NewService(reader).GetSubjectState(t.Context(), subject) + state, err := newSubjectStateTestService(reader).GetSubjectState(t.Context(), subject) assert.ErrorIs(t, err, moderation.ErrInvalidSubject) assert.Nil(t, state) assert.Empty(t, reader.calls, "invalid subjects must not reach the reader") @@ -57,7 +71,7 @@ func TestGetSubjectStateNeverIndexed(t *testing.T) { uri := "at://did:plc:neverindexed/social.coves.community.postv2/3kabc" reader := &fakeSubjectReader{err: moderation.ErrSubjectNotIndexed} - state, err := moderation.NewService(reader).GetSubjectState(t.Context(), uri) + state, err := newSubjectStateTestService(reader).GetSubjectState(t.Context(), uri) require.NoError(t, err) assertInitialSubjectState(t, state, uri, moderation.RecordStateUnavailable, nil) assert.Equal(t, []string{uri}, reader.calls) @@ -77,7 +91,7 @@ func TestGetSubjectStateIndexedPresent(t *testing.T) { cid := "bafy...distinct" reader := &fakeSubjectReader{record: &moderation.IndexedRecord{URI: uri, CID: cid}} - state, err := moderation.NewService(reader).GetSubjectState(t.Context(), uri) + state, err := newSubjectStateTestService(reader).GetSubjectState(t.Context(), uri) require.NoError(t, err) assertInitialSubjectState(t, state, uri, moderation.RecordStatePresent, &moderation.StrongRef{URI: uri, CID: cid}) assert.Equal(t, []string{uri}, reader.calls) @@ -89,7 +103,7 @@ func TestGetSubjectStateAuthorDeleted(t *testing.T) { uri := "at://did:plc:x/social.coves.community.comment/3kabc" reader := &fakeSubjectReader{record: &moderation.IndexedRecord{URI: uri, CID: "bafy...distinct", Deleted: true}} - state, err := moderation.NewService(reader).GetSubjectState(t.Context(), uri) + state, err := newSubjectStateTestService(reader).GetSubjectState(t.Context(), uri) require.NoError(t, err) assertInitialSubjectState(t, state, uri, moderation.RecordStateDeleted, nil) assert.Equal(t, []string{uri}, reader.calls) @@ -99,7 +113,7 @@ func TestGetSubjectStateReaderFailure(t *testing.T) { uri := "at://did:plc:x/social.coves.community.postv2/3kabc" reader := &fakeSubjectReader{err: errors.New("db down")} - state, err := moderation.NewService(reader).GetSubjectState(t.Context(), uri) + state, err := newSubjectStateTestService(reader).GetSubjectState(t.Context(), uri) assert.ErrorIs(t, err, moderation.ErrModerationUnavailable) assert.Nil(t, state) assert.Equal(t, []string{uri}, reader.calls) @@ -108,7 +122,7 @@ func TestGetSubjectStateReaderFailure(t *testing.T) { func TestGetSubjectStateNeverIndexedVersionIsStable(t *testing.T) { uri := "at://did:plc:neverindexed/social.coves.community.postv2/3kabc" reader := &fakeSubjectReader{err: moderation.ErrSubjectNotIndexed} - service := moderation.NewService(reader) + service := newSubjectStateTestService(reader) first, err := service.GetSubjectState(t.Context(), uri) require.NoError(t, err) diff --git a/internal/core/moderation/store.go b/internal/core/moderation/store.go new file mode 100644 index 0000000..8a5281a --- /dev/null +++ b/internal/core/moderation/store.go @@ -0,0 +1,120 @@ +package moderation + +import ( + "context" + "time" +) + +// Action kinds, scope kinds and origins recorded on moderation actions. +const ( + ActionRemove = "remove" + ActionRestore = "restore" + + ScopeInstance = "instance" + + OriginLocal = "local" + + // ModerationStateRemoved is the effective state of a subject with an + // active removal decision. + ModerationStateRemoved = "removed" + + OutcomeApplied = "applied" + OutcomeUnchanged = "unchanged" +) + +// Action is one immutable row of the moderation action log. +type Action struct { + ID string + ActorDID string + AuthorityDID string + ScopeKind string + ScopeCommunityDID string + SubjectURI string + SubjectCollection string + SubjectCommunityDID string + ObservedCID string + Action string + LabelValue string + Reason string + PrivateNote string + ReversesActionID string + Origin string + CreatedAt time.Time +} + +// IndexedComment is the indexed comment row a mutation inspects, read under a +// share lock so consumer writes serialize against the CID check. +type IndexedComment struct { + URI string + CID string + AuthorDeleted bool + // OwnerDID is the repository holding the comment and its blobs. + OwnerDID string + // CommunityDID is the root post's community, empty when the root post is + // not indexed. + CommunityDID string + // ImageCIDs are the blob CIDs of the comment's indexed image embed. + ImageCIDs []string +} + +// MediaBlock suppresses Coves-served bytes of a blob. An empty OwnerDID +// blocks the CID for every owner. +type MediaBlock struct { + OwnerDID string + BlobCID string + ActionID string +} + +// IdempotencyRecord is a stored mutation result bound to a request +// fingerprint, scoped to (actor DID, authority DID, key). +type IdempotencyRecord struct { + ActorDID string + AuthorityDID string + Key string + Fingerprint string + Result MutationResult + CreatedAt time.Time + ExpiresAt time.Time +} + +// SubjectModeration is the stored moderation state of a subject. +type SubjectModeration struct { + Version int64 + ActiveRemoval *Action +} + +// Store persists moderation state. Mutations run inside InTransaction. +type Store interface { + InTransaction(ctx context.Context, fn func(ctx context.Context, tx Transaction) error) error + SubjectModeration(ctx context.Context, authorityDID, subjectURI string) (*SubjectModeration, error) +} + +// Transaction is the set of operations a mutation performs atomically. +type Transaction interface { + // LockActor serializes idempotency admission for one actor. + LockActor(ctx context.Context, actorDID string) error + // LiveIdempotencyRecord returns the unexpired record for the key, or nil. + LiveIdempotencyRecord(ctx context.Context, actorDID, authorityDID, key string, now time.Time) (*IdempotencyRecord, error) + CountLiveIdempotencyKeys(ctx context.Context, actorDID string, now time.Time) (int, error) + // SaveIdempotencyRecord stores the record, replacing an expired one. + SaveIdempotencyRecord(ctx context.Context, record IdempotencyRecord) error + // LockSubject locks the subject's version row, creating it at 0. + LockSubject(ctx context.Context, subjectURI string) (int64, error) + // ReadIndexedComment returns ErrSubjectNotIndexed for a never-indexed URI. + ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + // GetAction returns ErrDecisionNotFound for an unknown id. + GetAction(ctx context.Context, actionID string) (*Action, error) + // ActiveRemoval returns the active removal action, or nil. + ActiveRemoval(ctx context.Context, authorityDID, subjectURI string) (*Action, error) + InsertAction(ctx context.Context, action Action) (*Action, error) + SetRemovalDecision(ctx context.Context, authorityDID, subjectURI, actionID string, active bool) error + SetSubjectVersion(ctx context.Context, subjectURI string, version int64) error + InsertMediaBlocks(ctx context.Context, blocks []MediaBlock) error + DeactivateMediaBlocks(ctx context.Context, actionID string) error +} + +// MediaPurger removes cached bytes of newly blocked blobs. +type MediaPurger interface { + PurgeOwnerBlob(ownerDID, blobCID string) error + PurgeBlob(blobCID string) error +} diff --git a/internal/core/moderation/subject_reader.go b/internal/core/moderation/subject_reader.go index 5c043c5..4bb7c38 100644 --- a/internal/core/moderation/subject_reader.go +++ b/internal/core/moderation/subject_reader.go @@ -46,7 +46,10 @@ func (r *repositorySubjectReader) ReadSubject(ctx context.Context, uri string) ( if err != nil { return nil, fmt.Errorf("reading comment: %w", err) } - return &IndexedRecord{URI: comment.URI, CID: comment.CID, Deleted: comment.DeletedAt != nil}, nil + return &IndexedRecord{ + URI: comment.URI, CID: comment.CID, + Deleted: comment.DeletedAt != nil && comment.DeletionReason != nil && *comment.DeletionReason == comments.DeletionReasonAuthor, + }, nil default: return nil, fmt.Errorf("%w: unsupported collection", ErrInvalidSubject) } diff --git a/internal/core/moderation/subject_reader_integration_test.go b/internal/core/moderation/subject_reader_integration_test.go index b6f7764..4126535 100644 --- a/internal/core/moderation/subject_reader_integration_test.go +++ b/internal/core/moderation/subject_reader_integration_test.go @@ -68,10 +68,16 @@ func TestRepositorySubjectReaderIndexedRecords(t *testing.T) { } commentURI := insertComment("bafyreipresentcomment") deletedCommentURI := insertComment("bafyreideletedcomment") + moderatorDeletedCommentCID := "bafyreimoderatordeletedcomment" + moderatorDeletedCommentURI := insertComment(moderatorDeletedCommentCID) _, err = db.ExecContext(ctx, ` UPDATE comments SET deleted_at = NOW(), deletion_reason = 'author', deleted_by = $1 WHERE uri = $2 `, authorDID, deletedCommentURI) require.NoError(t, err) + _, err = db.ExecContext(ctx, ` + UPDATE comments SET deleted_at = NOW(), deletion_reason = 'moderator', deleted_by = $1, content = '' WHERE uri = $2 + `, communityDID, moderatorDeletedCommentURI) + require.NoError(t, err) for _, test := range []struct { name string @@ -83,6 +89,7 @@ func TestRepositorySubjectReaderIndexedRecords(t *testing.T) { {"indexed legacy post", legacyURI, legacyCID, false}, {"indexed comment", commentURI, "bafyreipresentcomment", false}, {"author-deleted comment", deletedCommentURI, "bafyreideletedcomment", true}, + {"legacy moderator-deleted comment", moderatorDeletedCommentURI, moderatorDeletedCommentCID, false}, {"soft-deleted postv2", deletedPostV2URI, "bafyreideletedpostv2", true}, {"soft-deleted legacy post", deletedLegacyURI, legacyCID, true}, {"pending postv2", pendingURI, "bafyreipendingpostv2", false}, @@ -109,7 +116,9 @@ func TestRepositorySubjectReaderIndexedRecords(t *testing.T) { }) } - service := moderation.NewService(reader) + service := moderation.NewService(reader, postgres.NewModerationRepository(db), moderation.Config{ + InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000, + }) t.Run("service author-deleted comment", func(t *testing.T) { state, err := service.GetSubjectState(t.Context(), deletedCommentURI) require.NoError(t, err) @@ -117,6 +126,20 @@ func TestRepositorySubjectReaderIndexedRecords(t *testing.T) { assert.Equal(t, moderation.RecordStateDeleted, state.RecordState) assert.Nil(t, state.CurrentSubject) }) + t.Run("service legacy moderator-deleted comment", func(t *testing.T) { + state, err := service.GetSubjectState(t.Context(), moderatorDeletedCommentURI) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, moderation.RecordStatePresent, state.RecordState) + assert.Equal(t, &moderation.StrongRef{URI: moderatorDeletedCommentURI, CID: moderatorDeletedCommentCID}, state.CurrentSubject) + }) + t.Run("legacy moderator-deleted comment still requires matching CID to remove", func(t *testing.T) { + _, err := service.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "legacyadmin")), moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: moderatorDeletedCommentURI, CID: "bafyreigj3fwnwjuzr35k2kuzmb5dixxczrzjhqkr5srlqplsh6gq3bj3si"}, + ExpectedVersion: "v0", IdempotencyKey: "legacy-mismatched-cid", Reason: "social.coves.moderation.defs#reasonSpam", + }) + assert.ErrorIs(t, err, moderation.ErrContentChanged) + }) t.Run("service indexed postv2", func(t *testing.T) { state, err := service.GetSubjectState(t.Context(), postV2URI) require.NoError(t, err) diff --git a/internal/core/moderation/subject_state.go b/internal/core/moderation/subject_state.go new file mode 100644 index 0000000..cf974f1 --- /dev/null +++ b/internal/core/moderation/subject_state.go @@ -0,0 +1,24 @@ +package moderation + +import "fmt" + +// newSubjectState constructs a state with a current strong reference exactly +// when the indexed record is present. +func newSubjectState(subject string, version int64, recordState RecordState, current *StrongRef, activeRemoval *Action, authorityDID string) (SubjectState, error) { + if (recordState == RecordStatePresent) != (current != nil) || + (recordState != RecordStatePresent && recordState != RecordStateDeleted && recordState != RecordStateUnavailable) || version < 0 { + return SubjectState{}, fmt.Errorf("%w: inconsistent subject state", ErrModerationUnavailable) + } + state := SubjectState{ + Subject: subject, + Version: versionToken(version), + Moderation: ModerationView{State: ModerationStateClear}, + RecordState: recordState, + CurrentSubject: current, + } + if activeRemoval != nil { + state.Moderation.State = ModerationStateRemoved + state.LocalRemoval = &ActionRef{ServiceDID: authorityDID, ActionID: activeRemoval.ID} + } + return state, nil +} diff --git a/internal/core/moderation/types.go b/internal/core/moderation/types.go index 45e60b2..db8961b 100644 --- a/internal/core/moderation/types.go +++ b/internal/core/moderation/types.go @@ -2,6 +2,8 @@ // (Authority) and what the AppView knows about a subject (SubjectState). package moderation +import "time" + // RecordState is the repository availability of a subject record as the // AppView has indexed it. type RecordState string @@ -76,3 +78,39 @@ type IndexedRecord struct { CID string Deleted bool } + +// MutationResult is the outcome of a removeContent/restoreContent call. Action +// is nil for an unchanged outcome. +type MutationResult struct { + Outcome string + State SubjectState + Action *Action +} + +// RemoveContentRequest is the caller-supplied part of a removeContent call. +type RemoveContentRequest struct { + Subject StrongRef + ExpectedVersion string + IdempotencyKey string + Reason string + PrivateNote string +} + +// RestoreContentRequest is the caller-supplied part of a restoreContent call. +type RestoreContentRequest struct { + ActionID string + ReviewedSubject *StrongRef + ExpectedVersion string + IdempotencyKey string + Reason string + PrivateNote string +} + +// Config is the moderation service's configuration. +type Config struct { + InstanceDID string + IdempotencyRetention time.Duration + MaxLiveIdempotencyKeys int + Purger MediaPurger + Now func() time.Time +} diff --git a/internal/core/moderation/version.go b/internal/core/moderation/version.go index 44404b7..1517799 100644 --- a/internal/core/moderation/version.go +++ b/internal/core/moderation/version.go @@ -1,5 +1,11 @@ package moderation +import "strconv" + // InitialVersion is the opaque state token of a subject with no moderation -// rows. Later chunks advance the encoding; readers treat it as opaque. +// rows. Readers treat later tokens as opaque. const InitialVersion = "v0" + +func versionToken(version int64) string { + return "v" + strconv.FormatInt(version, 10) +} diff --git a/internal/db/migrations/049_moderation_state.sql b/internal/db/migrations/049_moderation_state.sql new file mode 100644 index 0000000..0446eb7 --- /dev/null +++ b/internal/db/migrations/049_moderation_state.sql @@ -0,0 +1,83 @@ +-- +goose Up +-- Moderation state is durable operator history, independent of indexed content. +CREATE TABLE moderation_actions ( + id TEXT PRIMARY KEY, + actor_did TEXT NOT NULL, + authority_did TEXT NOT NULL, + scope_kind TEXT NOT NULL CHECK (scope_kind IN ('instance', 'community')), + scope_community_did TEXT, + subject_uri TEXT NOT NULL, + subject_collection TEXT NOT NULL, + subject_community_did TEXT, + observed_cid TEXT, + action TEXT NOT NULL CHECK (action IN ('remove', 'restore', 'apply-removal', 'retract-removal', 'label', 'retract-label')), + label_value TEXT, + reason TEXT, + private_classification TEXT, + private_note TEXT, + reverses_action_id TEXT REFERENCES moderation_actions(id), + origin TEXT NOT NULL CHECK (origin IN ('local', 'inherited')), + created_at TIMESTAMPTZ NOT NULL, + CONSTRAINT moderation_action_scope CHECK ((scope_kind = 'instance') = (scope_community_did IS NULL)) +); + +CREATE INDEX moderation_actions_subject ON moderation_actions (subject_uri, created_at DESC, id DESC); + +CREATE TABLE moderation_subjects ( + subject_uri TEXT PRIMARY KEY, + version BIGINT NOT NULL DEFAULT 0 CHECK (version >= 0), + updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() +); + +CREATE TABLE moderation_decisions ( + authority_did TEXT NOT NULL, + scope_kind TEXT NOT NULL CHECK (scope_kind IN ('instance', 'community')), + scope_community_did TEXT, + subject_uri TEXT NOT NULL, + kind TEXT NOT NULL CHECK (kind IN ('removal', 'label')), + value TEXT, + active_action_id TEXT NOT NULL REFERENCES moderation_actions(id), + active BOOLEAN NOT NULL, + CONSTRAINT moderation_decision_scope CHECK ((scope_kind = 'instance') = (scope_community_did IS NULL)), + CONSTRAINT moderation_decision_value CHECK ((kind = 'removal') = (value IS NULL)), + CONSTRAINT moderation_decisions_key UNIQUE NULLS NOT DISTINCT + (authority_did, scope_kind, scope_community_did, subject_uri, kind, value) +); + +CREATE INDEX moderation_decisions_active_subject ON moderation_decisions (subject_uri, authority_did) + WHERE active; + +CREATE TABLE moderation_idempotency_keys ( + actor_did TEXT NOT NULL, + authority_did TEXT NOT NULL, + key TEXT NOT NULL, + fingerprint TEXT NOT NULL, + stored_result JSONB NOT NULL, + created_at TIMESTAMPTZ NOT NULL, + expires_at TIMESTAMPTZ NOT NULL, + PRIMARY KEY (actor_did, authority_did, key) +); + +CREATE INDEX moderation_idempotency_actor_live ON moderation_idempotency_keys (actor_did, expires_at); +CREATE INDEX moderation_idempotency_expiry ON moderation_idempotency_keys (expires_at); + +CREATE TABLE moderation_media_blocks ( + id BIGINT GENERATED ALWAYS AS IDENTITY PRIMARY KEY, + owner_did TEXT, + blob_cid TEXT NOT NULL, + action_id TEXT NOT NULL REFERENCES moderation_actions(id), + active BOOLEAN NOT NULL DEFAULT TRUE, + CONSTRAINT moderation_media_block_key UNIQUE NULLS NOT DISTINCT (owner_did, blob_cid, action_id) +); + +CREATE INDEX moderation_media_blocks_owner_active ON moderation_media_blocks (owner_did, blob_cid) WHERE active; +CREATE INDEX moderation_media_blocks_cid_active ON moderation_media_blocks (blob_cid) WHERE active AND owner_did IS NULL; +CREATE INDEX moderation_media_blocks_action_active ON moderation_media_blocks (action_id) WHERE active; + +-- +goose Down +-- NEVER run this Down migration in production: moderation history is not disposable indexing state. +DROP TABLE IF EXISTS moderation_media_blocks; +DROP TABLE IF EXISTS moderation_idempotency_keys; +DROP TABLE IF EXISTS moderation_decisions; +DROP TABLE IF EXISTS moderation_subjects; +DROP TABLE IF EXISTS moderation_actions; diff --git a/internal/db/postgres/admission_repo_schema_test.go b/internal/db/postgres/admission_repo_schema_test.go index b9e93a8..8697d1f 100644 --- a/internal/db/postgres/admission_repo_schema_test.go +++ b/internal/db/postgres/admission_repo_schema_test.go @@ -322,6 +322,8 @@ func TestMigration034_DownRestoresTheAuthorForeignKeyUnvalidated(t *testing.T) { // all sit on top of 034, so all fourteen have to come off first. Rolling back explicitly, // one asserted step at a time, is what keeps the assertions below pointed at // 034's Down rather than at whatever happens to be newest. + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "048 (Discover Hot snapshot tables) must be rolled back before testing earlier migrations") require.EqualValues(t, 47, testkit.MigrateDownOne(t, db, 47), diff --git a/internal/db/postgres/comment_repo.go b/internal/db/postgres/comment_repo.go index 573c288..ede49fe 100644 --- a/internal/db/postgres/comment_repo.go +++ b/internal/db/postgres/comment_repo.go @@ -521,6 +521,10 @@ func (r *postgresCommentRepo) ListByCommenterWithCursor(ctx context.Context, req LEFT JOIN users u ON c.commenter_did = u.did WHERE c.commenter_did = $1 AND c.deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM moderation_decisions d + WHERE d.subject_uri = c.uri AND d.kind = 'removal' AND d.active + ) %s %s ORDER BY c.created_at DESC, c.uri DESC @@ -1392,3 +1396,33 @@ func (r *postgresCommentRepo) GetVoteStateForComments(ctx context.Context, viewe return result, nil } + +// ActiveRemovalsByURIs returns the active removal sources of each URI. +func (r *postgresCommentRepo) ActiveRemovalsByURIs(ctx context.Context, uris []string) (map[string][]comments.RemovalSource, error) { + removals := make(map[string][]comments.RemovalSource) + if len(uris) == 0 { + return removals, nil + } + rows, err := r.db.QueryContext(ctx, ` + SELECT subject_uri, authority_did, scope_kind + FROM moderation_decisions + WHERE subject_uri = ANY($1) AND kind = 'removal' AND active + ORDER BY subject_uri, authority_did, scope_kind, scope_community_did + `, pq.Array(uris)) + if err != nil { + return nil, fmt.Errorf("fetch active comment removals: %w", err) + } + defer rows.Close() + for rows.Next() { + var uri string + var source comments.RemovalSource + if err := rows.Scan(&uri, &source.AuthorityDID, &source.ScopeKind); err != nil { + return nil, fmt.Errorf("scan active comment removal: %w", err) + } + removals[uri] = append(removals[uri], source) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate active comment removals: %w", err) + } + return removals, nil +} diff --git a/internal/db/postgres/community_subscriber_recount_migration_test.go b/internal/db/postgres/community_subscriber_recount_migration_test.go index 1398f20..ba35e7d 100644 --- a/internal/db/postgres/community_subscriber_recount_migration_test.go +++ b/internal/db/postgres/community_subscriber_recount_migration_test.go @@ -22,6 +22,8 @@ func TestMigration045RecountsAndMaintainsCommunitySubscribers(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "048 (Discover Hot snapshot tables) must be rolled back before testing earlier migrations") require.EqualValues(t, 47, testkit.MigrateDownOne(t, db, 47), diff --git a/internal/db/postgres/credential_reencrypt_migration_test.go b/internal/db/postgres/credential_reencrypt_migration_test.go index 9fc5e27..ac3ac75 100644 --- a/internal/db/postgres/credential_reencrypt_migration_test.go +++ b/internal/db/postgres/credential_reencrypt_migration_test.go @@ -27,6 +27,8 @@ func TestMigration046DownRestoresUsableEncryptionKey(t *testing.T) { db := testkit.DB(t) assert.False(t, credentialReencryptKeyTable(t, db).Valid, "migration 046 Up must drop encryption_keys before its Down behavior can be tested") + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "048 (Discover Hot snapshot tables) must be rolled back before testing earlier migrations") require.EqualValues(t, 47, testkit.MigrateDownOne(t, db, 47), @@ -393,6 +395,8 @@ func TestCredentialReencryptRejectsLegacyRowsAfterEncryptionKeysDropped(t *testi func credentialReencryptVersion45Database(t *testing.T) *sql.DB { t.Helper() db := testkit.DB(t) + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "048 (Discover Hot snapshot tables) must be rolled back before testing earlier migrations") require.EqualValues(t, 47, testkit.MigrateDownOne(t, db, 47), diff --git a/internal/db/postgres/discover_hot_migration_test.go b/internal/db/postgres/discover_hot_migration_test.go index cbbd2f1..6e26056 100644 --- a/internal/db/postgres/discover_hot_migration_test.go +++ b/internal/db/postgres/discover_hot_migration_test.go @@ -35,6 +35,8 @@ func TestMigration048DiscoverHotSnapshots(t *testing.T) { _, err := db.ExecContext(ctx, `CREATE TABLE discover_hot_migration_sentinel (id BIGINT PRIMARY KEY)`) require.NoError(t, err) + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "this test must exercise migration 048's Down section") for _, table := range discoverHotTables { diff --git a/internal/db/postgres/future_comment_created_at_migration_test.go b/internal/db/postgres/future_comment_created_at_migration_test.go index 32f1279..4abae40 100644 --- a/internal/db/postgres/future_comment_created_at_migration_test.go +++ b/internal/db/postgres/future_comment_created_at_migration_test.go @@ -18,6 +18,8 @@ func TestMigration041_ClampsFutureCommentCreatedAt(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "048 (Discover Hot snapshot tables) must be rolled back before testing earlier migrations") require.EqualValues(t, 47, testkit.MigrateDownOne(t, db, 47), diff --git a/internal/db/postgres/moderation_actor_lock_integration_test.go b/internal/db/postgres/moderation_actor_lock_integration_test.go new file mode 100644 index 0000000..b7c2148 --- /dev/null +++ b/internal/db/postgres/moderation_actor_lock_integration_test.go @@ -0,0 +1,67 @@ +//go:build integration + +package postgres_test + +import ( + "context" + "database/sql" + "testing" + "time" + + "Coves/internal/core/moderation" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// waitForGatedInsertAndOneWaiter waits until one request is held inside its +// action insert by holdModerationActionInsert and exactly one other request is +// blocked on a lock. With the actor lock the second request waits in +// pg_advisory_xact_lock before reading its key; without it the second request +// waits on the subject row or the same insert gate, and the outcome differs. +func waitForGatedInsertAndOneWaiter(t *testing.T, connection *sql.Conn) { + t.Helper() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + var waiting, inserting int + err := connection.QueryRowContext(t.Context(), ` + SELECT count(*), count(*) FILTER (WHERE query LIKE '%INSERT INTO moderation_actions%') + FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() + AND wait_event_type = 'Lock' + `).Scan(&waiting, &inserting) + return waiting == 2 && inserting >= 1, err + }, testkit.WithDescription("one action insert gated while the other request of the same actor waits on a lock")) +} + +func TestModerationConcurrentIdenticalKeyRetryAppliesOnce(t *testing.T) { + db := testkit.DB(t) + subject, _, _ := indexedModerationComment(t, db, true, "") + service := newPostgresModerationService(db) + actor := fixtures.DID("retryraceadmin") + request := moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "retry-race-key", + Reason: moderationConcurrencyReason, + } + connection, release := holdModerationActionInsert(t, db) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + remove := func() (*moderation.MutationResult, error) { + return service.RemoveContent(ctx, actor, request) + } + results := startConcurrentModerationCalls(remove, remove) + waitForGatedInsertAndOneWaiter(t, connection) + release() + first, second := <-results, <-results + require.NoError(t, first.err) + require.NoError(t, second.err) + require.NotNil(t, first.result) + require.NotNil(t, second.result) + assert.Equal(t, moderation.OutcomeApplied, first.result.Outcome) + assert.Equal(t, first.result, second.result, "the retry must replay the stored result of the first request") + assert.Equal(t, 1, countModerationActions(t, db, subject.URI, moderation.ActionRemove)) + var keys int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1`, actor).Scan(&keys)) + assert.Equal(t, 1, keys) +} diff --git a/internal/db/postgres/moderation_concurrency_integration_test.go b/internal/db/postgres/moderation_concurrency_integration_test.go new file mode 100644 index 0000000..a86239c --- /dev/null +++ b/internal/db/postgres/moderation_concurrency_integration_test.go @@ -0,0 +1,541 @@ +//go:build integration + +package postgres_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "strings" + "sync" + "testing" + "time" + + "Coves/internal/atproto/jetstream" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const moderationConcurrencyReason = "social.coves.moderation.defs#reasonSpam" + +type moderationConcurrentOutcome struct { + result *moderation.MutationResult + err error +} + +func startConcurrentModerationCalls(first, second func() (*moderation.MutationResult, error)) <-chan moderationConcurrentOutcome { + ready := make(chan struct{}, 2) + start := make(chan struct{}) + results := make(chan moderationConcurrentOutcome, 2) + for _, call := range []func() (*moderation.MutationResult, error){first, second} { + go func() { + ready <- struct{}{} + <-start + result, err := call() + results <- moderationConcurrentOutcome{result: result, err: err} + }() + } + <-ready + <-ready + close(start) + return results +} + +// Keep the first action insert in flight while the other mutation attempts to +// lock the same existing subject row. Both actors are distinct, so the actor +// advisory lock cannot accidentally serialize this race. +func holdModerationActionInsert(t *testing.T, db *sql.DB) (*sql.Conn, func()) { + t.Helper() + _, err := db.ExecContext(t.Context(), ` + CREATE FUNCTION hold_moderation_action_insert() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended(current_database(), 0)); + RETURN NEW; + END; + $$; + `) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + CREATE TRIGGER hold_moderation_action_insert BEFORE INSERT ON moderation_actions + FOR EACH ROW EXECUTE FUNCTION hold_moderation_action_insert(); + `) + require.NoError(t, err) + connection, err := db.Conn(t.Context()) + require.NoError(t, err) + _, err = connection.ExecContext(t.Context(), `SELECT pg_advisory_lock(hashtextextended(current_database(), 0))`) + require.NoError(t, err) + released := false + release := func() { + if released { + return + } + released = true + var unlocked bool + assert.NoError(t, connection.QueryRowContext(context.Background(), `SELECT pg_advisory_unlock(hashtextextended(current_database(), 0))`).Scan(&unlocked)) + assert.True(t, unlocked) + assert.NoError(t, connection.Close()) + } + t.Cleanup(release) + return connection, release +} + +func waitForSubjectLockContention(t *testing.T, connection *sql.Conn) { + t.Helper() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + var inserting, waitingForSubject int + err := connection.QueryRowContext(t.Context(), ` + SELECT count(*) FILTER (WHERE query LIKE '%INSERT INTO moderation_actions%'), + count(*) FILTER (WHERE query LIKE '%SELECT version FROM moderation_subjects%' + AND query LIKE '%FOR UPDATE%') + FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() + AND wait_event_type = 'Lock' + `).Scan(&inserting, &waitingForSubject) + return inserting == 1 && waitingForSubject == 1, err + }, testkit.WithDescription("one action insert gated while the other request waits for the subject row lock")) +} + +func countModerationActions(t *testing.T, db *sql.DB, subjectURI, action string) int { + t.Helper() + var count int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_actions WHERE subject_uri = $1 AND action = $2 + `, subjectURI, action).Scan(&count)) + return count +} + +func TestModerationConcurrentRemovesRejectStaleVersion(t *testing.T) { + db := testkit.DB(t) + subject, _, _ := indexedModerationComment(t, db, true, "") + service := newPostgresModerationService(db) + seed, err := service.RemoveContent(t.Context(), fixtures.DID("seedremoveadmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "seed-remove", + Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, seed) + require.NotNil(t, seed.Action) + restored, err := service.RestoreContent(t.Context(), fixtures.DID("seedrestoreadmin"), moderation.RestoreContentRequest{ + ActionID: seed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: "v1", + IdempotencyKey: "seed-restore", Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, restored) + require.Equal(t, "v2", restored.State.Version) + require.Equal(t, moderation.ModerationStateClear, restored.State.Moderation.State) + connection, release := holdModerationActionInsert(t, db) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + request := moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v2", Reason: moderationConcurrencyReason, + } + first := request + first.IdempotencyKey = "concurrent-remove-first" + second := request + second.IdempotencyKey = "concurrent-remove-second" + results := startConcurrentModerationCalls( + func() (*moderation.MutationResult, error) { + return service.RemoveContent(ctx, fixtures.DID("concurrentadminone"), first) + }, + func() (*moderation.MutationResult, error) { + return service.RemoveContent(ctx, fixtures.DID("concurrentadmintwo"), second) + }, + ) + waitForSubjectLockContention(t, connection) + release() + var applied, conflicts int + for range 2 { + outcome := <-results + if outcome.err != nil { + assert.ErrorIs(t, outcome.err, moderation.ErrStateConflict) + assert.Nil(t, outcome.result) + if errors.Is(outcome.err, moderation.ErrStateConflict) { + conflicts++ + } + continue + } + require.NotNil(t, outcome.result) + assert.Equal(t, moderation.OutcomeApplied, outcome.result.Outcome) + if outcome.result.Outcome == moderation.OutcomeApplied { + applied++ + } + } + assert.Equal(t, 1, applied) + assert.Equal(t, 1, conflicts) + assert.Equal(t, 2, countModerationActions(t, db, subject.URI, moderation.ActionRemove), "one seed removal and exactly one racing removal") +} + +// gatedMediaReconciler runs the real reconciliation inside the consumer's +// transaction, then holds that transaction open until the test releases it. +type gatedMediaReconciler struct { + inner *moderation.MediaReconciler + reconciled chan struct{} + release chan struct{} + once sync.Once +} + +func (g *gatedMediaReconciler) ReconcileTx(ctx context.Context, tx *sql.Tx, subjectURI string) ([]moderation.MediaBlock, error) { + blocks, err := g.inner.ReconcileTx(ctx, tx, subjectURI) + g.once.Do(func() { close(g.reconciled) }) + select { + case <-g.release: + case <-ctx.Done(): + return nil, ctx.Err() + } + return blocks, err +} + +func (g *gatedMediaReconciler) Purge(blocks []moderation.MediaBlock) { + g.inner.Purge(blocks) +} + +// holdRestoreBeforeCommit parks a mutation at its idempotency-record insert, +// its last statement, after the decision update and media-block deactivation. +// It returns the gate's own connection, which stays usable for observation +// while the test's small pool is held by the parked transactions. +func holdRestoreBeforeCommit(t *testing.T, db *sql.DB) (*sql.Conn, func()) { + t.Helper() + _, err := db.ExecContext(t.Context(), ` + CREATE FUNCTION hold_moderation_idempotency_insert() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + PERFORM pg_advisory_xact_lock(hashtextextended(current_database(), 1)); + RETURN NEW; + END; + $$; + `) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + CREATE TRIGGER hold_moderation_idempotency_insert BEFORE INSERT ON moderation_idempotency_keys + FOR EACH ROW EXECUTE FUNCTION hold_moderation_idempotency_insert(); + `) + require.NoError(t, err) + connection, err := db.Conn(t.Context()) + require.NoError(t, err) + _, err = connection.ExecContext(t.Context(), `SELECT pg_advisory_lock(hashtextextended(current_database(), 1))`) + require.NoError(t, err) + released := false + release := func() { + if released { + return + } + released = true + var unlocked bool + assert.NoError(t, connection.QueryRowContext(context.Background(), `SELECT pg_advisory_unlock(hashtextextended(current_database(), 1))`).Scan(&unlocked)) + assert.True(t, unlocked) + assert.NoError(t, connection.Close()) + } + t.Cleanup(release) + return connection, release +} + +// countLockWaiters counts backends of this database waiting on a heavyweight +// lock, split into advisory-lock waits (the test's own gates) and all others. +func countLockWaiters(ctx context.Context, observer interface { + QueryRowContext(ctx context.Context, query string, args ...any) *sql.Row +}) (advisory, other int, err error) { + err = observer.QueryRowContext(ctx, ` + SELECT count(*) FILTER (WHERE wait_event = 'advisory'), + count(*) FILTER (WHERE wait_event <> 'advisory') + FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() AND wait_event_type = 'Lock' + `).Scan(&advisory, &other) + return advisory, other, err +} + +func moderationConsumerCommentEvent(subject moderation.StrongRef, authorDID, rootURI, rootCID, operation, recordCID, imageCID string) *jetstream.JetstreamEvent { + rkey := subject.URI[strings.LastIndex(subject.URI, "/")+1:] + return &jetstream.JetstreamEvent{ + Did: authorDID, Kind: "commit", TimeUS: time.Now().UnixMicro(), + Commit: &jetstream.CommitEvent{ + Rev: testkit.TID(), Operation: operation, Collection: moderation.CommentCollection, + RKey: rkey, CID: recordCID, + Record: map[string]interface{}{ + "$type": moderation.CommentCollection, "content": "author rewrite", + "reply": map[string]interface{}{ + "root": map[string]interface{}{"uri": rootURI, "cid": rootCID}, + "parent": map[string]interface{}{"uri": rootURI, "cid": rootCID}, + }, + "embed": map[string]interface{}{ + "$type": "social.coves.embed.images", + "images": []interface{}{map[string]interface{}{"alt": "rewritten image", "image": map[string]interface{}{ + "$type": "blob", "ref": map[string]interface{}{"$link": imageCID}, "mimeType": "image/png", "size": 10, + }}}, + }, + "createdAt": time.Now().Format(time.RFC3339), + }, + }, + } +} + +// A restore and the comment consumer's in-transaction media reconciliation +// race on one removed comment. Whichever commits first, no active media block +// may be left tied to a restored action, and a consumer write that commits +// first must be seen by the restore. The comment row may be absent — purged by +// account deletion and then recreated by a fresh insert — in which case no +// content-row lock orders the two. +func TestModerationRestoreRacesCommentConsumer(t *testing.T) { + for _, test := range []struct { + name string + rowPresent bool + consumerFirst bool + }{ + {name: "consumer edit commits first", rowPresent: true, consumerFirst: true}, + {name: "restore commits before consumer edit", rowPresent: true}, + {name: "consumer reinsertion of a purged row commits first", consumerFirst: true}, + {name: "restore commits before consumer reinsertion of a purged row"}, + } { + t.Run(test.name, func(t *testing.T) { + db := testkit.DB(t) + subject, authorDID, _ := indexedModerationComment(t, db, true, "") + var rootURI, rootCID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT root_uri, root_cid FROM comments WHERE uri = $1`, subject.URI).Scan(&rootURI, &rootCID)) + service := newPostgresModerationService(db) + removed, err := service.RemoveContent(t.Context(), fixtures.DID("consumerraceadmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "consumer-race-remove", + Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, removed.Action) + operation := "update" + if !test.rowPresent { + // Account deletion purges the row outright (user_repo), leaving + // the moderation decision and no content row to lock. + _, err := db.ExecContext(t.Context(), `DELETE FROM comments WHERE uri = $1`, subject.URI) + require.NoError(t, err) + operation = "create" + } + + gate := &gatedMediaReconciler{ + inner: moderation.NewMediaReconciler(postgres.NewModerationRepository(db), fixtures.InstanceDID(), nil), + reconciled: make(chan struct{}), release: make(chan struct{}), + } + consumer := jetstream.NewCommentEventConsumer(postgres.NewCommentRepository(db), db, jetstream.WithCommentMediaReconciler(gate)) + event := moderationConsumerCommentEvent(subject, authorDID, rootURI, rootCID, operation, moderationImageCIDOne, moderationImageCIDTwo) + + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + consumerDone := make(chan error, 1) + restoreDone := make(chan moderationConcurrentOutcome, 1) + startConsumer := func() { + go func() { consumerDone <- consumer.HandleEvent(ctx, event) }() + } + startRestore := func() { + go func() { + result, restoreErr := service.RestoreContent(ctx, fixtures.DID("consumerraceadmin"), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: "v1", + IdempotencyKey: "consumer-race-restore", Reason: moderationConcurrencyReason, + }) + restoreDone <- moderationConcurrentOutcome{result: result, err: restoreErr} + }() + } + + if test.consumerFirst { + startConsumer() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + select { + case <-gate.reconciled: + return true, nil + case consumerErr := <-consumerDone: + return false, fmt.Errorf("consumer finished before reconciling: %v", consumerErr) + default: + return false, nil + } + }, testkit.WithDescription("consumer holding its reconciled write open")) + startRestore() + // The restore either blocks behind the open consumer write or, + // if nothing orders them, runs to completion. + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + if len(restoreDone) > 0 { + return true, nil + } + _, waiting, err := countLockWaiters(t.Context(), db) + return waiting > 0, err + }, testkit.WithDescription("restore waiting on the consumer's locks or finished")) + close(gate.release) + } else { + close(gate.release) + observer, releaseRestore := holdRestoreBeforeCommit(t, db) + startRestore() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + if len(restoreDone) > 0 { + return false, fmt.Errorf("restore finished before reaching its commit gate") + } + parked, _, err := countLockWaiters(t.Context(), observer) + return parked == 1, err + }, testkit.WithDescription("restore parked before commit")) + startConsumer() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + if len(consumerDone) > 0 { + return true, nil + } + _, waiting, err := countLockWaiters(t.Context(), observer) + return waiting > 0, err + }, testkit.WithDescription("consumer waiting on the restore's locks or finished")) + releaseRestore() + } + + require.NoError(t, <-consumerDone) + restored := <-restoreDone + var indexedCID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT cid FROM comments WHERE uri = $1`, subject.URI).Scan(&indexedCID)) + assert.Equal(t, moderationImageCIDOne, indexedCID, "the consumer write must be indexed") + var activeBlocks int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks WHERE action_id = $1 AND active + `, removed.Action.ID).Scan(&activeBlocks)) + var decisionActive bool + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT active FROM moderation_decisions WHERE subject_uri = $1 AND kind = 'removal' + `, subject.URI).Scan(&decisionActive)) + + if test.rowPresent && test.consumerFirst { + // The restore waited for the edit and saw the reviewed CID go stale. + require.ErrorIs(t, restored.err, moderation.ErrContentChanged) + assert.True(t, decisionActive) + assert.Equal(t, 1, activeBlocks, "the edit's new image stays blocked under the still-active removal") + assert.Zero(t, countModerationActions(t, db, subject.URI, moderation.ActionRestore)) + return + } + require.NoError(t, restored.err) + require.NotNil(t, restored.result) + assert.Equal(t, moderation.OutcomeApplied, restored.result.Outcome) + assert.False(t, decisionActive) + assert.Zero(t, activeBlocks, "no active media block may remain tied to a restored action") + blocked, err := postgres.NewModerationRepository(db).IsBlocked(t.Context(), authorDID, moderationImageCIDTwo) + require.NoError(t, err) + assert.False(t, blocked, "the restored comment's image must be servable") + }) + } +} + +func TestModerationConcurrentRestoresReverseActionOnlyOnce(t *testing.T) { + db := testkit.DB(t) + subject, _, _ := indexedModerationComment(t, db, true, "") + service := newPostgresModerationService(db) + removed, err := service.RemoveContent(t.Context(), fixtures.DID("restoreraceadmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "restore-race-remove", + Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, removed) + require.NotNil(t, removed.Action) + require.Equal(t, "v1", removed.State.Version) + connection, release := holdModerationActionInsert(t, db) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + request := moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: "v1", Reason: moderationConcurrencyReason, + } + first := request + first.IdempotencyKey = "restore-race-first" + second := request + second.IdempotencyKey = "restore-race-second" + firstAdminDID := fixtures.DID("restoreraceone") + secondAdminDID := fixtures.DID("restoreracetwo") + require.NotEqual(t, firstAdminDID, secondAdminDID) + results := startConcurrentModerationCalls( + func() (*moderation.MutationResult, error) { + return service.RestoreContent(ctx, firstAdminDID, first) + }, + func() (*moderation.MutationResult, error) { + return service.RestoreContent(ctx, secondAdminDID, second) + }, + ) + waitForSubjectLockContention(t, connection) + release() + var applied, rejected int + for range 2 { + outcome := <-results + if outcome.err != nil { + assert.Truef(t, errors.Is(outcome.err, moderation.ErrStateConflict) || errors.Is(outcome.err, moderation.ErrInvalidDecision), "unexpected restore error: %v", outcome.err) + assert.Nil(t, outcome.result) + if errors.Is(outcome.err, moderation.ErrStateConflict) || errors.Is(outcome.err, moderation.ErrInvalidDecision) { + rejected++ + } + continue + } + require.NotNil(t, outcome.result) + assert.Equal(t, moderation.OutcomeApplied, outcome.result.Outcome) + if outcome.result.Outcome == moderation.OutcomeApplied { + applied++ + } + } + assert.Equal(t, 1, applied) + assert.Equal(t, 1, rejected) + assert.Equal(t, 1, countModerationActions(t, db, subject.URI, moderation.ActionRestore)) +} + +func TestModerationConcurrentNewKeysCannotExceedActorCap(t *testing.T) { + db := testkit.DB(t) + service := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 2}, + ) + actor := fixtures.DID("capraceadmin") + seed, _, _ := indexedModerationComment(t, db, true, "") + firstSubject, _, _ := indexedModerationComment(t, db, true, "") + secondSubject, _, _ := indexedModerationComment(t, db, true, "") + seedResult, err := service.RemoveContent(t.Context(), actor, moderation.RemoveContentRequest{ + Subject: seed, ExpectedVersion: "v0", IdempotencyKey: "cap-seed", + Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, seedResult) + require.Equal(t, moderation.OutcomeApplied, seedResult.Outcome) + // Hold the first admitted request inside its action insert until the other + // request is blocked, so both overlap instead of running one after another. + connection, release := holdModerationActionInsert(t, db) + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + pending := startConcurrentModerationCalls( + func() (*moderation.MutationResult, error) { + return service.RemoveContent(ctx, actor, moderation.RemoveContentRequest{ + Subject: firstSubject, ExpectedVersion: "v0", IdempotencyKey: "cap-first", + Reason: moderationConcurrencyReason, + }) + }, + func() (*moderation.MutationResult, error) { + return service.RemoveContent(ctx, actor, moderation.RemoveContentRequest{ + Subject: secondSubject, ExpectedVersion: "v0", IdempotencyKey: "cap-second", + Reason: moderationConcurrencyReason, + }) + }, + ) + waitForGatedInsertAndOneWaiter(t, connection) + release() + var applied, rejected int + for _, outcome := range [2]moderationConcurrentOutcome{<-pending, <-pending} { + if outcome.err != nil { + assert.ErrorIs(t, outcome.err, moderation.ErrInvalidRequest) + assert.Nil(t, outcome.result) + if errors.Is(outcome.err, moderation.ErrInvalidRequest) { + rejected++ + } + continue + } + require.NotNil(t, outcome.result) + assert.Equal(t, moderation.OutcomeApplied, outcome.result.Outcome) + if outcome.result.Outcome == moderation.OutcomeApplied { + applied++ + } + } + assert.Equal(t, 1, applied) + assert.Equal(t, 1, rejected) + var liveKeys int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1 AND expires_at > NOW() + `, actor).Scan(&liveKeys)) + assert.Equal(t, 2, liveKeys) + firstActions := countModerationActions(t, db, firstSubject.URI, moderation.ActionRemove) + secondActions := countModerationActions(t, db, secondSubject.URI, moderation.ActionRemove) + assert.Equal(t, 1, firstActions+secondActions, "only one new subject may be removed") + assert.LessOrEqual(t, firstActions, 1) + assert.LessOrEqual(t, secondActions, 1) +} diff --git a/internal/db/postgres/moderation_idempotency_integration_test.go b/internal/db/postgres/moderation_idempotency_integration_test.go new file mode 100644 index 0000000..32220aa --- /dev/null +++ b/internal/db/postgres/moderation_idempotency_integration_test.go @@ -0,0 +1,219 @@ +//go:build integration + +package postgres_test + +import ( + "context" + "testing" + "time" + + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestModerationRepositoryReplaysOriginalRemovalAfterRestore(t *testing.T) { + db := testkit.DB(t) + subject, _, _ := indexedModerationComment(t, db, true, "") + service := newPostgresModerationService(db) + actor := fixtures.DID("persistreplayadmin") + request := moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "k-remove", + Reason: moderationConcurrencyReason, + } + + original, err := service.RemoveContent(t.Context(), actor, request) + require.NoError(t, err) + require.NotNil(t, original) + require.NotNil(t, original.Action) + require.Equal(t, moderation.OutcomeApplied, original.Outcome) + require.Equal(t, "v1", original.State.Version) + restored, err := service.RestoreContent(t.Context(), actor, moderation.RestoreContentRequest{ + ActionID: original.Action.ID, ReviewedSubject: &subject, + ExpectedVersion: "v1", IdempotencyKey: "k-restore", Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, restored) + require.Equal(t, "v2", restored.State.Version) + + // A persisted replay must return the old result even though the subject is clear now. + replayed, err := newPostgresModerationService(db).RemoveContent(t.Context(), actor, request) + require.NoError(t, err) + assert.Equal(t, original, replayed) + assert.Equal(t, 1, countModerationActions(t, db, subject.URI, moderation.ActionRemove)) + assert.Equal(t, 1, countModerationActions(t, db, subject.URI, moderation.ActionRestore)) + state, err := service.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, restored.State.Version, state.Version) + assert.Equal(t, moderation.ModerationStateClear, state.Moderation.State) + assert.Nil(t, state.LocalRemoval) +} + +func TestModerationRepositorySweepsExpiredIdempotencyKeyWithoutResettingVersion(t *testing.T) { + db := testkit.DB(t) + subject, _, _ := indexedModerationComment(t, db, true, "") + repository := postgres.NewModerationRepository(db) + now := time.Now().UTC().Truncate(time.Second) + service := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + repository, + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: time.Hour, + MaxLiveIdempotencyKeys: 1000, Now: func() time.Time { return now }}, + ) + actor := fixtures.DID("expiryadmin") + request := moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "k1", Reason: moderationConcurrencyReason, + } + removed, err := service.RemoveContent(t.Context(), actor, request) + require.NoError(t, err) + require.NotNil(t, removed) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + require.Equal(t, "v1", removed.State.Version) + var expiresAt time.Time + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT expires_at FROM moderation_idempotency_keys + WHERE actor_did = $1 AND authority_did = $2 AND key = $3 + `, actor, fixtures.InstanceDID(), request.IdempotencyKey).Scan(&expiresAt)) + assert.WithinDuration(t, now.Add(time.Hour), expiresAt, time.Second) + + // A key written later is still live at sweep time and must survive it. + liveSubject, _, _ := indexedModerationComment(t, db, true, "") + liveActor := fixtures.DID("liveexpiryadmin") + started := now + now = started.Add(30 * time.Minute) + live, err := service.RemoveContent(t.Context(), liveActor, moderation.RemoveContentRequest{ + Subject: liveSubject, ExpectedVersion: "v0", IdempotencyKey: "k-live", Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, live) + require.Equal(t, moderation.OutcomeApplied, live.Outcome) + + now = started.Add(time.Hour + time.Second) + deleted, err := repository.DeleteExpiredIdempotencyKeys(t.Context(), now) + require.NoError(t, err) + assert.EqualValues(t, 1, deleted) + var liveKeyCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1 AND key = $2`, liveActor, "k-live").Scan(&liveKeyCount)) + assert.Equal(t, 1, liveKeyCount, "the sweep must keep keys that have not expired") + var keyCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1 AND authority_did = $2 AND key = $3`, actor, fixtures.InstanceDID(), request.IdempotencyKey).Scan(&keyCount)) + assert.Zero(t, keyCount) + + result, err := service.RemoveContent(t.Context(), actor, request) + require.ErrorIs(t, err, moderation.ErrStateConflict) + assert.Nil(t, result) + assert.Equal(t, 1, countModerationActions(t, db, subject.URI, moderation.ActionRemove)) + state, err := service.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, "v1", state.Version) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1`, actor).Scan(&keyCount)) + assert.Zero(t, keyCount, "failed retries must not be saved") +} + +func TestModerationRepositoryEnforcesLiveKeyCapWithoutChangingSecondSubject(t *testing.T) { + db := testkit.DB(t) + firstSubject, _, _ := indexedModerationComment(t, db, true, "") + secondSubject, _, _ := indexedModerationComment(t, db, true, "") + service := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: time.Hour, MaxLiveIdempotencyKeys: 1}, + ) + actor := fixtures.DID("singlekeyadmin") + firstRequest := moderation.RemoveContentRequest{ + Subject: firstSubject, ExpectedVersion: "v0", IdempotencyKey: "first-key", Reason: moderationConcurrencyReason, + } + first, err := service.RemoveContent(t.Context(), actor, firstRequest) + require.NoError(t, err) + require.NotNil(t, first) + result, err := service.RemoveContent(t.Context(), actor, moderation.RemoveContentRequest{ + Subject: secondSubject, ExpectedVersion: "v0", IdempotencyKey: "second-key", Reason: moderationConcurrencyReason, + }) + require.ErrorIs(t, err, moderation.ErrInvalidRequest) + assert.ErrorContains(t, err, "1") + assert.Nil(t, result) + assert.Zero(t, countModerationActions(t, db, secondSubject.URI, moderation.ActionRemove)) + state, err := service.GetSubjectState(t.Context(), secondSubject.URI) + require.NoError(t, err) + assert.Equal(t, "v0", state.Version) + assert.Equal(t, moderation.ModerationStateClear, state.Moderation.State) + var count int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1`, actor).Scan(&count)) + assert.Equal(t, 1, count) + replayed, err := service.RemoveContent(t.Context(), actor, firstRequest) + require.NoError(t, err) + assert.Equal(t, first, replayed, "the cap must still allow an existing key to replay") +} + +func TestModerationRepositoryReplacesOnlyExpiredUnsweptIdempotencyKey(t *testing.T) { + db := testkit.DB(t) + firstSubject, _, _ := indexedModerationComment(t, db, true, "") + secondSubject, _, _ := indexedModerationComment(t, db, true, "") + repository := postgres.NewModerationRepository(db) + now := time.Now().UTC().Truncate(time.Second) + service := moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + repository, + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: time.Hour, + MaxLiveIdempotencyKeys: 1000, Now: func() time.Time { return now }}, + ) + actor := fixtures.DID("reusedkeyadmin") + const key = "reused-key" + storedKey := func() (fingerprint string, createdAt, expiresAt time.Time) { + t.Helper() + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT fingerprint, created_at, expires_at FROM moderation_idempotency_keys + WHERE actor_did = $1 AND authority_did = $2 AND key = $3 + `, actor, fixtures.InstanceDID(), key).Scan(&fingerprint, &createdAt, &expiresAt)) + return fingerprint, createdAt, expiresAt + } + first, err := service.RemoveContent(t.Context(), actor, moderation.RemoveContentRequest{ + Subject: firstSubject, ExpectedVersion: "v0", IdempotencyKey: key, Reason: moderationConcurrencyReason, + }) + require.NoError(t, err) + require.NotNil(t, first) + require.Equal(t, moderation.OutcomeApplied, first.Outcome) + firstFingerprint, _, firstExpiry := storedKey() + + // While the key is live, a direct save of a different record must not + // overwrite it. + now = now.Add(time.Minute) + require.NoError(t, repository.InTransaction(t.Context(), func(ctx context.Context, tx moderation.Transaction) error { + return tx.SaveIdempotencyRecord(ctx, moderation.IdempotencyRecord{ + ActorDID: actor, AuthorityDID: fixtures.InstanceDID(), Key: key, Fingerprint: "live-overwrite-attempt", + Result: moderation.MutationResult{Outcome: moderation.OutcomeUnchanged}, CreatedAt: now, ExpiresAt: now.Add(time.Hour), + }) + })) + fingerprint, _, expiresAt := storedKey() + assert.Equal(t, firstFingerprint, fingerprint) + assert.True(t, firstExpiry.Equal(expiresAt), "a live key keeps its expiry") + + // After expiry and before any sweep, the same key with a different request + // is admitted and replaces the stored row. + now = firstExpiry.Add(time.Second) + secondRequest := moderation.RemoveContentRequest{ + Subject: secondSubject, ExpectedVersion: "v0", IdempotencyKey: key, Reason: moderationConcurrencyReason, + } + second, err := service.RemoveContent(t.Context(), actor, secondRequest) + require.NoError(t, err, "an expired key must not conflict with a new request") + require.NotNil(t, second) + require.Equal(t, moderation.OutcomeApplied, second.Outcome) + fingerprint, createdAt, expiresAt := storedKey() + assert.NotEqual(t, firstFingerprint, fingerprint) + assert.WithinDuration(t, now, createdAt, time.Second) + assert.WithinDuration(t, now.Add(time.Hour), expiresAt, time.Second) + var rows int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1`, actor).Scan(&rows)) + assert.Equal(t, 1, rows) + + replayed, err := service.RemoveContent(t.Context(), actor, secondRequest) + require.NoError(t, err) + assert.Equal(t, second, replayed, "the replaced row must replay the new result") + assert.Equal(t, 1, countModerationActions(t, db, firstSubject.URI, moderation.ActionRemove)) + assert.Equal(t, 1, countModerationActions(t, db, secondSubject.URI, moderation.ActionRemove)) +} diff --git a/internal/db/postgres/moderation_media_blocks_integration_test.go b/internal/db/postgres/moderation_media_blocks_integration_test.go new file mode 100644 index 0000000..f1fa3d5 --- /dev/null +++ b/internal/db/postgres/moderation_media_blocks_integration_test.go @@ -0,0 +1,62 @@ +//go:build integration + +package postgres_test + +import ( + "fmt" + "testing" + + "Coves/internal/core/imageproxy" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestModerationRepositoryListsActiveBlockedBlobs(t *testing.T) { + db := testkit.DB(t) + service := newPostgresModerationService(db) + remove := func(subject moderation.StrongRef, reason string) *moderation.MutationResult { + t.Helper() + result, err := service.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "modadmin")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-" + testkit.UniqueID(t), Reason: reason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + return result + } + const spam = "social.coves.moderation.defs#reasonSpam" + embed := fmt.Sprintf(`{"$type":"social.coves.embed.images","images":[{"image":{"$type":"blob","ref":{"$link":"%s"},"mimeType":"image/png","size":10},"alt":""}]}`, moderationImageCIDOne) + + spamSubject, spamOwner, _ := indexedModerationComment(t, db, false, embed) + remove(spamSubject, spam) + // A second removal of the same owner's images adds duplicate block rows. + rkey := testkit.TID() + duplicateSubject := moderation.StrongRef{URI: "at://" + spamOwner + "/" + moderation.CommentCollection + "/" + rkey, CID: moderationCommentCID} + _, err := db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, 'bafyreiunindexedroot', $5, 'bafyreiunindexedroot', 'second comment', $6::jsonb, NOW()) + `, duplicateSubject.URI, duplicateSubject.CID, rkey, spamOwner, + "at://"+fixtures.DID(testkit.UniqueIDWithPrefix(t, "modroot"))+"/"+moderation.PostV2Collection+"/"+testkit.TID(), embed) + require.NoError(t, err) + remove(duplicateSubject, spam) + + illegalSubject, illegalOwner, _ := indexedModerationComment(t, db, false, embed) + remove(illegalSubject, "social.coves.moderation.defs#reasonIllegalContent") + + inactiveSubject, _, _ := indexedModerationComment(t, db, false, embed) + inactive := remove(inactiveSubject, spam) + _, err = db.ExecContext(t.Context(), `UPDATE moderation_media_blocks SET active = FALSE WHERE action_id = $1`, inactive.Action.ID) + require.NoError(t, err) + + blobs, err := postgres.NewModerationRepository(db).ListActiveBlockedBlobs(t.Context()) + require.NoError(t, err) + assert.ElementsMatch(t, []imageproxy.BlockedBlob{ + {OwnerDID: spamOwner, CID: moderationImageCIDOne}, + {OwnerDID: illegalOwner, CID: moderationImageCIDOne}, + {CID: moderationImageCIDOne}, + }, blobs, "each active block once, every-owner blocks with an empty owner, inactive blocks left out") +} diff --git a/internal/db/postgres/moderation_repo.go b/internal/db/postgres/moderation_repo.go new file mode 100644 index 0000000..c92b521 --- /dev/null +++ b/internal/db/postgres/moderation_repo.go @@ -0,0 +1,415 @@ +package postgres + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "database/sql" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "time" + + "Coves/internal/core/embeds" + "Coves/internal/core/imageproxy" + "Coves/internal/core/moderation" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// ModerationRepository is the Postgres moderation.Store. +type ModerationRepository struct { + db *sql.DB +} + +// NewModerationRepository builds the Postgres moderation store. +func NewModerationRepository(db *sql.DB) *ModerationRepository { + return &ModerationRepository{db: db} +} + +// InTransaction runs fn in one transaction. +func (r *ModerationRepository) InTransaction(ctx context.Context, fn func(ctx context.Context, tx moderation.Transaction) error) (err error) { + tx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer func() { + if rollbackErr := tx.Rollback(); rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) { + err = errors.Join(err, rollbackErr) + } + }() + if err := fn(ctx, &moderationTransaction{tx: tx}); err != nil { + return err + } + return tx.Commit() +} + +// SubjectModeration reads the stored moderation state of a subject. +func (r *ModerationRepository) SubjectModeration(ctx context.Context, authorityDID, subjectURI string) (*moderation.SubjectModeration, error) { + var version int64 + row := r.db.QueryRowContext(ctx, ` + SELECT COALESCE(s.version, 0), + a.id, a.actor_did, a.authority_did, a.scope_kind, a.scope_community_did, + a.subject_uri, a.subject_collection, a.subject_community_did, a.observed_cid, + a.action, a.label_value, a.reason, a.private_classification, a.private_note, + a.reverses_action_id, a.origin, a.created_at + FROM (SELECT $2::text AS uri) AS target + LEFT JOIN moderation_subjects s ON s.subject_uri = target.uri + LEFT JOIN moderation_decisions d ON d.subject_uri = target.uri + AND d.authority_did = $1 AND d.scope_kind = 'instance' + AND d.kind = 'removal' AND d.active + LEFT JOIN moderation_actions a ON a.id = d.active_action_id + `, authorityDID, subjectURI) + action, err := scanModerationAction(row, &version) + if err != nil { + return nil, err + } + return &moderation.SubjectModeration{Version: version, ActiveRemoval: action}, nil +} + +type moderationTransaction struct { + tx *sql.Tx +} + +var moderationActionClock = newModerationActionClock() + +func newModerationActionClock() *syntax.TIDClock { + var clockID [2]byte + if _, err := rand.Read(clockID[:]); err != nil { + panic(fmt.Sprintf("moderation action clock: %v", err)) + } + return syntax.NewTIDClock(uint(binary.BigEndian.Uint16(clockID[:]) % 1024)) +} + +func (t *moderationTransaction) LockActor(ctx context.Context, actorDID string) error { + // One lock per actor serializes admission for distinct idempotency keys as + // well as replays. Namespace the hash away from other advisory lock users. + digest := sha256.Sum256([]byte("coves/moderation-actor/" + actorDID)) + key := int64(binary.BigEndian.Uint64(digest[:8])) + _, err := t.tx.ExecContext(ctx, `SELECT pg_advisory_xact_lock($1)`, key) + return err +} + +func (t *moderationTransaction) LiveIdempotencyRecord(ctx context.Context, actorDID, authorityDID, key string, now time.Time) (*moderation.IdempotencyRecord, error) { + var record moderation.IdempotencyRecord + var storedResult []byte + err := t.tx.QueryRowContext(ctx, ` + SELECT fingerprint, stored_result, created_at, expires_at + FROM moderation_idempotency_keys + WHERE actor_did = $1 AND authority_did = $2 AND key = $3 AND expires_at > $4 + `, actorDID, authorityDID, key, now).Scan(&record.Fingerprint, &storedResult, &record.CreatedAt, &record.ExpiresAt) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + if err != nil { + return nil, err + } + if err := json.Unmarshal(storedResult, &record.Result); err != nil { + return nil, fmt.Errorf("decode stored moderation result: %w", err) + } + record.ActorDID, record.AuthorityDID, record.Key = actorDID, authorityDID, key + return &record, nil +} + +func (t *moderationTransaction) CountLiveIdempotencyKeys(ctx context.Context, actorDID string, now time.Time) (int, error) { + var count int + err := t.tx.QueryRowContext(ctx, ` + SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1 AND expires_at > $2 + `, actorDID, now).Scan(&count) + return count, err +} + +func (t *moderationTransaction) SaveIdempotencyRecord(ctx context.Context, record moderation.IdempotencyRecord) error { + result, err := json.Marshal(record.Result) + if err != nil { + return err + } + _, err = t.tx.ExecContext(ctx, ` + INSERT INTO moderation_idempotency_keys + (actor_did, authority_did, key, fingerprint, stored_result, created_at, expires_at) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (actor_did, authority_did, key) DO UPDATE SET + fingerprint = EXCLUDED.fingerprint, stored_result = EXCLUDED.stored_result, + created_at = EXCLUDED.created_at, expires_at = EXCLUDED.expires_at + WHERE moderation_idempotency_keys.expires_at <= EXCLUDED.created_at + `, record.ActorDID, record.AuthorityDID, record.Key, record.Fingerprint, result, record.CreatedAt, record.ExpiresAt) + return err +} + +func (t *moderationTransaction) LockSubject(ctx context.Context, subjectURI string) (int64, error) { + if _, err := t.tx.ExecContext(ctx, ` + INSERT INTO moderation_subjects (subject_uri, version) VALUES ($1, 0) + ON CONFLICT (subject_uri) DO NOTHING + `, subjectURI); err != nil { + return 0, err + } + var version int64 + err := t.tx.QueryRowContext(ctx, ` + SELECT version FROM moderation_subjects WHERE subject_uri = $1 FOR UPDATE + `, subjectURI).Scan(&version) + return version, err +} + +func (t *moderationTransaction) ReadIndexedComment(ctx context.Context, subjectURI string) (*moderation.IndexedComment, error) { + var comment moderation.IndexedComment + var communityDID, embed sql.NullString + err := t.tx.QueryRowContext(ctx, ` + SELECT c.uri, c.cid, c.commenter_did, + (c.deleted_at IS NOT NULL AND c.deletion_reason = 'author'), + p.community_did, c.embed + FROM comments c + LEFT JOIN posts p ON p.uri = c.root_uri + WHERE c.uri = $1 FOR SHARE OF c + `, subjectURI).Scan(&comment.URI, &comment.CID, &comment.OwnerDID, + &comment.AuthorDeleted, &communityDID, &embed) + if errors.Is(err, sql.ErrNoRows) { + return nil, moderation.ErrSubjectNotIndexed + } + if err != nil { + return nil, err + } + comment.CommunityDID = communityDID.String + if embed.Valid { + // The embed is author-controlled and stored unvalidated. A shape the + // comment view cannot serve images from has no images to block, and + // must never make the comment impossible to moderate. + var decoded any + if err := json.Unmarshal([]byte(embed.String), &decoded); err == nil { + if object, isObject := decoded.(map[string]any); isObject { + comment.ImageCIDs = embeds.CommentImageCIDs(object) + } + } + } + return &comment, nil +} + +const moderationActionColumns = ` + a.id, a.actor_did, a.authority_did, a.scope_kind, a.scope_community_did, + a.subject_uri, a.subject_collection, a.subject_community_did, a.observed_cid, + a.action, a.label_value, a.reason, a.private_classification, a.private_note, + a.reverses_action_id, a.origin, a.created_at` + +type moderationRow interface { + Scan(dest ...any) error +} + +func scanModerationAction(row moderationRow, version *int64) (*moderation.Action, error) { + var id, actorDID, authorityDID, scopeKind, scopeCommunityDID sql.NullString + var subjectURI, subjectCollection, subjectCommunityDID, observedCID sql.NullString + var actionKind, labelValue, reason, privateClassification, privateNote sql.NullString + var reversesActionID, origin sql.NullString + var createdAt sql.NullTime + dest := []any{ + &id, &actorDID, &authorityDID, &scopeKind, &scopeCommunityDID, + &subjectURI, &subjectCollection, &subjectCommunityDID, &observedCID, + &actionKind, &labelValue, &reason, &privateClassification, &privateNote, + &reversesActionID, &origin, &createdAt, + } + if version != nil { + dest = append([]any{version}, dest...) + } + if err := row.Scan(dest...); err != nil { + return nil, err + } + if !id.Valid { + return nil, nil + } + return &moderation.Action{ + ID: id.String, ActorDID: actorDID.String, AuthorityDID: authorityDID.String, + ScopeKind: scopeKind.String, ScopeCommunityDID: scopeCommunityDID.String, + SubjectURI: subjectURI.String, SubjectCollection: subjectCollection.String, + SubjectCommunityDID: subjectCommunityDID.String, ObservedCID: observedCID.String, + Action: actionKind.String, LabelValue: labelValue.String, Reason: reason.String, + PrivateNote: privateNote.String, ReversesActionID: reversesActionID.String, + Origin: origin.String, CreatedAt: createdAt.Time, + }, nil +} + +func (t *moderationTransaction) GetAction(ctx context.Context, actionID string) (*moderation.Action, error) { + action, err := scanModerationAction(t.tx.QueryRowContext(ctx, + `SELECT `+moderationActionColumns+` FROM moderation_actions a WHERE a.id = $1`, actionID), nil) + if errors.Is(err, sql.ErrNoRows) { + return nil, moderation.ErrDecisionNotFound + } + return action, err +} + +// ActiveRemoval reads the active removal and share-locks its decision row +// until the transaction ends. The comment consumer reconciles media blocks +// through this read, and when the comment row is absent (purged, then +// recreated by a fresh insert) no content-row lock orders it against a +// restore. The share lock does: a restore that already deactivated the +// decision makes this read wait and then re-check d.active against the +// committed row, and a restore that arrives later waits at its decision +// update for this transaction's blocks to commit, so its later +// DeactivateMediaBlocks sees them. Mutations take it after the subject lock +// and the consumer after its own row write; neither then waits on a lock the +// other holds, so the order cannot deadlock. +func (t *moderationTransaction) ActiveRemoval(ctx context.Context, authorityDID, subjectURI string) (*moderation.Action, error) { + action, err := scanModerationAction(t.tx.QueryRowContext(ctx, ` + SELECT `+moderationActionColumns+` FROM moderation_actions a + JOIN moderation_decisions d ON d.active_action_id = a.id + WHERE d.authority_did = $1 AND d.subject_uri = $2 + AND d.scope_kind = 'instance' AND d.kind = 'removal' AND d.active + FOR SHARE OF d + `, authorityDID, subjectURI), nil) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return action, err +} + +func (t *moderationTransaction) InsertAction(ctx context.Context, action moderation.Action) (*moderation.Action, error) { + action.ID = moderationActionClock.Next().String() + // Postgres timestamps have microsecond precision. Return the same instant + // that will be read back, without a monotonic clock or local time zone. + action.CreatedAt = action.CreatedAt.UTC().Truncate(time.Microsecond) + _, err := t.tx.ExecContext(ctx, ` + INSERT INTO moderation_actions + (id, actor_did, authority_did, scope_kind, scope_community_did, + subject_uri, subject_collection, subject_community_did, observed_cid, + action, label_value, reason, private_note, reverses_action_id, origin, created_at) + VALUES ($1, $2, $3, $4, NULLIF($5, ''), $6, $7, NULLIF($8, ''), NULLIF($9, ''), + $10, NULLIF($11, ''), NULLIF($12, ''), NULLIF($13, ''), NULLIF($14, ''), $15, $16) + `, action.ID, action.ActorDID, action.AuthorityDID, action.ScopeKind, action.ScopeCommunityDID, + action.SubjectURI, action.SubjectCollection, action.SubjectCommunityDID, action.ObservedCID, + action.Action, action.LabelValue, action.Reason, action.PrivateNote, action.ReversesActionID, + action.Origin, action.CreatedAt) + if err != nil { + return nil, err + } + return &action, nil +} + +func (t *moderationTransaction) SetRemovalDecision(ctx context.Context, authorityDID, subjectURI, actionID string, active bool) error { + if !active { + result, err := t.tx.ExecContext(ctx, ` + UPDATE moderation_decisions SET active = FALSE + WHERE authority_did = $1 AND subject_uri = $2 AND scope_kind = 'instance' + AND kind = 'removal' AND active_action_id = $3 AND active + `, authorityDID, subjectURI, actionID) + if err != nil { + return err + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count != 1 { + return fmt.Errorf("active moderation decision not found for restore") + } + return nil + } + _, err := t.tx.ExecContext(ctx, ` + INSERT INTO moderation_decisions + (authority_did, scope_kind, scope_community_did, subject_uri, kind, value, active_action_id, active) + VALUES ($1, 'instance', NULL, $2, 'removal', NULL, $3, TRUE) + ON CONFLICT ON CONSTRAINT moderation_decisions_key DO UPDATE SET + active_action_id = EXCLUDED.active_action_id, active = TRUE + `, authorityDID, subjectURI, actionID) + return err +} + +func (t *moderationTransaction) SetSubjectVersion(ctx context.Context, subjectURI string, version int64) error { + result, err := t.tx.ExecContext(ctx, ` + UPDATE moderation_subjects SET version = $2, updated_at = NOW() WHERE subject_uri = $1 + `, subjectURI, version) + if err != nil { + return err + } + count, err := result.RowsAffected() + if err != nil { + return err + } + if count != 1 { + return fmt.Errorf("moderation subject version row not found") + } + return nil +} + +func (t *moderationTransaction) InsertMediaBlocks(ctx context.Context, blocks []moderation.MediaBlock) error { + _, err := t.InsertNewMediaBlocks(ctx, blocks) + return err +} + +// InsertNewMediaBlocks returns only blocks inserted by this transaction, for +// post-commit cache purging. The NULLS NOT DISTINCT constraint deduplicates +// ownerless blocks as well as owner-scoped blocks. +func (t *moderationTransaction) InsertNewMediaBlocks(ctx context.Context, blocks []moderation.MediaBlock) ([]moderation.MediaBlock, error) { + var inserted []moderation.MediaBlock + for _, block := range blocks { + var id int64 + err := t.tx.QueryRowContext(ctx, ` + INSERT INTO moderation_media_blocks (owner_did, blob_cid, action_id, active) + VALUES (NULLIF($1, ''), $2, $3, TRUE) + ON CONFLICT ON CONSTRAINT moderation_media_block_key DO NOTHING + RETURNING id + `, block.OwnerDID, block.BlobCID, block.ActionID).Scan(&id) + if errors.Is(err, sql.ErrNoRows) { + continue + } + if err != nil { + return nil, err + } + inserted = append(inserted, block) + } + return inserted, nil +} + +func (t *moderationTransaction) DeactivateMediaBlocks(ctx context.Context, actionID string) error { + _, err := t.tx.ExecContext(ctx, ` + UPDATE moderation_media_blocks SET active = FALSE WHERE action_id = $1 AND active + `, actionID) + return err +} + +// DeleteExpiredIdempotencyKeys deletes idempotency rows expired at now. +func (r *ModerationRepository) DeleteExpiredIdempotencyKeys(ctx context.Context, now time.Time) (int64, error) { + result, err := r.db.ExecContext(ctx, `DELETE FROM moderation_idempotency_keys WHERE expires_at <= $1`, now) + if err != nil { + return 0, err + } + return result.RowsAffected() +} + +// IsBlocked reports whether an active media block covers the owner's blob. +func (r *ModerationRepository) IsBlocked(ctx context.Context, ownerDID, cid string) (bool, error) { + var blocked bool + err := r.db.QueryRowContext(ctx, ` + SELECT EXISTS ( + SELECT 1 FROM moderation_media_blocks + WHERE owner_did = $1 AND blob_cid = $2 AND active + ) OR EXISTS ( + SELECT 1 FROM moderation_media_blocks + WHERE owner_did IS NULL AND blob_cid = $2 AND active + ) + `, ownerDID, cid).Scan(&blocked) + return blocked, err +} + +// ListActiveBlockedBlobs lists every blob an active media block covers, once +// each. An every-owner block comes back with an empty OwnerDID. +func (r *ModerationRepository) ListActiveBlockedBlobs(ctx context.Context) ([]imageproxy.BlockedBlob, error) { + rows, err := r.db.QueryContext(ctx, ` + SELECT DISTINCT COALESCE(owner_did, ''), blob_cid FROM moderation_media_blocks WHERE active + `) + if err != nil { + return nil, err + } + defer rows.Close() + var blobs []imageproxy.BlockedBlob + for rows.Next() { + var blob imageproxy.BlockedBlob + if err := rows.Scan(&blob.OwnerDID, &blob.CID); err != nil { + return nil, err + } + blobs = append(blobs, blob) + } + return blobs, rows.Err() +} + +// BindTransaction binds media reconciliation operations to tx. +func (r *ModerationRepository) BindTransaction(tx *sql.Tx) moderation.MediaTransaction { + return &moderationTransaction{tx: tx} +} diff --git a/internal/db/postgres/moderation_repo_integration_test.go b/internal/db/postgres/moderation_repo_integration_test.go new file mode 100644 index 0000000..51e356d --- /dev/null +++ b/internal/db/postgres/moderation_repo_integration_test.go @@ -0,0 +1,281 @@ +//go:build integration + +package postgres_test + +import ( + "database/sql" + "fmt" + "testing" + "time" + + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + moderationImageCIDOne = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" + moderationImageCIDTwo = "bafkreicy44vctf2bgqnn5wwzdern7bc2khwi7ku2r66bozl4x6bsrvuj2q" + moderationCommentCID = "bafyreigj3fwnwjuzr35k2kuzmb5dixxczrzjhqkr5srlqplsh6gq3bj3si" +) + +func moderationTwoImageEmbed() string { + return fmt.Sprintf(`{"$type":"social.coves.embed.images","images":[{"image":{"$type":"blob","ref":{"$link":"%s"},"mimeType":"image/png","size":10},"alt":""},{"image":{"$type":"blob","ref":{"$link":"%s"},"mimeType":"image/png","size":10},"alt":""}]}`, moderationImageCIDOne, moderationImageCIDTwo) +} + +func indexedModerationComment(t *testing.T, db *sql.DB, indexedRoot bool, embed string) (moderation.StrongRef, string, string) { + t.Helper() + authorName := testkit.UniqueIDWithPrefix(t, "modauthor") + authorDID := fixtures.DID(authorName) + fixtures.User(t, db, authorName+".test", authorDID) + communityDID := "" + rootURI := "at://" + fixtures.DID(testkit.UniqueIDWithPrefix(t, "modroot")) + "/" + moderation.PostV2Collection + "/" + testkit.TID() + rootCID := "bafyreiunindexedroot" + if indexedRoot { + communityName := testkit.UniqueIDWithPrefix(t, "modcommunity") + var err error + communityDID, err = fixtures.Community(t.Context(), db, communityName, "owner"+communityName) + require.NoError(t, err) + rootURI = fixtures.Post(t, db, communityDID, authorDID, "moderated post", 0, time.Now()) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), rootURI) + require.NoError(t, err) + rootCID = post.CID + } + rkey := testkit.TID() + subject := moderation.StrongRef{ + URI: "at://" + authorDID + "/" + moderation.CommentCollection + "/" + rkey, + CID: moderationCommentCID, + } + _, err := db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $5, $6, 'original indexed content', NULLIF($7, '')::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, authorDID, rootURI, rootCID, embed) + require.NoError(t, err) + return subject, authorDID, communityDID +} + +func newPostgresModerationService(db *sql.DB) moderation.Service { + return moderation.NewService( + moderation.NewRepositorySubjectReader(postgres.NewPostRepository(db), postgres.NewCommentRepository(db)), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) +} + +func TestModerationRepositoryPersistsRemovalAndAssociation(t *testing.T) { + for _, test := range []struct { + name string + indexedRoot bool + embed string + }{ + {name: "indexed root captures community and image blocks", indexedRoot: true, embed: moderationTwoImageEmbed()}, + {name: "unindexed root has no community association", indexedRoot: false}, + } { + t.Run(test.name, func(t *testing.T) { + db := testkit.DB(t) + subject, authorDID, communityDID := indexedModerationComment(t, db, test.indexedRoot, test.embed) + service := newPostgresModerationService(db) + adminA := fixtures.DID(testkit.UniqueIDWithPrefix(t, "modadmina")) + adminB := fixtures.DID(testkit.UniqueIDWithPrefix(t, "modadminb")) + request := moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-a", + Reason: "social.coves.moderation.defs#reasonSpam", PrivateNote: "reviewed report", + } + + removed, err := service.RemoveContent(t.Context(), adminA, request) + require.NoError(t, err) + require.NotNil(t, removed) + require.NotNil(t, removed.Action) + assert.Equal(t, moderation.OutcomeApplied, removed.Outcome) + assert.Equal(t, "v1", removed.State.Version) + assert.Equal(t, moderation.ModerationStateRemoved, removed.State.Moderation.State) + require.NotNil(t, removed.State.LocalRemoval) + assert.Equal(t, removed.Action.ID, removed.State.LocalRemoval.ActionID) + + var actionCount int + var actor, authority, scope, collection, observedCID, actionKind, reason, note, origin string + var associatedCommunity sql.NullString + err = db.QueryRowContext(t.Context(), ` + SELECT actor_did, authority_did, scope_kind, subject_collection, + subject_community_did, observed_cid, action, reason, private_note, origin + FROM moderation_actions WHERE subject_uri = $1 AND id = $2 + `, subject.URI, removed.Action.ID).Scan( + &actor, &authority, &scope, &collection, &associatedCommunity, + &observedCID, &actionKind, &reason, ¬e, &origin, + ) + require.NoError(t, err) + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_actions WHERE subject_uri = $1`, subject.URI).Scan(&actionCount)) + assert.Equal(t, 1, actionCount) + assert.Equal(t, adminA, actor) + assert.Equal(t, fixtures.InstanceDID(), authority) + assert.Equal(t, moderation.ScopeInstance, scope) + assert.Equal(t, moderation.CommentCollection, collection) + assert.Equal(t, sql.NullString{String: communityDID, Valid: test.indexedRoot}, associatedCommunity) + assert.Equal(t, subject.CID, observedCID) + assert.Equal(t, moderation.ActionRemove, actionKind) + assert.Equal(t, request.Reason, reason) + assert.Equal(t, request.PrivateNote, note) + assert.Equal(t, moderation.OriginLocal, origin) + + var version int64 + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT version FROM moderation_subjects WHERE subject_uri = $1`, subject.URI).Scan(&version)) + assert.EqualValues(t, 1, version) + var decisionCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_decisions WHERE subject_uri = $1`, subject.URI).Scan(&decisionCount)) + assert.Equal(t, 1, decisionCount) + var activeAction string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT active_action_id FROM moderation_decisions + WHERE subject_uri = $1 AND authority_did = $2 AND scope_kind = 'instance' AND kind = 'removal' AND active + `, subject.URI, fixtures.InstanceDID()).Scan(&activeAction)) + assert.Equal(t, removed.Action.ID, activeAction) + var keyCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys`).Scan(&keyCount)) + assert.Equal(t, 1, keyCount) + var storedKeyCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_idempotency_keys WHERE actor_did = $1 AND authority_did = $2 AND key = $3`, adminA, fixtures.InstanceDID(), "remove-a").Scan(&storedKeyCount)) + assert.Equal(t, 1, storedKeyCount) + + var blockCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_media_blocks WHERE action_id = $1`, removed.Action.ID).Scan(&blockCount)) + if test.indexedRoot { + assert.Equal(t, 2, blockCount) + rows, err := db.QueryContext(t.Context(), `SELECT owner_did, blob_cid, action_id, active FROM moderation_media_blocks WHERE action_id = $1`, removed.Action.ID) + require.NoError(t, err) + seen := make(map[string]bool) + for rows.Next() { + var owner sql.NullString + var blobCID, actionID string + var active bool + require.NoError(t, rows.Scan(&owner, &blobCID, &actionID, &active)) + assert.Equal(t, sql.NullString{String: authorDID, Valid: true}, owner, "spam blocks only the comment owner's blob") + assert.Equal(t, removed.Action.ID, actionID) + assert.True(t, active) + seen[blobCID] = true + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + assert.Equal(t, map[string]bool{moderationImageCIDOne: true, moderationImageCIDTwo: true}, seen) + } else { + assert.Zero(t, blockCount) + } + + var originalContent, indexedCID string + var storedEmbed sql.NullString + var deletedAt sql.NullTime + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT content, cid, embed::text, deleted_at FROM comments WHERE uri = $1`, subject.URI).Scan(&originalContent, &indexedCID, &storedEmbed, &deletedAt)) + assert.Equal(t, "original indexed content", originalContent, "removal must overlay, not alter the indexed row") + assert.Equal(t, subject.CID, indexedCID) + if test.indexedRoot { + assert.JSONEq(t, test.embed, storedEmbed.String) + } + assert.False(t, deletedAt.Valid) + + state, err := service.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, "v1", state.Version) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + + request.ExpectedVersion = "v1" + request.IdempotencyKey = "remove-b" + unchanged, err := service.RemoveContent(t.Context(), adminB, request) + require.NoError(t, err) + require.NotNil(t, unchanged) + assert.Equal(t, moderation.OutcomeUnchanged, unchanged.Outcome) + assert.Nil(t, unchanged.Action) + assert.Equal(t, "v1", unchanged.State.Version) + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_actions WHERE subject_uri = $1`, subject.URI).Scan(&actionCount)) + assert.Equal(t, 1, actionCount) + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT version FROM moderation_subjects WHERE subject_uri = $1`, subject.URI).Scan(&version)) + assert.EqualValues(t, 1, version) + }) + } +} + +func TestModerationRepositoryRollsBackAfterActionInsertFailure(t *testing.T) { + db := testkit.DB(t) + subject, _, _ := indexedModerationComment(t, db, true, moderationTwoImageEmbed()) + _, err := db.ExecContext(t.Context(), ` + CREATE FUNCTION reject_moderation_decision() RETURNS trigger LANGUAGE plpgsql AS $$ + BEGIN + RAISE EXCEPTION 'injected decision failure after action insert'; + END; + $$; + `) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + CREATE TRIGGER reject_moderation_decision BEFORE INSERT ON moderation_decisions + FOR EACH ROW EXECUTE FUNCTION reject_moderation_decision(); + `) + require.NoError(t, err) + result, err := newPostgresModerationService(db).RemoveContent(t.Context(), fixtures.DID("modrollback"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "rollback-key", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.ErrorIs(t, err, moderation.ErrModerationUnavailable) + assert.ErrorContains(t, err, "injected decision failure after action insert") + assert.Nil(t, result) + for _, test := range []struct { + name string + query string + }{ + {"action", `SELECT count(*) FROM moderation_actions WHERE subject_uri = $1`}, + {"decision", `SELECT count(*) FROM moderation_decisions WHERE subject_uri = $1`}, + {"version greater than zero", `SELECT count(*) FROM moderation_subjects WHERE subject_uri = $1 AND version > 0`}, + {"idempotency key", `SELECT count(*) FROM moderation_idempotency_keys WHERE key = 'rollback-key' AND stored_result IS NOT NULL`}, + } { + t.Run(test.name, func(t *testing.T) { + var count int + if test.name == "idempotency key" { + require.NoError(t, db.QueryRowContext(t.Context(), test.query).Scan(&count)) + } else { + require.NoError(t, db.QueryRowContext(t.Context(), test.query, subject.URI).Scan(&count)) + } + assert.Zero(t, count) + }) + } + var mediaBlockCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_media_blocks`).Scan(&mediaBlockCount)) + assert.Zero(t, mediaBlockCount) +} + +func TestModerationRepositoryRemovesAuthorDeletedCommentWithDifferentCID(t *testing.T) { + db := testkit.DB(t) + subject, authorDID, _ := indexedModerationComment(t, db, true, "") + _, err := db.ExecContext(t.Context(), ` + UPDATE comments SET deleted_at = NOW(), deletion_reason = 'author', deleted_by = $1 WHERE uri = $2 + `, authorDID, subject.URI) + require.NoError(t, err) + lastIndexedCID := subject.CID + subject.CID = moderationImageCIDTwo // Deliberately different from the last indexed comment CID. + removed, err := newPostgresModerationService(db).RemoveContent(t.Context(), fixtures.DID("moddeleteadmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "deleted-comment-removal", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err) + require.NotNil(t, removed) + require.NotNil(t, removed.Action) + assert.Equal(t, moderation.OutcomeApplied, removed.Outcome) + assert.Equal(t, moderation.RecordStateDeleted, removed.State.RecordState) + assert.Equal(t, moderation.ModerationStateRemoved, removed.State.Moderation.State) + var observedCID string + var actionCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT observed_cid FROM moderation_actions WHERE id = $1 AND subject_uri = $2 AND action = 'remove'`, removed.Action.ID, subject.URI).Scan(&observedCID)) + assert.Equal(t, lastIndexedCID, observedCID) + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_actions WHERE subject_uri = $1`, subject.URI).Scan(&actionCount)) + assert.Equal(t, 1, actionCount) + state, err := newPostgresModerationService(db).GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + require.NotNil(t, state) + assert.Equal(t, moderation.RecordStateDeleted, state.RecordState) + assert.Nil(t, state.CurrentSubject) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) +} diff --git a/internal/db/postgres/rematerialize_ledger_schema_test.go b/internal/db/postgres/rematerialize_ledger_schema_test.go index 4bae6d4..b3c1f47 100644 --- a/internal/db/postgres/rematerialize_ledger_schema_test.go +++ b/internal/db/postgres/rematerialize_ledger_schema_test.go @@ -196,6 +196,8 @@ func TestRematerializeLedgerMigration_RollsBack(t *testing.T) { // 037 and come off first, one asserted step at a time. Asserting which // migration rolled back is what keeps this pointed at 037's Down rather than // drifting onto a newer one later. + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "048 (Discover Hot snapshot tables) must be rolled back before testing earlier migrations") require.EqualValues(t, 47, testkit.MigrateDownOne(t, db, 47), diff --git a/internal/db/postgres/vote_drift_recount_migration_test.go b/internal/db/postgres/vote_drift_recount_migration_test.go index fd06ec7..75878b8 100644 --- a/internal/db/postgres/vote_drift_recount_migration_test.go +++ b/internal/db/postgres/vote_drift_recount_migration_test.go @@ -80,6 +80,8 @@ func TestMigration040_RecountsVoteDriftAndSweepsLegacyOrphans(t *testing.T) { // point of a repair migration and cannot be observed by seeding after it has // run. Asserting the version that came off is the tripwire that keeps this // pointed at 040 when later migrations land. + require.EqualValues(t, 49, testkit.MigrateDownOne(t, db, 49), + "049 (moderation state) must be rolled back before testing earlier migrations") require.EqualValues(t, 48, testkit.MigrateDownOne(t, db, 48), "048 (Discover Hot snapshot tables) must be rolled back before testing earlier migrations") require.EqualValues(t, 47, testkit.MigrateDownOne(t, db, 47), diff --git a/tests/e2e/comment_contract_test.go b/tests/e2e/comment_contract_test.go index 9abb8ea..edac13d 100644 --- a/tests/e2e/comment_contract_test.go +++ b/tests/e2e/comment_contract_test.go @@ -127,18 +127,22 @@ const commentCollection = "social.coves.community.comment" // observe. As elsewhere in this package, modelling only the asserted fields // keeps a new lexicon field from breaking every contract that reads a comment. type commentView struct { - URI string `json:"uri"` - CID string `json:"cid"` - Author identityRef `json:"author"` - Record map[string]any `json:"record"` - Post *strongRef `json:"post"` - Parent *strongRef `json:"parent"` - Stats commentStats `json:"stats"` - CreatedAt string `json:"createdAt"` - IndexedAt string `json:"indexedAt"` - IsDeleted bool `json:"isDeleted"` - DeletionReason *string `json:"deletionReason"` - DeletedAt *string `json:"deletedAt"` + URI string `json:"uri"` + CID string `json:"cid"` + Author identityRef `json:"author"` + Record map[string]any `json:"record"` + Embed map[string]any `json:"embed"` + Moderation *struct { + State string `json:"state"` + } `json:"moderation"` + Post *strongRef `json:"post"` + Parent *strongRef `json:"parent"` + Stats commentStats `json:"stats"` + CreatedAt string `json:"createdAt"` + IndexedAt string `json:"indexedAt"` + IsDeleted bool `json:"isDeleted"` + DeletionReason *string `json:"deletionReason"` + DeletedAt *string `json:"deletedAt"` } // strongRef is the uri+cid pair the lexicon uses for threading references. diff --git a/tests/e2e/moderation_contract_test.go b/tests/e2e/moderation_contract_test.go new file mode 100644 index 0000000..c1ade12 --- /dev/null +++ b/tests/e2e/moderation_contract_test.go @@ -0,0 +1,238 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "net/http" + "net/url" + "strings" + "testing" + + "Coves/tests/testkit" + + "github.com/stretchr/testify/require" +) + +const ( + removeContentMethod = "social.coves.moderation.removeContent" + restoreContentMethod = "social.coves.moderation.restoreContent" +) + +// TestModerationCommentRemovalContract crosses the real PDS → consumer → AppView +// boundary twice: once for the comment and again for the author's edit while the +// moderation overlay is active. The edited CID is the delivery barrier for the +// negative assertion: a placeholder seen before that edit was indexed proves +// nothing about whether the edit can resurrect removed content. +func TestModerationCommentRemovalContract(t *testing.T) { + p := newPipeline(t) + author := p.IndexedAccount(t, "mca") + community := indexedCommunity(t, p, "mca", author.DID) + post := indexedPost(t, p, community, author, "moderation image comment") + outsider := p.IndexedAccount(t, "mco") + admin := testkit.ModerationAdmin(t, 1) + + image := author.UploadBlob(t, testkit.TestPNG(64, 64), "image/png") + // The edit introduces a second image. Only the consumer's media + // reconciliation can block it, since the removal never saw it. + editImage := author.UploadBlob(t, testkit.TestPNG(96, 96), "image/png") + require.NotEqual(t, image.CID(), editImage.CID()) + rkey := testkit.TID() + uri := commentURI(author.DID, rkey) + initialText := "original comment " + testkit.UniqueID(t) + editedText := "edited comment " + testkit.UniqueID(t) + writeComment := func(content string, blobs ...testkit.BlobRef) string { + t.Helper() + record := commentRecord(post, post, content) + images := make([]any, 0, len(blobs)) + for _, blob := range blobs { + images = append(images, map[string]any{"image": blobRefValue(blob), "alt": "moderation contract image"}) + } + record["embed"] = map[string]any{"$type": "social.coves.embed.images", "images": images} + return author.PutRecord(t, commentCollection, rkey, record).CID + } + createdCID := writeComment(initialText, image) + + var imageURL string + p.Await(t, "the directly written image comment to appear in the thread", func() (bool, error) { + thread, err := p.Thread(context.Background(), post.URI, nil) + if err != nil { + return false, err + } + node, found := thread.find(uri) + if !found || node.Comment.CID != createdCID || node.Comment.Record["content"] != initialText { + return false, nil + } + images, ok := node.Comment.Embed["images"].([]any) + if !ok || len(images) != 1 { + return false, nil + } + servedImage, ok := images[0].(map[string]any) + if !ok { + return false, nil + } + imageURL, ok = servedImage["fullsize"].(string) + return ok && imageURL != "", nil + }, withReadCadence()) + require.Contains(t, imageURL, image.CID()) + requireServesImage(t, p, "comment image before removal", imageURL) + parsedImage, err := url.Parse(imageURL) + require.NoError(t, err) + require.True(t, strings.HasPrefix(parsedImage.Path, "/img/"), "the served URL must use the AppView image proxy") + + stateToken := admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + readState := func() (moderationSubjectStateResponse, error) { + var response moderationSubjectStateResponse + err := p.AppView.As(stateToken).Query( + context.Background(), subjectStateMethod, url.Values{"subject": {uri}}, &response) + return response, err + } + initialState, err := readState() + require.NoError(t, err) + require.Equal(t, "v0", initialState.State.Version) + require.Equal(t, "clear", initialState.State.Moderation.State) + require.Equal(t, createdCID, initialState.State.CurrentSubject.CID) + + removeInput := map[string]any{ + "subject": map[string]any{"uri": uri, "cid": initialState.State.CurrentSubject.CID}, + "expectedVersion": initialState.State.Version, + "idempotencyKey": testkit.UniqueID(t), + "reason": "social.coves.moderation.defs#reasonSpam", + } + err = p.AppView.As(outsider.ServiceAuth(t, communityInstanceDID, removeContentMethod)).Procedure( + t.Context(), removeContentMethod, removeInput, nil) + requireXRPCRefusal(t, err, http.StatusForbidden, "Forbidden", "a non-admin removal") + unchangedState, err := readState() + require.NoError(t, err) + require.Equal(t, "v0", unchangedState.State.Version) + require.Equal(t, "clear", unchangedState.State.Moderation.State) + + // Re-read immediately before the mutation: the request must carry the + // inspected indexed CID and version, not just the PDS's write response. + inspected, err := readState() + require.NoError(t, err) + require.Equal(t, createdCID, inspected.State.CurrentSubject.CID) + removeInput["subject"] = map[string]any{"uri": uri, "cid": inspected.State.CurrentSubject.CID} + removeInput["expectedVersion"] = inspected.State.Version + var removal struct { + Outcome string `json:"outcome"` + Action struct { + Action struct { + Ref struct { + ActionID string `json:"actionId"` + } `json:"ref"` + } `json:"action"` + } `json:"action"` + } + err = p.AppView.As(admin.ServiceAuth(t, communityInstanceDID, removeContentMethod)).Procedure( + t.Context(), removeContentMethod, removeInput, &removal) + require.NoError(t, err) + require.Equal(t, "applied", removal.Outcome) + require.NotEmpty(t, removal.Action.Action.Ref.ActionID) + + placeholder := func() (bool, error) { + thread, err := p.Thread(context.Background(), post.URI, nil) + if err != nil { + return false, err + } + node, found := thread.find(uri) + if !found { + return false, fmt.Errorf("removed comment disappeared from thread: %s", thread.uris()) + } + comment := node.Comment + if !comment.IsDeleted || comment.Record != nil || comment.Embed != nil || + comment.DeletionReason == nil || *comment.DeletionReason != "moderator" || + comment.Moderation == nil || comment.Moderation.State != "removed" || + comment.Author.Handle != "handle.invalid" { + return false, nil + } + return true, nil + } + // getComments has a separate 20/minute per-IP limit. A phase boundary + // leaves room for both the placeholder wait and the post-edit Holds window. + p.FreshReadQuota(t, "moderator-placeholder") + p.Await(t, "moderator removal to render as a content-free placeholder", placeholder, withReadCadence()) + pathBlocked := func(path string) (bool, error) { + _, err := p.AppView.GetBinary(context.Background(), path) + if testkit.IsStatus(err, http.StatusNotFound) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + } + imageBlocked := func() (bool, error) { return pathBlocked(parsedImage.Path) } + p.Await(t, "the previously served comment image to return 404", imageBlocked) + + // The edit's image is served under the same owner and preset as the + // original, so its proxy path differs only in the CID. + editImagePath := strings.Replace(parsedImage.Path, image.CID(), editImage.CID(), 1) + require.NotEqual(t, parsedImage.Path, editImagePath) + editedCID := writeComment(editedText, image, editImage) + require.NotEqual(t, createdCID, editedCID) + // R8: observe the edited record in the AppView index before asserting that + // the author's putRecord did not undo the active removal. + stateToken = admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + p.Await(t, "getSubjectState to report the author's edited CID", func() (bool, error) { + state, err := readState() + if err != nil { + return false, err + } + return state.State.CurrentSubject.CID == editedCID, nil + }) + p.Holds(t, "the edited comment to remain hidden and both its images blocked", func() (bool, error) { + stillRemoved, err := placeholder() + if err != nil || !stillRemoved { + return stillRemoved, err + } + blocked, err := imageBlocked() + if err != nil || !blocked { + return blocked, err + } + return pathBlocked(editImagePath) + }) + + currentState, err := readState() + require.NoError(t, err) + require.Equal(t, editedCID, currentState.State.CurrentSubject.CID) + require.Equal(t, "removed", currentState.State.Moderation.State) + var restoration struct { + Outcome string `json:"outcome"` + } + err = p.AppView.As(admin.ServiceAuth(t, communityInstanceDID, restoreContentMethod)).Procedure( + t.Context(), restoreContentMethod, map[string]any{ + "actionId": removal.Action.Action.Ref.ActionID, + "reviewedSubject": map[string]any{"uri": uri, "cid": editedCID}, + "expectedVersion": currentState.State.Version, + "idempotencyKey": testkit.UniqueID(t), + "reason": "social.coves.moderation.defs#reasonModeratorDiscretion", + }, &restoration) + require.NoError(t, err) + require.Equal(t, "applied", restoration.Outcome) + + p.FreshReadQuota(t, "restored-comment") + p.Await(t, "restored comment to serve the edited text", func() (bool, error) { + thread, err := p.Thread(context.Background(), post.URI, nil) + if err != nil { + return false, err + } + node, found := thread.find(uri) + return found && !node.Comment.IsDeleted && node.Comment.Record["content"] == editedText && + node.Comment.Moderation == nil, nil + }, withReadCadence()) + for _, path := range []string{parsedImage.Path, editImagePath} { + p.Await(t, "the restored comment's images to serve again", func() (bool, error) { + response, err := p.AppView.GetBinary(context.Background(), path) + if testkit.IsStatus(err, http.StatusNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return response.Status == http.StatusOK && len(response.Body) > 0 && + strings.HasPrefix(response.ContentType, "image/"), nil + }) + } +} -- 2.51.2