From a6574844ad59fcf0df9b00dfbe46e9b83d313c57 Mon Sep 17 00:00:00 2001 From: Bretton Date: Thu, 24 Sep 2026 09:34:01 -0700 Subject: [PATCH] feat(moderation): remove posts from every read path Enable post subjects in instance moderation and enforce removal at the server boundary on every post surface. Community acceptance and removal records are untouched, and restore brings the thread back. - removeContent/restoreContent accept social.coves.community.postv2 and legacy social.coves.community.post subjects. - visiblePostsPredicate excludes actively removed posts from feeds, search, profiles, post counts, Discover Hot (snapshots and continuations refill) and the getComments header. - post.get serves #moderatedPost only to viewers admitted before the removal. - Coves quote embeds are projected to strongRefs; Bluesky quotes are resolved only for app.bsky.feed.post URIs. - embeds.ServableEmbed makes served media equal the blob-derived CIDs. - embeds.PostBlobCIDs and post media reconciliation cover edits and delete-then-recreate, with a rev gate on the soft-deleted path. - The profile comment_count agrees with actor.getComments. - T0/T1/T2 contracts for the post removal arc. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/server/consumers.go | 3 +- cmd/server/wiring.go | 4 +- .../api/handlers/moderation/remove_content.go | 2 +- .../moderation/remove_content_test.go | 8 +- .../api/handlers/post/get_moderated_test.go | 69 ++++ .../getstatus_moderation_integration_test.go | 83 ++++ .../moderation_media_integration_test.go | 5 +- .../moderation_post_media_integration_test.go | 298 ++++++++++++++ ...oderation_post_removal_integration_test.go | 258 ++++++++++++ ...tion_post_served_media_integration_test.go | 145 +++++++ ...ation_quote_projection_integration_test.go | 245 ++++++++++++ internal/atproto/jetstream/authorpost.go | 2 +- .../atproto/jetstream/comment_consumer.go | 24 +- internal/atproto/jetstream/post_consumer.go | 96 ++++- .../post_moderation_consumer_test.go | 366 ++++++++++++++++++ .../comments/comment_servable_embed_test.go | 84 ++++ internal/core/comments/comment_service.go | 23 +- .../core/comments/comment_service_test.go | 14 +- ...ost_removal_read_paths_integration_test.go | 194 ++++++++++ internal/core/embeds/post_blob_cids_test.go | 63 +++ internal/core/embeds/servable_embed_test.go | 254 ++++++++++++ internal/core/embeds/view.go | 175 ++++++++- internal/core/moderation/fake_store_test.go | 16 + internal/core/moderation/indexed_subject.go | 55 +++ internal/core/moderation/media.go | 32 +- .../core/moderation/mutation_validation.go | 9 +- internal/core/moderation/post_rules_test.go | 287 ++++++++++++++ internal/core/moderation/remove.go | 24 +- internal/core/moderation/remove_rules_test.go | 7 + .../core/moderation/remove_validation_test.go | 6 +- internal/core/moderation/restore.go | 20 +- internal/core/moderation/store.go | 16 + internal/core/posts/blob_transform.go | 13 +- .../posts/blob_transform_quote_uri_test.go | 54 +++ internal/core/posts/interfaces.go | 11 + internal/core/posts/post.go | 104 ++++- internal/core/posts/service.go | 94 ++--- .../core/posts/service_author_posts_test.go | 8 + internal/db/postgres/comment_repo.go | 3 +- .../discover_hot_moderation_race_test.go | 143 +++++++ .../moderation_post_integration_test.go | 359 +++++++++++++++++ ...eration_post_tombstone_integration_test.go | 269 +++++++++++++ internal/db/postgres/moderation_repo.go | 35 ++ internal/db/postgres/post_repo.go | 106 ++++- internal/db/postgres/post_repo_cursor_test.go | 8 + internal/db/postgres/post_visibility.go | 34 +- internal/db/postgres/user_repo.go | 28 +- tests/e2e/moderation_contract_test.go | 287 ++++++++++++++ 48 files changed, 4261 insertions(+), 182 deletions(-) create mode 100644 internal/api/handlers/post/get_moderated_test.go create mode 100644 internal/api/handlers/post/getstatus_moderation_integration_test.go create mode 100644 internal/api/routes/moderation_post_media_integration_test.go create mode 100644 internal/api/routes/moderation_post_removal_integration_test.go create mode 100644 internal/api/routes/moderation_post_served_media_integration_test.go create mode 100644 internal/api/routes/moderation_quote_projection_integration_test.go create mode 100644 internal/atproto/jetstream/post_moderation_consumer_test.go create mode 100644 internal/core/comments/comment_servable_embed_test.go create mode 100644 internal/core/comments/post_removal_read_paths_integration_test.go create mode 100644 internal/core/embeds/post_blob_cids_test.go create mode 100644 internal/core/embeds/servable_embed_test.go create mode 100644 internal/core/moderation/indexed_subject.go create mode 100644 internal/core/moderation/post_rules_test.go create mode 100644 internal/core/posts/blob_transform_quote_uri_test.go create mode 100644 internal/db/postgres/discover_hot_moderation_race_test.go create mode 100644 internal/db/postgres/moderation_post_integration_test.go create mode 100644 internal/db/postgres/moderation_post_tombstone_integration_test.go diff --git a/cmd/server/consumers.go b/cmd/server/consumers.go index fbaa453..324054c 100644 --- a/cmd/server/consumers.go +++ b/cmd/server/consumers.go @@ -230,6 +230,7 @@ func (a *application) registerFeedConsumers() ([]feedConsumer, error) { jetstream.WithAdmissions(a.admissionRepo), jetstream.WithDeletedAccounts(postgresRepo.NewDeletedAccountRepository(a.db)), jetstream.WithPostRecordFetcher(postFetcher), + jetstream.WithPostMediaReconciler(a.mediaReconciler), // The host-side half of an author's own deletion (§5.3): when the // author tombstones a post this instance's community accepted, the // acceptance in that community's repo is withdrawn. It refuses @@ -264,7 +265,7 @@ func (a *application) registerFeedConsumers() ([]feedConsumer, error) { name: jetstream.ConsumerComments, handler: jetstream.NewCommentEventConsumer(a.commentRepo, a.db, jetstream.WithCommentBridgeTrust(a.bridgeTrust), - jetstream.WithCommentMediaReconciler(a.commentMediaReconciler)), + jetstream.WithCommentMediaReconciler(a.mediaReconciler)), }) return consumers, nil diff --git a/cmd/server/wiring.go b/cmd/server/wiring.go index c847f86..6be96c7 100644 --- a/cmd/server/wiring.go +++ b/cmd/server/wiring.go @@ -139,7 +139,7 @@ type application struct { userBlockService userblocks.Service adminReportService adminreports.Service moderationService moderation.Service - commentMediaReconciler jetstream.CommentMediaReconciler + mediaReconciler *moderation.MediaReconciler communitySuggestionService communitysuggestions.Service feedService communityFeeds.Service timelineService timeline.Service @@ -221,7 +221,7 @@ func buildApplication( Purger: purger, }, ) - app.commentMediaReconciler = moderation.NewMediaReconciler( + app.mediaReconciler = moderation.NewMediaReconciler( postgresRepo.NewModerationRepository(app.db), app.cfg.Instance.DID, purger) app.buildJetstreamInfrastructure() if err = app.buildBridgedVotePoller(); err != nil { diff --git a/internal/api/handlers/moderation/remove_content.go b/internal/api/handlers/moderation/remove_content.go index 79f548b..216a772 100644 --- a/internal/api/handlers/moderation/remove_content.go +++ b/internal/api/handlers/moderation/remove_content.go @@ -172,7 +172,7 @@ func writeMutationError(w http.ResponseWriter, operation string, err error) { } { if errors.Is(err, entry.cause) { // Rule errors carry only fixed text and configured limits, such as - // "post removal is unsupported" or the live-key limit, never request + // "unsupported subject collection" or the live-key limit, never request // payloads, so the detail is safe to show the caller. xrpc.WriteError(w, http.StatusBadRequest, entry.code, strings.TrimPrefix(err.Error(), "moderation: ")) return diff --git a/internal/api/handlers/moderation/remove_content_test.go b/internal/api/handlers/moderation/remove_content_test.go index 9224c02..3b8ac38 100644 --- a/internal/api/handlers/moderation/remove_content_test.go +++ b/internal/api/handlers/moderation/remove_content_test.go @@ -224,16 +224,16 @@ func mutationErrorMessage(t *testing.T, response *httptest.ResponseRecorder) str return message } -func TestRemoveContentHandlerNamesPostRemovalAsUnsupported(t *testing.T) { +func TestRemoveContentHandlerRejectsUnsupportedCollectionBeforeStoreAccess(t *testing.T) { service := moderation.NewService(nil, nil, moderation.Config{ InstanceDID: mutationInstanceDID, IdempotencyRetention: time.Hour, MaxLiveIdempotencyKeys: 1, }) - postBody := strings.Replace(removeBody, moderation.CommentCollection, moderation.PostV2Collection, 1) + unsupportedBody := strings.Replace(removeBody, moderation.CommentCollection, "app.bsky.feed.post", 1) response := httptest.NewRecorder() NewRemoveContentHandler(service).HandleRemoveContent(response, - mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", postBody, "application/json")) + mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", unsupportedBody, "application/json")) assertMutationError(t, response, http.StatusBadRequest, "InvalidSubject") - assert.Equal(t, "invalid subject: post removal is unsupported", mutationErrorMessage(t, response)) + assert.Equal(t, "invalid subject: unsupported subject collection", mutationErrorMessage(t, response)) } func TestMutationHandlersWriteRuleDetailButKeepUnavailableGeneric(t *testing.T) { diff --git a/internal/api/handlers/post/get_moderated_test.go b/internal/api/handlers/post/get_moderated_test.go new file mode 100644 index 0000000..ca2067b --- /dev/null +++ b/internal/api/handlers/post/get_moderated_test.go @@ -0,0 +1,69 @@ +package post + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "Coves/internal/core/posts" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHandleGetModeratedPostUnionEncoding(t *testing.T) { + uri := "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/social.coves.community.postv2/abc123" + handler := NewGetHandler(&mockGetPostService{getPostsFunc: func(_ context.Context, _ posts.GetPostsRequest) ([]*posts.PostResult, error) { + return []*posts.PostResult{{Moderated: &posts.ModeratedPost{ + URI: uri, AuthorDID: "did:plc:postauthor", + Community: &posts.CommunityRef{DID: "did:plc:community", Handle: "community.test", Name: "community"}, + Moderation: &posts.ModerationView{State: "removed", Sources: []posts.ModerationSourceView{{ + AuthorityDID: "did:web:instance.test", Scope: posts.ModerationScopeView{Kind: "instance"}, + }}}, + }}}, nil + }}, nil, nil) + recorder := httptest.NewRecorder() + handler.HandleGet(recorder, httptest.NewRequest(http.MethodGet, "/xrpc/social.coves.community.post.get?"+url.Values{"uris": {uri}}.Encode(), nil)) + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + var body struct { + Posts []map[string]any `json:"posts"` + } + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &body)) + require.Len(t, body.Posts, 1) + item := body.Posts[0] + assert.Equal(t, "social.coves.community.post.defs#moderatedPost", item["$type"]) + assert.Equal(t, uri, item["uri"]) + assert.Equal(t, "did:plc:postauthor", item["authorDid"]) + assert.Equal(t, map[string]any{ + "state": "removed", + "sources": []any{map[string]any{"authorityDid": "did:web:instance.test", "scope": map[string]any{"kind": "instance"}}}, + }, item["moderation"]) + assert.Equal(t, map[string]any{"did": "did:plc:community", "handle": "community.test", "name": "community"}, item["community"]) + for _, key := range []string{"record", "title", "embed", "cid", "content", "notFound", "removed", "blocked"} { + assert.NotContains(t, item, key, "a moderated tombstone cannot leak post content or claim another union member") + } +} + +func TestHandleGetExistingTombstoneUnionEncoding(t *testing.T) { + uri := "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/social.coves.community.post/abc123" + for _, scenario := range []struct { + name, expected string + result *posts.PostResult + }{ + {"not found", `{"uri":"` + uri + `","notFound":true}`, &posts.PostResult{NotFound: &posts.NotFoundPost{URI: uri, NotFound: true}}}, + {"community removed", `{"uri":"` + uri + `","removed":true,"code":"rule-violation"}`, &posts.PostResult{Removed: &posts.RemovedPost{URI: uri, Removed: true, Code: "rule-violation"}}}, + } { + t.Run(scenario.name, func(t *testing.T) { + handler := NewGetHandler(&mockGetPostService{getPostsFunc: func(_ context.Context, _ posts.GetPostsRequest) ([]*posts.PostResult, error) { + return []*posts.PostResult{scenario.result}, nil + }}, nil, nil) + recorder := httptest.NewRecorder() + handler.HandleGet(recorder, httptest.NewRequest(http.MethodGet, "/xrpc/social.coves.community.post.get?"+url.Values{"uris": {uri}}.Encode(), nil)) + require.Equal(t, http.StatusOK, recorder.Code) + assert.JSONEq(t, `{"posts":[`+scenario.expected+`]}`, recorder.Body.String()) + }) + } +} diff --git a/internal/api/handlers/post/getstatus_moderation_integration_test.go b/internal/api/handlers/post/getstatus_moderation_integration_test.go new file mode 100644 index 0000000..0b27ee5 --- /dev/null +++ b/internal/api/handlers/post/getstatus_moderation_integration_test.go @@ -0,0 +1,83 @@ +//go:build integration + +package post_test + +import ( + "database/sql" + "testing" + "time" + + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func admissionSnapshot(t *testing.T, db *sql.DB, communityDID, postURI string) string { + t.Helper() + var row string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT row_to_json(a)::text FROM community_post_admissions a + WHERE community_did = $1 AND post_uri = $2 + `, communityDID, postURI).Scan(&row)) + return row +} + +func TestGetStatus_InstancePostRemovalPreservesCommunityAcceptance(t *testing.T) { + db := testkit.DB(t) + stack := newStatusStack(db) + subject := newStatusSubject(t, db) // fixtures.Community sets hosted_by_did to this instance. + const postCID = "bafyreistatusseed" + rkey := testkit.TID() + acceptanceURI := "at://" + subject.CommunityDID + "/" + posts.AcceptanceCollection + "/" + rkey + _, err := stack.admissions.UpsertPending(t.Context(), posts.UpsertPendingCommand{ + CommunityDID: subject.CommunityDID, PostURI: subject.PostURI, EvaluatedCID: postCID, + }) + require.NoError(t, err) + result, err := stack.admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: subject.CommunityDID, PostURI: subject.PostURI, + AcceptanceURI: acceptanceURI, AcceptanceRkey: rkey, PinnedCID: postCID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, result.Outcome) + var hostingDID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT hosted_by_did FROM communities WHERE did = $1`, subject.CommunityDID).Scan(&hostingDID)) + require.Equal(t, fixtures.InstanceDID(), hostingDID) + before := admissionSnapshot(t, db, subject.CommunityDID, subject.PostURI) + checkAcceptance := func(t *testing.T) { + t.Helper() + assert.Equal(t, before, admissionSnapshot(t, db, subject.CommunityDID, subject.PostURI), + "instance moderation must not alter any admission column, including record URI, CID, watermarks and timestamps") + body := decodeStatus(t, getStatus(t, stack.handler, subject.PostURI, subject.CommunityDID)) + assert.Equal(t, "accepted", body["status"]) + assert.Equal(t, acceptanceURI, body["acceptanceUri"]) + assert.NotContains(t, body, "decisionCode", "instance removal is not a community removal") + } + checkAcceptance(t) + postRepository := postgres.NewPostRepository(db) + commentRepository := postgres.NewCommentRepository(db) + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepository, commentRepository), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + ref := moderation.StrongRef{URI: subject.PostURI, CID: postCID} + removed, err := moderationService.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "statusadmina")), moderation.RemoveContentRequest{ + Subject: ref, ExpectedVersion: "v0", IdempotencyKey: "remove-status-post", Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + checkAcceptance(t) + restored, err := moderationService.RestoreContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "statusadminb")), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &ref, ExpectedVersion: removed.State.Version, + IdempotencyKey: "restore-status-post", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + checkAcceptance(t) +} diff --git a/internal/api/routes/moderation_media_integration_test.go b/internal/api/routes/moderation_media_integration_test.go index c821c71..022da38 100644 --- a/internal/api/routes/moderation_media_integration_test.go +++ b/internal/api/routes/moderation_media_integration_test.go @@ -141,6 +141,9 @@ type moderationMediaHarness struct { postCID string ownerA string ownerB string + + // proxyService is the purger a consumer's media reconciler shares with the proxy. + proxyService *imageproxy.ImageProxyService } func newModerationMediaHarness(t *testing.T, blockFetch bool) (*moderationMediaHarness, *waitingMediaFetcher) { @@ -190,7 +193,7 @@ func newModerationMediaHarness(t *testing.T, blockFetch bool) (*moderationMediaH routes.RegisterImageProxyRoutes(router, imagehandler.NewHandler(proxyService, mediaPDSResolver{url: pdsServer.URL})) proxy := httptest.NewServer(router) t.Cleanup(proxy.Close) - return &moderationMediaHarness{db: db, cache: cache, cacheDir: cacheDir, proxy: proxy, pds: pds, + return &moderationMediaHarness{db: db, cache: cache, cacheDir: cacheDir, proxy: proxy, proxyService: proxyService, pds: pds, moderation: service, postURI: postURI, postCID: post.CID, ownerA: ownerA, ownerB: ownerB}, waiting } diff --git a/internal/api/routes/moderation_post_media_integration_test.go b/internal/api/routes/moderation_post_media_integration_test.go new file mode 100644 index 0000000..77d0749 --- /dev/null +++ b/internal/api/routes/moderation_post_media_integration_test.go @@ -0,0 +1,298 @@ +//go:build integration + +package routes_test + +import ( + "encoding/json" + "net/http" + "os" + "testing" + "time" + + "Coves/internal/atproto/jetstream" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/core/users" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + postMediaSpamReason = "social.coves.moderation.defs#reasonSpam" + postMediaIllegalReason = "social.coves.moderation.defs#reasonIllegalContent" + postMediaPreset = "content_preview" +) + +func (h *moderationMediaHarness) indexedImagePost(t *testing.T, collection, imageCID string) (moderation.StrongRef, string) { + t.Helper() + var communityDID string + require.NoError(t, h.db.QueryRowContext(t.Context(), `SELECT community_did FROM posts WHERE uri = $1`, h.postURI).Scan(&communityDID)) + rkey := testkit.TID() + authorDID := h.ownerA + blobOwnerDID := authorDID + uriAuthority := authorDID + if collection == moderation.LegacyPostCollection { + blobOwnerDID = communityDID + uriAuthority = communityDID + } + subject := moderation.StrongRef{ + URI: "at://" + uriAuthority + "/" + collection + "/" + rkey, + CID: mediaImageCID("post record " + rkey), + } + embed, err := json.Marshal(map[string]any{ + "$type": "social.coves.embed.images", + "images": []any{map[string]any{ + "alt": "shared post image", + "image": map[string]any{ + "$type": "blob", "ref": map[string]any{"$link": imageCID}, + "mimeType": "image/png", "size": 10, + }, + }}, + }) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, 'image post', 'body', $6::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, authorDID, communityDID, string(embed)) + require.NoError(t, err) + if collection == moderation.PostV2Collection { + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, NOW(), NOW()) + `, communityDID, subject.URI, subject.CID) + require.NoError(t, err) + } + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{blobOwnerDID, imageCID}] = true + h.pds.mu.Unlock() + indexed, err := postgres.NewPostRepository(h.db).GetRawIndexedRow(t.Context(), subject.URI) + require.NoError(t, err) + require.Equal(t, subject.CID, indexed.CID) + require.NotNil(t, indexed.Embed) + assert.Contains(t, *indexed.Embed, imageCID) + return subject, blobOwnerDID +} + +func TestModerationPostV2MediaRemovalColdAndWarm(t *testing.T) { + for _, warm := range []bool{false, true} { + name := "cold" + if warm { + name = "warm" + } + t.Run(name, func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + imageCID := mediaImageCID("postv2 removal " + name) + subject, ownerDID := h.indexedImagePost(t, moderation.PostV2Collection, imageCID) + require.Equal(t, h.ownerA, ownerDID, "postv2 image blobs belong to the author") + if warm { + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, ownerDID, imageCID)) + _, err := os.Stat(h.cachePath(preset, ownerDID, imageCID)) + require.NoError(t, err, "the image must be on disk before removal") + } + } + before := h.pds.count(ownerDID, imageCID) + if warm { + require.Equal(t, 2, before) + } else { + require.Zero(t, before, "a cold image has not been fetched") + } + removed := h.remove(t, subject, postMediaSpamReason) + h.assertNoCachedBlob(t, imageCID, ownerDID) + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusNotFound, h.request(t, preset, ownerDID, imageCID)) + } + assert.Equal(t, before, h.pds.count(ownerDID, imageCID), "blocked post image must not be fetched") + h.restore(t, subject, removed) + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, ownerDID, imageCID)) + assert.Equal(t, before+1, h.pds.count(ownerDID, imageCID), "restored postv2 image must come from a new PDS fetch") + }) + } +} + +func TestModerationLegacyPostSharedImageRemovalAndRestore(t *testing.T) { + for _, warm := range []bool{false, true} { + name := "cold" + if warm { + name = "warm" + } + t.Run(name, func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + imageCID := mediaImageCID("legacy shared post " + name) + first, ownerDID := h.indexedImagePost(t, moderation.LegacyPostCollection, imageCID) + second, secondOwner := h.indexedImagePost(t, moderation.LegacyPostCollection, imageCID) + require.Equal(t, ownerDID, secondOwner) + require.NotEqual(t, h.ownerA, ownerDID, "legacy post blobs belong to the community") + secondView, err := postgres.NewPostRepository(h.db).GetViewsByURIs(t.Context(), []string{second.URI}, "") + require.NoError(t, err) + require.NotNil(t, secondView[second.URI], "the unremoved legacy post must remain served") + if warm { + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, ownerDID, imageCID)) + _, err := os.Stat(h.cachePath(preset, ownerDID, imageCID)) + require.NoError(t, err) + } + } + before := h.pds.count(ownerDID, imageCID) + removed := h.remove(t, first, postMediaSpamReason) + h.assertNoCachedBlob(t, imageCID, ownerDID) + // PRD §9 Media: one community-owned blob shared by two legacy posts is blocked for both when either is removed. + secondView, err = postgres.NewPostRepository(h.db).GetViewsByURIs(t.Context(), []string{second.URI}, "") + require.NoError(t, err) + require.NotNil(t, secondView[second.URI], "the second post remains visible even though its shared image is blocked") + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusNotFound, h.request(t, preset, secondOwner, imageCID), "the shared image on L2 must be refused") + } + assert.Equal(t, before, h.pds.count(ownerDID, imageCID)) + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{h.ownerB, imageCID}] = true + h.pds.mu.Unlock() + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, h.ownerB, imageCID), "spam is scoped to the community blob owner") + assert.Equal(t, 1, h.pds.count(h.ownerB, imageCID)) + h.restore(t, first, removed) + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, ownerDID, imageCID)) + assert.Equal(t, before+1, h.pds.count(ownerDID, imageCID), "restore must re-fetch the purged community-owned image") + }) + } +} + +func TestModerationLegacyPostIllegalContentBlocksEveryOwner(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + imageCID := mediaImageCID("legacy global removal") + subject, communityDID := h.indexedImagePost(t, moderation.LegacyPostCollection, imageCID) + owners := []string{communityDID, h.ownerA, h.ownerB} + h.pds.mu.Lock() + for _, owner := range owners { + h.pds.known[mediaBlobKey{owner, imageCID}] = true + } + h.pds.mu.Unlock() + for _, owner := range owners { + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, owner, imageCID)) + _, err := os.Stat(h.cachePath(preset, owner, imageCID)) + require.NoError(t, err, "each owner's blob must be cached before global removal") + } + } + removed := h.remove(t, subject, postMediaIllegalReason) + h.assertNoCachedBlob(t, imageCID, "") + for _, owner := range owners { + before := h.pds.count(owner, imageCID) + require.Equal(t, http.StatusNotFound, h.request(t, postMediaPreset, owner, imageCID)) + assert.Equal(t, before, h.pds.count(owner, imageCID), "ownerless block must refuse every owner's blob without fetching") + } + h.restore(t, subject, removed) + for _, owner := range owners { + before := h.pds.count(owner, imageCID) + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, owner, imageCID)) + assert.Equal(t, before+1, h.pds.count(owner, imageCID), "restore must trigger a real PDS fetch for each purged owner") + } +} + +// postV2ImageEvent is a postv2 commit in ownerDID's repo whose record embeds imageCIDs. +func postV2ImageEvent(ownerDID, communityDID, operation, rkey, rev, recordCID string, timeUS int64, imageCIDs ...string) *jetstream.JetstreamEvent { + var record map[string]interface{} + if operation != "delete" { + images := make([]interface{}, 0, len(imageCIDs)) + for _, imageCID := range imageCIDs { + images = append(images, map[string]interface{}{"alt": "recreated image", "image": map[string]interface{}{ + "$type": "blob", "ref": map[string]interface{}{"$link": imageCID}, "mimeType": "image/png", "size": 10, + }}) + } + record = map[string]interface{}{ + "$type": jetstream.PostV2Collection, "community": communityDID, "title": "recreated post", + "content": "body", "createdAt": "2026-03-01T00:00:00Z", + "embed": map[string]interface{}{"$type": "social.coves.embed.images", "images": images}, + } + } + return &jetstream.JetstreamEvent{Kind: "commit", Did: ownerDID, TimeUS: timeUS, Commit: &jetstream.CommitEvent{ + Rev: rev, Operation: operation, Collection: jetstream.PostV2Collection, RKey: rkey, CID: recordCID, Record: record, + }} +} + +// PRD Definition of done: deleting and recreating a removed postv2 keeps it +// removed and blocks the new blob, which the author's repo serves even though +// the AppView keeps the tombstone. +func TestModerationPostV2RecreatedImageBlockedColdAndWarm(t *testing.T) { + for _, warm := range []bool{false, true} { + name := "cold" + if warm { + name = "warm" + } + t.Run(name, func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + var communityDID string + require.NoError(t, h.db.QueryRowContext(t.Context(), `SELECT community_did FROM posts WHERE uri = $1`, h.postURI).Scan(&communityDID)) + admissions := postgres.NewAdmissionRepository(h.db) + consumer := jetstream.NewPostEventConsumer( + postgres.NewPostRepository(h.db), postgres.NewCommunityRepository(h.db, credentialciphertest.Fixed()), + users.NewUserService(postgres.NewUserRepository(h.db), nil, testkit.Endpoints().PDS.BaseURL, nil, ""), h.db, + jetstream.WithAdmissions(admissions), + jetstream.WithPostMediaReconciler(moderation.NewMediaReconciler( + postgres.NewModerationRepository(h.db), fixtures.InstanceDID(), h.proxyService)), + ) + originalCID, recreatedCID := mediaImageCID("recreate original "+name), mediaImageCID("recreate new "+name) + rkey := testkit.TID() + revs := []string{testkit.TID(), testkit.TID(), testkit.TID()} + createdAt := time.Now().Add(-time.Minute).UnixMicro() + subject := moderation.StrongRef{URI: "at://" + h.ownerA + "/" + jetstream.PostV2Collection + "/" + rkey, CID: mediaImageCID("recreate record " + name)} + require.NoError(t, consumer.HandleEvent(t.Context(), + postV2ImageEvent(h.ownerA, communityDID, "create", rkey, revs[0], subject.CID, createdAt, originalCID))) + acceptanceRkey := testkit.TID() + accepted, err := admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: communityDID, PostURI: subject.URI, + AcceptanceURI: "at://" + communityDID + "/" + posts.AcceptanceCollection + "/" + acceptanceRkey, + AcceptanceRkey: acceptanceRkey, PinnedCID: subject.CID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, accepted.Outcome) + removed := h.remove(t, subject, postMediaSpamReason) + require.NoError(t, consumer.HandleEvent(t.Context(), + postV2ImageEvent(h.ownerA, communityDID, "delete", rkey, revs[1], "", createdAt+1_000_000))) + + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{h.ownerA, recreatedCID}] = true + h.pds.mu.Unlock() + if warm { + // The new blob is fetchable from the author's repo before the + // recreate reaches the AppView, so it can already be cached. + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, h.ownerA, recreatedCID)) + _, err := os.Stat(h.cachePath(preset, h.ownerA, recreatedCID)) + require.NoError(t, err, "the new image must be on disk before the recreate") + } + } + before := h.pds.count(h.ownerA, recreatedCID) + if warm { + require.Equal(t, 2, before) + } else { + require.Zero(t, before, "a cold image has not been fetched") + } + + require.NoError(t, consumer.HandleEvent(t.Context(), + postV2ImageEvent(h.ownerA, communityDID, "create", rkey, revs[2], subject.CID, createdAt+2_000_000, originalCID, recreatedCID))) + indexed, err := postgres.NewPostRepository(h.db).GetRawIndexedRow(t.Context(), subject.URI) + require.NoError(t, err) + require.NotNil(t, indexed.DeletedAt, "the recreate must not resurrect the tombstone") + state, err := h.moderation.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + + h.assertNoCachedBlob(t, recreatedCID, h.ownerA) + for _, preset := range []string{postMediaPreset, "content_full"} { + for range 2 { + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, recreatedCID)) + } + } + assert.Equal(t, before, h.pds.count(h.ownerA, recreatedCID), "the blocked recreated image must not be fetched") + }) + } +} diff --git a/internal/api/routes/moderation_post_removal_integration_test.go b/internal/api/routes/moderation_post_removal_integration_test.go new file mode 100644 index 0000000..c57b3eb --- /dev/null +++ b/internal/api/routes/moderation_post_removal_integration_test.go @@ -0,0 +1,258 @@ +//go:build integration + +package routes_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + commentsAPI "Coves/internal/api/handlers/comments" + "Coves/internal/api/middleware" + "Coves/internal/api/routes" + "Coves/internal/core/comments" + "Coves/internal/core/communities" + "Coves/internal/core/communityFeeds" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/internal/validation" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/require" +) + +func moderationPostAcceptanceNoContentKeys(t *testing.T, value any) { + t.Helper() + switch value := value.(type) { + case map[string]any: + for key, child := range value { + require.NotContains(t, []string{"record", "title", "embed"}, key, "content key in moderated post: %s", key) + moderationPostAcceptanceNoContentKeys(t, child) + } + case []any: + for _, child := range value { + moderationPostAcceptanceNoContentKeys(t, child) + } + } +} + +func moderationPostAcceptanceFeedURIs(t *testing.T, body map[string]any) []string { + t.Helper() + var uris []string + for _, entry := range moderationAcceptanceArray(t, body["feed"]) { + post := moderationAcceptanceObject(t, moderationAcceptanceObject(t, entry)["post"]) + uri, ok := post["uri"].(string) + require.True(t, ok) + uris = append(uris, uri) + } + return uris +} + +func TestModerationPostRemovalAcceptance(t *testing.T) { + db := testkit.DB(t) + postRepo := postgres.NewPostRepository(db) + commentRepo := postgres.NewCommentRepository(db) + userRepo := postgres.NewUserRepository(db) + communityRepo := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()) + admissionRepo := postgres.NewAdmissionRepository(db) + instanceDID := fixtures.InstanceDID() + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepo, commentRepo), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: instanceDID, IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + pdsURL := testkit.Endpoints().PDS.BaseURL + communityService := communities.NewCommunityServiceWithPDSFactory( + communityRepo, pdsURL, instanceDID, "", nil, nil, nil, + communities.PrivateHostOptions(true)..., + ) + postService := posts.NewPostService(postRepo, communityService, nil, nil, nil, nil, pdsURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(admissionRepo, nil), + ) + commentService := comments.NewCommentService(commentRepo, userRepo, postRepo, communityRepo, nil, nil, nil) + feedService := communityFeeds.NewCommunityFeedService( + postgres.NewCommunityFeedRepository(db, "moderation-acceptance-cursor-secret"), communityService, + ) + + authorName := testkit.UniqueIDWithPrefix(t, "postauthor") + const authorDID = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "postcommunity") + const communityDID = "did:plc:cccccccccccccccccccccccc" + const ownerDID = "did:plc:dddddddddddddddddddddddd" + fixtures.User(t, db, "owner"+communityName+".test", ownerDID) + _, err := db.ExecContext(t.Context(), ` + INSERT INTO communities (did, name, owner_did, created_by_did, hosted_by_did, handle, pds_url, created_at) + VALUES ($1, $2, $3, $3, $4, $5, $6, NOW()) + `, communityDID, communityName, ownerDID, instanceDID, communityName+".coves.social", pdsURL) + require.NoError(t, err) + const postTitle = "original post removal acceptance title" + const postBody = "original post removal acceptance body text" + const postCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + insertAcceptedPost := func(title, content string) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + rkey + _, err := db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, NOW()) + `, uri, postCID, rkey, authorDID, communityDID, title, content) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, last_community_rev, last_community_op_rank, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, '3lqqqqqqqqqq2', 1, NOW(), NOW()) + `, communityDID, uri, postCID) + require.NoError(t, err) + return uri + } + postURI := insertAcceptedPost(postTitle, postBody) + controlURI := insertAcceptedPost("unremoved acceptance control", "control body") + commentURI := moderationAcceptanceInsertComment(t, db, authorDID, postURI, postCID, postURI, postCID, + "bafyreipostacceptancecomment", "comment under the accepted post") + + adminADID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "postadmina")) + adminBDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "postadminb")) + const adminAToken = "post-removal-admin-a-session" + const adminBToken = "post-removal-admin-b-session" + const authorToken = "post-removal-author-session" + unsealer := fixtures.NewSessionUnsealer() + oauthStore := fixtures.NewOAuthStore() + for _, session := range []struct{ token, did, id string }{ + {adminAToken, adminADID, "post-admin-a"}, + {adminBToken, adminBDID, "post-admin-b"}, + {authorToken, authorDID, "post-author"}, + } { + unsealer.AddSession(session.token, session.did, session.id) + oauthStore.AddSession(session.did, session.id, "test-access-token") + } + adminAuth := middleware.NewInstanceAdminMiddleware(unsealer, oauthStore, nil, + moderation.NewAllowlistAuthority([]string{adminADID, adminBDID})) + optionalAuth := middleware.NewOAuthAuthMiddleware(unsealer, oauthStore) + mux := chi.NewRouter() + routes.RegisterModerationRoutes(mux, moderationService, adminAuth) + routes.RegisterPostRoutes(mux, postService, nil, nil, optionalAuth, optionalAuth) + routes.RegisterCommunityFeedRoutes(mux, feedService, nil, nil, optionalAuth) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.community.comment.getComments", + commentsAPI.NewGetCommentsHandler(commentsAPI.NewServiceAdapter(commentService), nil).HandleGetComments) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + client := server.Client() + postURL := server.URL + "/xrpc/social.coves.community.post.get?" + url.Values{"uris": {postURI}}.Encode() + threadURL := server.URL + "/xrpc/social.coves.community.comment.getComments?" + url.Values{"post": {postURI}, "sort": {"new"}}.Encode() + feedURL := server.URL + "/xrpc/social.coves.communityFeed.getCommunity?" + url.Values{"community": {communityDID}, "sort": {"new"}}.Encode() + + initialPost := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, postURL, "", nil)) + initialPosts := moderationAcceptanceArray(t, initialPost["posts"]) + require.Len(t, initialPosts, 1) + require.Equal(t, postTitle, moderationAcceptanceObject(t, moderationAcceptanceObject(t, initialPosts[0])["record"])["title"]) + initialThread := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil)) + initialComments := moderationAcceptanceArray(t, initialThread["comments"]) + require.Len(t, initialComments, 1) + initialComment := moderationAcceptanceObject(t, moderationAcceptanceObject(t, initialComments[0])["comment"]) + require.Equal(t, commentURI, initialComment["uri"]) + initialFeed := moderationPostAcceptanceFeedURIs(t, moderationAcceptanceBody(t, + moderationAcceptanceRequest(t, client, http.MethodGet, feedURL, "", nil))) + require.ElementsMatch(t, []string{postURI, controlURI}, initialFeed) + + state := moderationAcceptanceObject(t, moderationAcceptanceBody(t, + requestSubjectState(t, client, server.URL, postURI, adminAToken))["state"]) + require.Equal(t, postCID, moderationAcceptanceObject(t, state["currentSubject"])["cid"]) + version, ok := state["version"].(string) + require.True(t, ok) + removeResponse := moderationAcceptanceRequest(t, client, http.MethodPost, + server.URL+"/xrpc/social.coves.moderation.removeContent", adminAToken, map[string]any{ + "subject": map[string]string{"uri": postURI, "cid": postCID}, "expectedVersion": version, + "idempotencyKey": "remove-post", "reason": "social.coves.moderation.defs#reasonSpam", + }) + removed := moderationAcceptanceBody(t, removeResponse) + catalog := lexicon.NewBaseCatalog() + require.NoError(t, catalog.LoadDirectory("../../atproto/lexicon")) + mutationData, err := atdata.UnmarshalJSON(removeResponse.body) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, mutationData, "social.coves.moderation.defs#mutationResult", 0)) + require.Equal(t, "applied", removed["outcome"]) + removalAction := moderationAcceptanceObject(t, moderationAcceptanceObject(t, removed["action"])["action"]) + removalID, ok := moderationAcceptanceObject(t, removalAction["ref"])["actionId"].(string) + require.True(t, ok) + require.NotEmpty(t, removalID) + removedVersion, ok := moderationAcceptanceObject(t, removed["state"])["version"].(string) + require.True(t, ok) + + for _, viewer := range []struct{ name, token string }{{"anonymous", ""}, {"author", authorToken}} { + t.Run(viewer.name, func(t *testing.T) { + response := moderationAcceptanceRequest(t, client, http.MethodGet, postURL, viewer.token, nil) + body := moderationAcceptanceBody(t, response) + items := moderationAcceptanceArray(t, body["posts"]) + require.Len(t, items, 1) + item := moderationAcceptanceObject(t, items[0]) + require.Equal(t, "social.coves.community.post.defs#moderatedPost", item["$type"]) + require.Equal(t, postURI, item["uri"]) + view := moderationAcceptanceObject(t, item["moderation"]) + require.Equal(t, "removed", view["state"]) + sources := moderationAcceptanceArray(t, view["sources"]) + require.Len(t, sources, 1) + source := moderationAcceptanceObject(t, sources[0]) + require.Equal(t, instanceDID, source["authorityDid"]) + require.Equal(t, "instance", moderationAcceptanceObject(t, source["scope"])["kind"]) + moderationPostAcceptanceNoContentKeys(t, item) + require.NotContains(t, string(response.body), postTitle) + require.NotContains(t, string(response.body), postBody) + itemJSON, err := json.Marshal(item) + require.NoError(t, err) + data, err := atdata.UnmarshalJSON(itemJSON) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, data, "social.coves.community.post.defs#moderatedPost", 0)) + }) + } + + missingRootURI := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + testkit.TID() + missingThreadURL := server.URL + "/xrpc/social.coves.community.comment.getComments?" + url.Values{"post": {missingRootURI}, "sort": {"new"}}.Encode() + missingRoot := moderationAcceptanceRequest(t, client, http.MethodGet, missingThreadURL, "", nil) + removedRoot := moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil) + requireXRPCError(t, missingRoot, http.StatusNotFound, "RootNotFound") + requireXRPCError(t, removedRoot, http.StatusNotFound, "RootNotFound") + require.JSONEq(t, string(missingRoot.body), string(removedRoot.body), "a removed root must have the same error code and message as a never-indexed root") + removedFeed := moderationPostAcceptanceFeedURIs(t, moderationAcceptanceBody(t, + moderationAcceptanceRequest(t, client, http.MethodGet, feedURL, "", nil))) + require.NotContains(t, removedFeed, postURI) + require.Contains(t, removedFeed, controlURI) + + restoreResponse := moderationAcceptanceRequest(t, client, http.MethodPost, + server.URL+"/xrpc/social.coves.moderation.restoreContent", adminBToken, map[string]any{ + "actionId": removalID, "reviewedSubject": map[string]string{"uri": postURI, "cid": postCID}, + "expectedVersion": removedVersion, "idempotencyKey": "restore-post", + "reason": "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + restored := moderationAcceptanceBody(t, restoreResponse) + restoreData, err := atdata.UnmarshalJSON(restoreResponse.body) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, restoreData, "social.coves.moderation.defs#mutationResult", 0)) + require.Equal(t, "applied", restored["outcome"]) + restoredPost := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, postURL, "", nil)) + restoredItems := moderationAcceptanceArray(t, restoredPost["posts"]) + require.Len(t, restoredItems, 1) + restoredItem := moderationAcceptanceObject(t, restoredItems[0]) + require.Equal(t, postURI, restoredItem["uri"]) + require.Equal(t, postTitle, moderationAcceptanceObject(t, restoredItem["record"])["title"]) + require.NotContains(t, restoredItem, "moderation") + restoredThread := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil)) + restoredComments := moderationAcceptanceArray(t, restoredThread["comments"]) + require.Len(t, restoredComments, 1) + restoredComment := moderationAcceptanceObject(t, moderationAcceptanceObject(t, restoredComments[0])["comment"]) + require.Equal(t, commentURI, restoredComment["uri"]) + restoredFeed := moderationPostAcceptanceFeedURIs(t, moderationAcceptanceBody(t, + moderationAcceptanceRequest(t, client, http.MethodGet, feedURL, "", nil))) + require.ElementsMatch(t, []string{postURI, controlURI}, restoredFeed) + +} diff --git a/internal/api/routes/moderation_post_served_media_integration_test.go b/internal/api/routes/moderation_post_served_media_integration_test.go new file mode 100644 index 0000000..f87484e --- /dev/null +++ b/internal/api/routes/moderation_post_served_media_integration_test.go @@ -0,0 +1,145 @@ +//go:build integration + +package routes_test + +import ( + "encoding/json" + "net/http" + "net/url" + "strings" + "testing" + + "Coves/internal/core/blobs" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/db/postgres" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// servedProxyBlobs returns the (owner DID, CID) pair of every image-proxy URL +// under proxyBaseURL anywhere in value. +func servedProxyBlobs(t *testing.T, proxyBaseURL string, value any) []mediaBlobKey { + t.Helper() + var found []mediaBlobKey + switch typed := value.(type) { + case string: + if !strings.HasPrefix(typed, proxyBaseURL+"/img/") { + return nil + } + parsed, err := url.Parse(typed) + require.NoError(t, err) + // /img/{preset}/plain/{did}/{cid} + segments := strings.Split(strings.TrimPrefix(parsed.Path, "/"), "/") + require.Len(t, segments, 5, "unexpected proxy URL shape: %s", typed) + found = append(found, mediaBlobKey{segments[3], segments[4]}) + case map[string]any: + for _, field := range typed { + found = append(found, servedProxyBlobs(t, proxyBaseURL, field)...) + } + case []any: + for _, entry := range typed { + found = append(found, servedProxyBlobs(t, proxyBaseURL, entry)...) + } + } + return found +} + +// PRD §9 Media: a removal blocks the blobs PostBlobCIDs derives, so the served +// view must carry no proxy URL outside them. An author who hand-writes a proxy +// URL for their own blob into the record gets no image in the view; the same +// image referenced as a blob is served and is blocked by an illegal-content +// removal. +func TestModerationPostServedMediaIsBlockedByRemoval(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + blobs.ResetImageURLConfigForTesting() + blobs.SetImageURLConfig(blobs.ImageURLConfig{ProxyEnabled: true, ProxyBaseURL: h.proxy.URL}) + t.Cleanup(blobs.ResetImageURLConfigForTesting) + var communityDID string + require.NoError(t, h.db.QueryRowContext(t.Context(), `SELECT community_did FROM posts WHERE uri = $1`, h.postURI).Scan(&communityDID)) + repository := postgres.NewPostRepository(h.db) + + blobRef := func(imageCID string) map[string]any { + return map[string]any{"$type": "blob", "ref": map[string]any{"$link": imageCID}, "mimeType": "image/png", "size": 10} + } + authorURL := func(preset, imageCID string) string { + return h.proxy.URL + "/img/" + preset + "/plain/" + h.ownerA + "/" + imageCID + } + for _, test := range []struct { + name string + embed func(imageCID string) map[string]any + wantServed bool + }{ + {name: "external thumb as a URL string", embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.external", "external": map[string]any{ + "uri": "https://example.com/article", "thumb": authorURL("embed_thumbnail", imageCID), + }} + }}, + {name: "stored images#view", embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.images#view", "images": []any{map[string]any{ + "thumb": authorURL(postMediaPreset, imageCID), "fullsize": authorURL("content_full", imageCID), "alt": "forged", + }}} + }}, + {name: "stored external#view", embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.external#view", "external": map[string]any{ + "uri": "https://example.com/article", "thumb": authorURL("embed_thumbnail", imageCID), + }} + }}, + {name: "external thumb as a blob", wantServed: true, embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.external", "external": map[string]any{ + "uri": "https://example.com/article", "thumb": blobRef(imageCID), + }} + }}, + } { + t.Run(test.name, func(t *testing.T) { + imageCID := mediaImageCID("served media " + test.name) + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{h.ownerA, imageCID}] = true + h.pds.mu.Unlock() + rkey := testkit.TID() + subject := moderation.StrongRef{ + URI: "at://" + h.ownerA + "/" + moderation.PostV2Collection + "/" + rkey, + CID: mediaImageCID("served media record " + rkey), + } + embed, err := json.Marshal(test.embed(imageCID)) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, 'served media post', 'body', $6::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, h.ownerA, communityDID, string(embed)) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, NOW(), NOW()) + `, communityDID, subject.URI, subject.CID) + require.NoError(t, err) + + views, err := repository.GetViewsByURIs(t.Context(), []string{subject.URI}, "") + require.NoError(t, err) + view := views[subject.URI] + require.NotNil(t, view, "the post must be served before removal") + posts.TransformBlobRefsToURLs(view) + encoded, err := json.Marshal(view.Embed) + require.NoError(t, err) + var servedEmbed any + require.NoError(t, json.Unmarshal(encoded, &servedEmbed)) + served := servedProxyBlobs(t, h.proxy.URL, servedEmbed) + if test.wantServed { + require.Equal(t, []mediaBlobKey{{h.ownerA, imageCID}}, served) + } else { + require.Empty(t, served, "a proxy URL the author wrote as a string must not be served: %s", encoded) + } + + h.remove(t, subject, postMediaIllegalReason) + for _, blob := range served { + blocked, err := h.proxyService.IsBlobBlocked(t.Context(), blob.did, blob.cid) + require.NoError(t, err) + assert.True(t, blocked, "served blob %s/%s must be blocked after removal", blob.did, blob.cid) + assert.Equal(t, http.StatusNotFound, h.request(t, postMediaPreset, blob.did, blob.cid)) + } + }) + } +} diff --git a/internal/api/routes/moderation_quote_projection_integration_test.go b/internal/api/routes/moderation_quote_projection_integration_test.go new file mode 100644 index 0000000..0ad48a4 --- /dev/null +++ b/internal/api/routes/moderation_quote_projection_integration_test.go @@ -0,0 +1,245 @@ +//go:build integration + +package routes_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + actorAPI "Coves/internal/api/handlers/actor" + commentsAPI "Coves/internal/api/handlers/comments" + "Coves/internal/api/middleware" + "Coves/internal/api/routes" + "Coves/internal/core/blueskypost" + "Coves/internal/core/comments" + "Coves/internal/core/communities" + "Coves/internal/core/communityFeeds" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Coves quotes must never invoke Bluesky resolution, but the serving handlers +// must still execute TransformPostEmbeds with a configured resolver. +type rejectingCovesQuoteResolver struct{} + +func (rejectingCovesQuoteResolver) ResolvePost(context.Context, string) (*blueskypost.BlueskyPostResult, error) { + panic("Coves quotes must not trigger Bluesky resolution") +} + +func (rejectingCovesQuoteResolver) ParseBlueskyURL(context.Context, string) (string, error) { + panic("Coves quote reads must not parse Bluesky URLs") +} + +func (rejectingCovesQuoteResolver) IsBlueskyURL(string) bool { + panic("Coves quote reads must not classify Bluesky URLs") +} + +func TestModerationCovesQuoteProjectionAcrossReadPaths(t *testing.T) { + for _, storedType := range []string{"social.coves.embed.post", "social.coves.embed.post#view"} { + t.Run(storedType, func(t *testing.T) { + db := testkit.DB(t) + postRepo := postgres.NewPostRepository(db) + commentRepo := postgres.NewCommentRepository(db) + userRepo := postgres.NewUserRepository(db) + communityRepo := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()) + instanceDID := fixtures.InstanceDID() + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepo, commentRepo), postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: instanceDID, IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + pdsURL := testkit.Endpoints().PDS.BaseURL + communityService := communities.NewCommunityServiceWithPDSFactory( + communityRepo, pdsURL, instanceDID, "", nil, nil, nil, + communities.PrivateHostOptions(true)..., + ) + postService := posts.NewPostService(postRepo, communityService, nil, nil, nil, nil, pdsURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(postgres.NewAdmissionRepository(db), nil), + ) + commentService := comments.NewCommentService(commentRepo, userRepo, postRepo, communityRepo, nil, nil, nil) + feedService := communityFeeds.NewCommunityFeedService( + postgres.NewCommunityFeedRepository(db, "moderation-quote-cursor-secret"), communityService, + ) + + authorName := testkit.UniqueIDWithPrefix(t, "quoteauthor") + const authorDID = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "quotecommunity") + const communityDID = "did:plc:cccccccccccccccccccccccc" + const ownerDID = "did:plc:dddddddddddddddddddddddd" + fixtures.User(t, db, "owner"+communityName+".test", ownerDID) + _, err := db.ExecContext(t.Context(), ` + INSERT INTO communities (did, name, owner_did, created_by_did, hosted_by_did, handle, pds_url, created_at) + VALUES ($1, $2, $3, $3, $4, $5, $6, NOW()) + `, communityDID, communityName, ownerDID, instanceDID, communityName+".coves.social", pdsURL) + require.NoError(t, err) + const postCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + insertPost := func(title string, embed any) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + rkey + encoded, err := json.Marshal(embed) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, $6, 'quote fixture body', $7, NOW()) + `, uri, postCID, rkey, authorDID, communityDID, title, encoded) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, last_community_rev, last_community_op_rank, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, '3lqqqqqqqqqq2', 1, NOW(), NOW()) + `, communityDID, uri, postCID) + require.NoError(t, err) + return uri + } + quotedURI := insertPost("quoted post private title", nil) + cleanQuotedURI := insertPost("clean quoted post", nil) + leakImage := map[string]any{"$type": "social.coves.embed.images#view", "images": []any{map[string]any{"fullsize": "leaked-image", "alt": "quoted image"}}} + quoterEmbed := map[string]any{ + "$type": storedType, "post": map[string]any{"uri": quotedURI, "cid": postCID}, + "resolved": map[string]any{"title": "quoted post private title", "text": "quoted private body", "embed": leakImage}, + "title": "leak", "text": "leak", "images": []any{leakImage}, + } + quoterURI := insertPost("quoter searchable marker", quoterEmbed) + storedQuoterEmbed, err := json.Marshal(quoterEmbed) + require.NoError(t, err) + cleanQuoterURI := insertPost("clean quoter", map[string]any{ + "$type": "social.coves.embed.post", "post": map[string]any{"uri": cleanQuotedURI, "cid": postCID}, + }) + + adminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "quoteadmin")) + const adminToken = "quote-projection-admin-session" + unsealer := fixtures.NewSessionUnsealer() + oauthStore := fixtures.NewOAuthStore() + unsealer.AddSession(adminToken, adminDID, "quote-admin") + oauthStore.AddSession(adminDID, "quote-admin", "test-access-token") + optionalAuth := middleware.NewOAuthAuthMiddleware(unsealer, oauthStore) + adminAuth := middleware.NewInstanceAdminMiddleware(unsealer, oauthStore, nil, moderation.NewAllowlistAuthority([]string{adminDID})) + quoteResolver := rejectingCovesQuoteResolver{} + mux := chi.NewRouter() + routes.RegisterModerationRoutes(mux, moderationService, adminAuth) + routes.RegisterPostRoutes(mux, postService, nil, quoteResolver, optionalAuth, optionalAuth) + routes.RegisterCommunityFeedRoutes(mux, feedService, nil, quoteResolver, optionalAuth) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.actor.getPosts", + actorAPI.NewGetPostsHandler(postService, nil, nil, quoteResolver).HandleGetPosts) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.community.comment.getComments", + commentsAPI.NewGetCommentsHandler(commentsAPI.NewServiceAdapter(commentService), nil).HandleGetComments) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + client := server.Client() + request := func(path string) map[string]any { + t.Helper() + return moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, server.URL+path, "", nil)) + } + postPath := func(uri string) string { + return "/xrpc/social.coves.community.post.get?" + url.Values{"uris": {uri}}.Encode() + } + postItem := func(uri string) map[string]any { + t.Helper() + items := moderationAcceptanceArray(t, request(postPath(uri))["posts"]) + require.Len(t, items, 1) + return moderationAcceptanceObject(t, items[0]) + } + // A clean Coves quote is currently served as social.coves.embed.post, + // without server-side resolution. R4 pins that same strongRef-only type + // even for stored #view embeds claiming pre-resolved data. + quoteType := "social.coves.embed.post" + assertQuote := func(t *testing.T, post map[string]any, quoted string) { + t.Helper() + require.Equal(t, quoterURI, post["uri"]) + require.NotContains(t, post, "$type", "a normal postView carries no union discriminator") + _, hasRecord := post["record"] + require.True(t, hasRecord, "the quoter itself must be served as a normal post view") + embed := moderationAcceptanceObject(t, post["embed"]) + assert.Equal(t, map[string]any{ + "$type": quoteType, "post": map[string]any{"uri": quoted, "cid": postCID}, + }, embed, "the served embed must contain only the quoted strongRef, with no stored preview or media") + // Decision (orchestrator, from the DoD "P's embed holds Q's + // strongRef only"): record is the quoter's own authored record, + // served verbatim per the lexicon, so record.embed keeps every + // byte the quoter wrote, including the preview fields it forged. + // None of it is AppView-held content of the quoted post: the + // server adds resolved only at serve time, and only for Bluesky + // URIs. The projection applies to the top-level embed alone. + record := moderationAcceptanceObject(t, post["record"]) + recordEmbed, err := json.Marshal(record["embed"]) + require.NoError(t, err) + assert.JSONEq(t, string(storedQuoterEmbed), string(recordEmbed), "record.embed must be the stored embed verbatim") + } + t.Run("clean-control", func(t *testing.T) { + clean := postItem(cleanQuoterURI) + require.Equal(t, cleanQuoterURI, clean["uri"]) + assert.Equal(t, map[string]any{ + "$type": quoteType, "post": map[string]any{"uri": cleanQuotedURI, "cid": postCID}, + }, moderationAcceptanceObject(t, clean["embed"]), "unremoved clean Coves quotes have the same strongRef-only projection") + }) + + state := moderationAcceptanceObject(t, moderationAcceptanceBody(t, + requestSubjectState(t, client, server.URL, quotedURI, adminToken))["state"]) + version, ok := state["version"].(string) + require.True(t, ok) + removed := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodPost, + server.URL+"/xrpc/social.coves.moderation.removeContent", adminToken, map[string]any{ + "subject": map[string]string{"uri": quotedURI, "cid": postCID}, "expectedVersion": version, + "idempotencyKey": "remove-quoted-post", "reason": "social.coves.moderation.defs#reasonSpam", + })) + require.Equal(t, "applied", removed["outcome"]) + require.Equal(t, "social.coves.community.post.defs#moderatedPost", postItem(quotedURI)["$type"]) + + readPaths := []struct { + name string + path string + selectPost func(*testing.T, map[string]any) map[string]any + }{ + {"post.get", postPath(quoterURI), func(t *testing.T, body map[string]any) map[string]any { + items := moderationAcceptanceArray(t, body["posts"]) + require.Len(t, items, 1) + return moderationAcceptanceObject(t, items[0]) + }}, + {"community feed", "/xrpc/social.coves.communityFeed.getCommunity?" + url.Values{"community": {communityDID}, "sort": {"new"}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationQuoteFeedPost(t, body, quoterURI) + }}, + {"actor.getPosts", "/xrpc/social.coves.actor.getPosts?" + url.Values{"actor": {authorDID}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationQuoteFeedPost(t, body, quoterURI) + }}, + {"searchPosts", "/xrpc/social.coves.feed.searchPosts?" + url.Values{"q": {"quoter searchable marker"}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationQuoteFeedPost(t, body, quoterURI) + }}, + {"getComments header", "/xrpc/social.coves.community.comment.getComments?" + url.Values{"post": {quoterURI}, "sort": {"new"}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationAcceptanceObject(t, body["post"]) + }}, + } + for _, readPath := range readPaths { + t.Run(readPath.name, func(t *testing.T) { + assertQuote(t, readPath.selectPost(t, request(readPath.path)), quotedURI) + }) + } + }) + } +} + +func moderationQuoteFeedPost(t *testing.T, body map[string]any, uri string) map[string]any { + t.Helper() + for _, item := range moderationAcceptanceArray(t, body["feed"]) { + post := moderationAcceptanceObject(t, moderationAcceptanceObject(t, item)["post"]) + if post["uri"] == uri { + return post + } + } + require.FailNow(t, "quoter missing from read path", "uri: %s; body: %#v", uri, body) + return nil +} diff --git a/internal/atproto/jetstream/authorpost.go b/internal/atproto/jetstream/authorpost.go index e5d20ad..ee85c2d 100644 --- a/internal/atproto/jetstream/authorpost.go +++ b/internal/atproto/jetstream/authorpost.go @@ -766,7 +766,7 @@ func (c *PostEventConsumer) upsertAuthorPost(ctx context.Context, authorDID stri facets: facetsJSON, embed: embedJSON, labels: labelsJSON, bridgedUpvotes: up, bridgedDownvotes: down, bridgedAsOf: asOf, storedAsOf: stored.bridgedAsOf, storedDeletedAt: stored.deletedAt, - storedIndexedAt: stored.indexedAt, timeUS: timeUS, + storedIndexedAt: stored.indexedAt, timeUS: timeUS, authorDID: authorDID, }) if err != nil { return err diff --git a/internal/atproto/jetstream/comment_consumer.go b/internal/atproto/jetstream/comment_consumer.go index 1174514..c4c035f 100644 --- a/internal/atproto/jetstream/comment_consumer.go +++ b/internal/atproto/jetstream/comment_consumer.go @@ -39,39 +39,45 @@ type CommentEventConsumer struct { // bridgeTrust gates whether a comment's user repo may assert bridgedStats. // nil means default-deny (bridgedStats are ignored for every comment). bridgeTrust *BridgeTrust - mediaReconciler CommentMediaReconciler + mediaReconciler MediaReconciler } // CommentEventConsumerOption configures optional CommentEventConsumer behaviour. type CommentEventConsumerOption func(*CommentEventConsumer) -// CommentMediaReconciler blocks images introduced on a removed comment. -type CommentMediaReconciler interface { +// MediaReconciler blocks images introduced on removed comments or posts. +type MediaReconciler interface { ReconcileTx(ctx context.Context, tx *sql.Tx, subjectURI string) ([]moderation.MediaBlock, error) Purge(blocks []moderation.MediaBlock) } // WithCommentMediaReconciler reconciles media blocks when a removed comment is rewritten. -func WithCommentMediaReconciler(reconciler CommentMediaReconciler) CommentEventConsumerOption { +func WithCommentMediaReconciler(reconciler MediaReconciler) CommentEventConsumerOption { return func(c *CommentEventConsumer) { c.mediaReconciler = reconciler } } // commitCommentWrite reconciles the indexed embed within the write transaction; // cached bytes are purged only after both the comment and its blocks commit. func (c *CommentEventConsumer) commitCommentWrite(ctx context.Context, tx *sql.Tx, uri string) error { + return commitMediaWrite(ctx, tx, uri, c.mediaReconciler, "comment") +} + +// commitMediaWrite reconciles newly indexed media before committing the content +// and its blocks together, then purges cached copies only after a successful commit. +func commitMediaWrite(ctx context.Context, tx *sql.Tx, uri string, reconciler MediaReconciler, kind string) error { var blocks []moderation.MediaBlock - if c.mediaReconciler != nil { + if reconciler != nil { var err error - blocks, err = c.mediaReconciler.ReconcileTx(ctx, tx, uri) + blocks, err = reconciler.ReconcileTx(ctx, tx, uri) if err != nil { - return fmt.Errorf("reconcile comment media: %w", err) + return fmt.Errorf("reconcile %s media: %w", kind, err) } } if err := tx.Commit(); err != nil { return err } - if c.mediaReconciler != nil { - c.mediaReconciler.Purge(blocks) + if reconciler != nil { + reconciler.Purge(blocks) } return nil } diff --git a/internal/atproto/jetstream/post_consumer.go b/internal/atproto/jetstream/post_consumer.go index 75e79e4..4c680e9 100644 --- a/internal/atproto/jetstream/post_consumer.go +++ b/internal/atproto/jetstream/post_consumer.go @@ -4,6 +4,8 @@ import ( "Coves/internal/atproto/identity" "Coves/internal/core/bridgedvotes" "Coves/internal/core/communities" + "Coves/internal/core/embeds" + "Coves/internal/core/moderation" "Coves/internal/core/posts" "Coves/internal/core/richtext" "Coves/internal/core/users" @@ -25,6 +27,9 @@ type PostEventConsumer struct { communityRepo communities.Repository userService users.UserService db *sql.DB // Direct DB access for atomic count reconciliation + + // nil keeps ingestion independent of moderation media reconciliation. + mediaReconciler PostMediaReconciler // bridgeTrust gates whether a post's author repo may assert bridgedStats. // nil means default-deny (bridgedStats are ignored for every post). bridgeTrust *BridgeTrust @@ -262,6 +267,10 @@ type postContentUpdate struct { storedDeletedAt *time.Time storedIndexedAt time.Time timeUS int64 + + // authorDID owns the incoming blobs, which are blocked when the post is + // removed even if the update is skipped. + authorDID string } // applyPostContentUpdate runs the rev gate and the atomic content UPDATE. @@ -273,8 +282,23 @@ type postContentUpdate struct { // as an error would dead-letter healthy events. func (c *PostEventConsumer) applyPostContentUpdate(ctx context.Context, in postContentUpdate) (bool, error) { // Skip soft-deleted rows: a deleted post should not be resurrected by an edit. + // The author's repo still serves the incoming blobs, so a removed post's + // recreated images are blocked even though the content is not indexed. The + // rev gate runs first (read-only: this path never advances the rev) so an + // out-of-order event that predates the delete blocks nothing. if in.storedDeletedAt != nil { + stale, err := recordRevIsStale(ctx, c.db, in.uri, in.rev) + if err != nil { + return false, fmt.Errorf("failed to check rev of soft-deleted post update: %w", err) + } + if stale { + logSkippedStaleRev(ConsumerPosts, "update", in.uri, in.rev) + return false, nil + } log.Printf("Update event for soft-deleted post: %s (skipping)", in.uri) + if err := c.blockIncomingMedia(ctx, in.uri, in.authorDID, in.embed); err != nil { + return false, fmt.Errorf("failed to block media of skipped post update: %w", err) + } return false, nil } @@ -402,7 +426,7 @@ func (c *PostEventConsumer) applyPostContentUpdate(ctx context.Context, in postC return false, nil } - if err := tx.Commit(); err != nil { + if err := commitMediaWrite(ctx, tx, in.uri, c.mediaReconciler, "post"); err != nil { return false, fmt.Errorf("failed to commit post update transaction: %w", err) } @@ -520,7 +544,9 @@ func (c *PostEventConsumer) indexPostIfRevWins(ctx context.Context, post *posts. // (comments implement the in-place re-create because their resurrection // machinery already exists; see comment_consumer.go). log.Printf("Post already indexed: %s (idempotent)", post.URI) - if commitErr := tx.Commit(); commitErr != nil { + // The dropped content's blobs are still served from the author's repo, + // so a removed post blocks them from the incoming embed. + if commitErr := c.commitIncomingMediaWrite(ctx, tx, post.URI, post.AuthorDID, incomingPostBlobCIDs(embedJSON)); commitErr != nil { return false, fmt.Errorf("failed to commit transaction: %w", commitErr) } // Reported as NOT applied: no content was written, so a caller that @@ -562,7 +588,7 @@ func (c *PostEventConsumer) indexPostIfRevWins(ctx context.Context, post *posts. } // Commit transaction - if err := tx.Commit(); err != nil { + if err := commitMediaWrite(ctx, tx, post.URI, c.mediaReconciler, "post"); err != nil { return false, fmt.Errorf("failed to commit transaction: %w", err) } @@ -656,3 +682,67 @@ func parseRecordCreatedAt(raw, uri string) time.Time { } return createdAt } + +// PostMediaReconciler also blocks the blobs of incoming post content that the +// consumer does not index, because the stored row cannot name them. +type PostMediaReconciler interface { + MediaReconciler + ReconcileIncomingTx(ctx context.Context, tx *sql.Tx, subjectURI, ownerDID string, blobCIDs []string) ([]moderation.MediaBlock, error) +} + +// WithPostMediaReconciler reconciles media blocks when a removed post is created or edited. +func WithPostMediaReconciler(reconciler PostMediaReconciler) PostEventConsumerOption { + return func(c *PostEventConsumer) { c.mediaReconciler = reconciler } +} + +// incomingPostBlobCIDs returns the proxy-served blob CIDs of a serialized +// incoming embed. A malformed embed has no served blobs to block. +func incomingPostBlobCIDs(embed sql.NullString) []string { + if !embed.Valid { + return nil + } + var decoded map[string]interface{} + if err := json.Unmarshal([]byte(embed.String), &decoded); err != nil { + return nil + } + return embeds.PostBlobCIDs(decoded) +} + +// commitIncomingMediaWrite blocks the incoming blobs of a removed post inside +// tx, commits, and purges cached copies only after the blocks commit. +func (c *PostEventConsumer) commitIncomingMediaWrite(ctx context.Context, tx *sql.Tx, uri, ownerDID string, blobCIDs []string) error { + var blocks []moderation.MediaBlock + if c.mediaReconciler != nil { + var err error + blocks, err = c.mediaReconciler.ReconcileIncomingTx(ctx, tx, uri, ownerDID, blobCIDs) + if err != nil { + return fmt.Errorf("reconcile incoming post media: %w", err) + } + } + if err := tx.Commit(); err != nil { + return err + } + if c.mediaReconciler != nil { + c.mediaReconciler.Purge(blocks) + } + return nil +} + +// blockIncomingMedia blocks the blobs of an incoming post event that writes no +// post row. It is a no-op unless the post has an active removal. +func (c *PostEventConsumer) blockIncomingMedia(ctx context.Context, uri, ownerDID string, embed sql.NullString) error { + blobCIDs := incomingPostBlobCIDs(embed) + if c.mediaReconciler == nil || len(blobCIDs) == 0 { + return nil + } + tx, err := c.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin media block transaction: %w", err) + } + defer func() { + if rollbackErr := tx.Rollback(); rollbackErr != nil && rollbackErr != sql.ErrTxDone { + log.Printf("Failed to rollback transaction: %v", rollbackErr) + } + }() + return c.commitIncomingMediaWrite(ctx, tx, uri, ownerDID, blobCIDs) +} diff --git a/internal/atproto/jetstream/post_moderation_consumer_test.go b/internal/atproto/jetstream/post_moderation_consumer_test.go new file mode 100644 index 0000000..7e05c02 --- /dev/null +++ b/internal/atproto/jetstream/post_moderation_consumer_test.go @@ -0,0 +1,366 @@ +//go:build integration + +package jetstream + +import ( + "context" + "database/sql" + "encoding/json" + "testing" + "time" + + "Coves/internal/core/communityFeeds" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + postModerationCIDOne = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" + postModerationCIDTwo = "bafkreicy44vctf2bgqnn5wwzdern7bc2khwi7ku2r66bozl4x6bsrvuj2q" + postModerationRecordCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" +) + +type postModerationPurgeCall struct { + ownerDID, blobCID string + committed bool +} + +type postModerationPurger struct { + db *sql.DB + calls []postModerationPurgeCall +} + +func (p *postModerationPurger) purge(ownerDID, blobCID string) error { + call := postModerationPurgeCall{ownerDID: ownerDID, blobCID: blobCID} + err := p.db.QueryRowContext(context.Background(), ` + SELECT EXISTS (SELECT 1 FROM moderation_media_blocks + WHERE owner_did IS NOT DISTINCT FROM NULLIF($1, '') + AND blob_cid = $2 AND active) + `, ownerDID, blobCID).Scan(&call.committed) + p.calls = append(p.calls, call) + return err +} + +func (p *postModerationPurger) PurgeOwnerBlob(ownerDID, blobCID string) error { + return p.purge(ownerDID, blobCID) +} + +func (p *postModerationPurger) PurgeBlob(blobCID string) error { + return p.purge("", blobCID) +} + +func postModerationRecord(imageCIDs ...string) map[string]interface{} { + record := pv2Record(pv2Community, "shared image post", "same authored body") + images := make([]interface{}, 0, len(imageCIDs)) + for _, imageCID := range imageCIDs { + images = append(images, map[string]interface{}{ + "alt": "indexed image", + "image": map[string]interface{}{ + "$type": "blob", "ref": map[string]interface{}{"$link": imageCID}, + "mimeType": "image/png", "size": 10, + }, + }) + } + record["embed"] = map[string]interface{}{"$type": "social.coves.embed.images", "images": images} + return record +} + +type postModerationConsumerFixture struct { + pv2Fixture + postRepository posts.Repository + postService posts.Service + moderator moderation.Service + purger *postModerationPurger + uri, rkey string + createdAt int64 + revs []string + create *JetstreamEvent +} + +func newPostModerationConsumerFixture(t *testing.T) postModerationConsumerFixture { + t.Helper() + db := testkit.DB(t) + f := newPV2Fixture(t, db) + postRepository := postgres.NewPostRepository(db) + moderationRepository := postgres.NewModerationRepository(db) + purger := &postModerationPurger{db: db} + f.consumer = NewPostEventConsumer( + postRepository, postgres.NewCommunityRepository(db, credentialciphertest.Fixed()), f.users, db, + WithAdmissions(f.admissions), WithDeletedAccounts(postgres.NewDeletedAccountRepository(db)), + WithPostMediaReconciler(moderation.NewMediaReconciler(moderationRepository, fixtures.InstanceDID(), purger)), + ) + moderator := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepository, postgres.NewCommentRepository(db)), + moderationRepository, + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + postService := posts.NewPostService(postRepository, nil, nil, nil, nil, nil, testkit.Endpoints().PDS.BaseURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(f.admissions, nil)) + rkey := testkit.TID() + uri := pv2URI(pv2Author, rkey) + revs := increasingTIDs(t, 4) + createdAt := time.Now().Add(-time.Minute).UnixMicro() + create := pv2Event(pv2Author, "create", rkey, revs[0], postModerationRecordCID, createdAt, + postModerationRecord(postModerationCIDOne)) + require.NoError(t, f.consumer.HandleEvent(t.Context(), create)) + indexed, err := postRepository.GetRawIndexedRow(t.Context(), uri) + require.NoError(t, err) + require.Equal(t, postModerationRecordCID, indexed.CID) + acceptanceRkey := testkit.TID() + accepted, err := f.admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: pv2Community, PostURI: uri, + AcceptanceURI: "at://" + pv2Community + "/" + posts.AcceptanceCollection + "/" + acceptanceRkey, + AcceptanceRkey: acceptanceRkey, PinnedCID: postModerationRecordCID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, accepted.Outcome) + return postModerationConsumerFixture{ + pv2Fixture: f, postRepository: postRepository, postService: postService, + moderator: moderator, purger: purger, uri: uri, rkey: rkey, + createdAt: createdAt, revs: revs, create: create, + } +} + +func (f postModerationConsumerFixture) remove(t *testing.T, reason string) *moderation.MutationResult { + t.Helper() + removed, err := f.moderator.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "postmediaadmin")), moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: f.uri, CID: postModerationRecordCID}, + ExpectedVersion: "v0", IdempotencyKey: "remove-post-media", Reason: reason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + require.NotNil(t, removed.Action) + var originalBlocks int + require.NoError(t, f.db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDOne).Scan(&originalBlocks)) + require.Equal(t, 1, originalBlocks, "the original image must be blocked before testing edit reconciliation") + assert.Empty(t, f.purger.calls, "removal service has no purger; calls must come from the consumer") + return removed +} + +func (f postModerationConsumerFixture) assertReadPaths(t *testing.T, viewerDID string, wantModerated, wantNotFound bool) { + t.Helper() + results, err := f.postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{f.uri}, ViewerDID: viewerDID}) + require.NoError(t, err) + require.Len(t, results, 1) + if wantModerated { + require.NotNil(t, results[0].Moderated, "a present post under an active removal must be a moderatedPost") + assert.Nil(t, results[0].Post) + } + if wantNotFound { + require.NotNil(t, results[0].NotFound, "a deleted post, or one this viewer could not see before the removal, must be notFound") + assert.Nil(t, results[0].Moderated) + } + feed, _, err := postgres.NewCommunityFeedRepository(f.db, "post-moderation-consumer-cursor").GetCommunityFeed(t.Context(), + communityFeeds.GetCommunityFeedRequest{Community: pv2Community, Sort: "new", Timeframe: "all", Limit: 10}) + require.NoError(t, err) + for _, item := range feed { + assert.NotEqual(t, f.uri, item.Post.URI, "removed or deleted post must not appear in the community feed") + } + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) +} + +func TestModerationPostConsumerReplayAndDuplicateKeepRemoval(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, "social.coves.moderation.defs#reasonSpam") + for _, delivery := range []string{"replay", "duplicate"} { + t.Run(delivery, func(t *testing.T) { + require.NoError(t, f.consumer.HandleEvent(t.Context(), f.create)) + f.assertReadPaths(t, "", true, false) + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + assert.Empty(t, f.purger.calls) + }) + } +} + +func TestModerationPostConsumerEditReconcilesNewImages(t *testing.T) { + for _, scenario := range []struct { + name, reason string + ownerless bool + }{ + {name: "spam owner block", reason: "social.coves.moderation.defs#reasonSpam"}, + {name: "illegal content ownerless block", reason: "social.coves.moderation.defs#reasonIllegalContent", ownerless: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, scenario.reason) + update := pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)) + require.NoError(t, f.consumer.HandleEvent(t.Context(), update)) + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, postModerationCIDOne, indexed.CID, "the edited record must be indexed before media reconciliation") + require.NotNil(t, indexed.Embed) + assert.Contains(t, *indexed.Embed, postModerationCIDTwo, "the new image must be present in the indexed embed") + // The edit moves the admission to pending_reacceptance, which only the + // author could see before the removal, so the tombstone is the + // author's; everyone else gets notFound (#moderatedPost never widens + // access). + f.assertReadPaths(t, pv2Author, true, false) + f.assertReadPaths(t, "", false, true) + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + require.NotNil(t, state.CurrentSubject) + assert.Equal(t, postModerationCIDOne, state.CurrentSubject.CID) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + var ownedBlocks int + require.NoError(t, f.db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDTwo).Scan(&ownedBlocks)) + assert.Equal(t, 1, ownedBlocks, "the edited image needs an active author-owned block on the original action") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{ownerDID: pv2Author, blobCID: postModerationCIDTwo, committed: true}, + "the consumer must purge the author-owned image after the block commits") + if scenario.ownerless { + var ownerlessBlocks int + require.NoError(t, f.db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did IS NULL AND blob_cid = $2 AND active + `, removed.Action.ID, postModerationCIDTwo).Scan(&ownerlessBlocks)) + assert.Equal(t, 1, ownerlessBlocks, "illegal content must also block the edited image for every owner") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{blobCID: postModerationCIDTwo, committed: true}, + "the ownerless image cache must be purged after commit") + } + }) + } +} + +func TestModerationPostConsumerDeleteThenCreateRemainsNotFound(t *testing.T) { + for _, scenario := range []struct { + name, reason string + ownerless bool + }{ + {name: "spam owner block", reason: "social.coves.moderation.defs#reasonSpam"}, + {name: "illegal content ownerless block", reason: "social.coves.moderation.defs#reasonIllegalContent", ownerless: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, scenario.reason) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "delete", f.rkey, f.revs[1], "", f.createdAt+1_000_000, nil))) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "create", f.rkey, f.revs[2], postModerationCIDOne, + f.createdAt+2_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)))) + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + require.NotNil(t, indexed.DeletedAt, "a newer create must not resurrect a soft-deleted postv2") + f.assertReadPaths(t, "", false, true) + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, moderation.RecordStateDeleted, state.RecordState) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + // The recreated record's new image is served from the author's repo + // even though the AppView keeps the tombstone, so it must be blocked. + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDTwo), "the recreated image needs an active author-owned block on the original action") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{ownerDID: pv2Author, blobCID: postModerationCIDTwo, committed: true}, + "the consumer must purge the recreated image after the block commits") + if scenario.ownerless { + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did IS NULL AND blob_cid = $2 AND active + `, removed.Action.ID, postModerationCIDTwo), "illegal content must also block the recreated image for every owner") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{blobCID: postModerationCIDTwo, committed: true}, + "the ownerless image cache must be purged after commit") + } + }) + } +} + +// The create path's ON CONFLICT branch discards incoming content when the row +// already exists (a concurrent insert, or the documented active-row re-create). +// The incoming images are still served from the author's repo, so a removed +// post must block them there too. +func TestModerationPostConsumerAlreadyIndexedCreateBlocksIncomingImages(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, "social.coves.moderation.defs#reasonIllegalContent") + embed, err := json.Marshal(postModerationRecord(postModerationCIDTwo)["embed"]) + require.NoError(t, err) + embedJSON := string(embed) + applied, err := f.consumer.indexPostIfRevWins(t.Context(), &posts.Post{ + URI: f.uri, CID: postModerationCIDOne, RKey: f.rkey, AuthorDID: pv2Author, CommunityDID: pv2Community, + Embed: &embedJSON, CreatedAt: time.Now(), IndexedAt: time.Now(), + }, f.revs[1]) + require.NoError(t, err) + assert.False(t, applied, "an existing row must keep its content") + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, postModerationRecordCID, indexed.CID) + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDTwo), "the discarded create's image needs an author-owned block") + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did IS NULL AND blob_cid = $2 AND active + `, removed.Action.ID, postModerationCIDTwo), "illegal content must block the discarded create's image for every owner") + assert.ElementsMatch(t, []postModerationPurgeCall{ + {ownerDID: pv2Author, blobCID: postModerationCIDTwo, committed: true}, + {blobCID: postModerationCIDTwo, committed: true}, + }, f.purger.calls) +} + +func TestModerationPostConsumerRemovalIsPerURI(t *testing.T) { + f := newPostModerationConsumerFixture(t) + f.remove(t, "social.coves.moderation.defs#reasonSpam") + otherRkey := testkit.TID() + otherURI := pv2URI(pv2Author, otherRkey) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "create", otherRkey, f.revs[1], + postModerationRecordCID, f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne)))) + acceptanceRkey := testkit.TID() + accepted, err := f.admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: pv2Community, PostURI: otherURI, + AcceptanceURI: "at://" + pv2Community + "/" + posts.AcceptanceCollection + "/" + acceptanceRkey, + AcceptanceRkey: acceptanceRkey, PinnedCID: postModerationRecordCID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, accepted.Outcome) + results, err := f.postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{f.uri, otherURI}}) + require.NoError(t, err) + require.Len(t, results, 2) + assert.NotNil(t, results[0].Moderated) + require.NotNil(t, results[1].Post, "a distinct accepted URI must remain a postView") + assert.Equal(t, otherURI, results[1].Post.URI) +} + +func TestModerationPostConsumerUnremovedEditCreatesNoBlocks(t *testing.T) { + f := newPostModerationConsumerFixture(t) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)))) + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, postModerationCIDOne, indexed.CID) + assert.Equal(t, 0, countRows(t, f.db, `SELECT count(*) FROM moderation_media_blocks WHERE blob_cid = $1`, postModerationCIDTwo)) + assert.Empty(t, f.purger.calls) +} + +// An out-of-order update that predates the delete is stale by rev; the rev gate +// must reject it before any media is blocked, or an older record's images would +// be blocked for a post whose newest state is the tombstone. +func TestModerationPostConsumerStaleUpdateAfterDeleteAddsNoBlocks(t *testing.T) { + f := newPostModerationConsumerFixture(t) + f.remove(t, "social.coves.moderation.defs#reasonIllegalContent") + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "delete", f.rkey, f.revs[2], "", f.createdAt+2_000_000, nil))) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)))) + assert.Equal(t, 0, countRows(t, f.db, `SELECT count(*) FROM moderation_media_blocks WHERE blob_cid = $1`, postModerationCIDTwo), + "a stale update must not block its images") + assert.Empty(t, f.purger.calls) +} diff --git a/internal/core/comments/comment_servable_embed_test.go b/internal/core/comments/comment_servable_embed_test.go new file mode 100644 index 0000000..45bbc64 --- /dev/null +++ b/internal/core/comments/comment_servable_embed_test.go @@ -0,0 +1,84 @@ +package comments + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "Coves/internal/core/blobs" + "Coves/internal/core/users" +) + +// A comment's served embed carries only the proxy URLs HydrateCommentView +// derives from blobs, the same CIDs CommentImageCIDs blocks. URL strings the +// author wrote into the record are not served in the view; the record keeps +// them verbatim. +func TestBuildCommentView_ServesOnlyBlobDerivedMedia(t *testing.T) { + blobs.ResetImageURLConfigForTesting() + blobs.SetImageURLConfig(blobs.ImageURLConfig{ProxyEnabled: true, ProxyBaseURL: "https://img.coves.social"}) + t.Cleanup(blobs.ResetImageURLConfigForTesting) + + const ( + commenterDID = "did:plc:commenter123" + authorCID = "bafyreigj3fwnwjuzr35k2kuzmb5dixxczrzjhqkr5srlqplsh6gq3bj3si" + quotedURI = "at://did:plc:quoted/social.coves.community.postv2/3kquoted" + ) + authorURL := "https://img.coves.social/img/content_preview/plain/" + commenterDID + "/" + authorCID + postURI := "at://did:plc:post123/social.coves.community.postv2/test" + + for _, test := range []struct { + name string + embed map[string]interface{} + want interface{} + }{ + { + name: "stored images#view is not served", + embed: map[string]interface{}{"$type": "social.coves.embed.images#view", "images": []interface{}{ + map[string]interface{}{"thumb": authorURL, "fullsize": authorURL, "alt": "forged"}, + }}, + }, + { + name: "images record entry with view URLs and no blob keeps only its alt", + embed: map[string]interface{}{"$type": "social.coves.embed.images", "images": []interface{}{ + map[string]interface{}{"thumb": authorURL, "fullsize": authorURL, "alt": "forged"}, + }}, + want: map[string]interface{}{"$type": "social.coves.embed.images", "images": []interface{}{ + map[string]interface{}{"alt": "forged"}, + }}, + }, + { + name: "quote view with a stored resolved keeps only its strongRef", + embed: map[string]interface{}{ + "$type": "social.coves.embed.post#view", "post": map[string]interface{}{"uri": quotedURI, "cid": "bafyquoted"}, + "resolved": map[string]interface{}{"images": []interface{}{map[string]interface{}{"thumb": authorURL}}}, + }, + want: map[string]interface{}{ + "$type": "social.coves.embed.post", "post": map[string]interface{}{"uri": quotedURI, "cid": "bafyquoted"}, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + encoded, err := json.Marshal(test.embed) + require.NoError(t, err) + embedJSON := string(encoded) + comment := createTestComment("at://"+commenterDID+"/social.coves.community.comment/1", commenterDID, "commenter.test", postURI, postURI, 0) + comment.Embed = &embedJSON + service := NewCommentService(newMockCommentRepo(), newMockUserRepo(), newMockPostRepo(), newMockCommunityRepo(), nil, nil, nil).(*commentService) + + view := service.buildCommentView(comment, nil, nil, map[string]*users.User{ + commenterDID: {DID: commenterDID, Handle: "commenter.test", PDSURL: "https://pds.example.com"}, + }) + + if test.want == nil { + assert.Nil(t, view.Embed) + } else { + assert.Equal(t, test.want, view.Embed) + } + record, ok := view.Record.(*CommentRecord) + require.True(t, ok) + assert.Equal(t, test.embed, record.Embed, "the comment record keeps the stored embed verbatim") + }) + } +} diff --git a/internal/core/comments/comment_service.go b/internal/core/comments/comment_service.go index f2947d4..28cba69 100644 --- a/internal/core/comments/comment_service.go +++ b/internal/core/comments/comment_service.go @@ -1,12 +1,6 @@ package comments import ( - "Coves/internal/core/blobs" - "Coves/internal/core/communities" - "Coves/internal/core/embeds" - "Coves/internal/core/posts" - "Coves/internal/core/richtext" - "Coves/internal/core/users" "context" "encoding/json" "errors" @@ -15,6 +9,13 @@ import ( "strings" "time" + "Coves/internal/core/blobs" + "Coves/internal/core/communities" + "Coves/internal/core/embeds" + "Coves/internal/core/posts" + "Coves/internal/core/richtext" + "Coves/internal/core/users" + "github.com/bluesky-social/indigo/atproto/auth/oauth" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/rivo/uniseg" @@ -633,13 +634,17 @@ func (s *commentService) buildCommentView( // union than posts, and the firehose applies no embed validation, so a // federated comment carrying a post-only embed type must not be stamped // with a #view type the comment union does not declare. + // + // ServableEmbed runs first so the view serves only proxy URLs derived from + // blobs, the CIDs CommentImageCIDs blocks; the record keeps the stored + // embed verbatim. var embed interface{} if comment.Embed != nil && *comment.Embed != "" { - var embedMap map[string]interface{} - if err := json.Unmarshal([]byte(*comment.Embed), &embedMap); err != nil { + var storedEmbed map[string]interface{} + if err := json.Unmarshal([]byte(*comment.Embed), &storedEmbed); err != nil { // Log error but don't fail request - embed is optional slog.Warn("failed to unmarshal embed for comment", "comment_uri", comment.URI, "error", err) - } else { + } else if embedMap, servable := embeds.ServableEmbed(storedEmbed).(map[string]interface{}); servable { var authorPDSURL string if user, found := usersByDID[comment.CommenterDID]; found && user != nil { authorPDSURL = user.PDSURL diff --git a/internal/core/comments/comment_service_test.go b/internal/core/comments/comment_service_test.go index 210d6bf..c62838f 100644 --- a/internal/core/comments/comment_service_test.go +++ b/internal/core/comments/comment_service_test.go @@ -300,6 +300,14 @@ func newMockPostRepo() *mockPostRepo { } } +func (m *mockPostRepo) ActiveRemovalsByURIs(context.Context, []string) (map[string][]posts.RemovalSource, error) { + return map[string][]posts.RemovalSource{}, nil +} + +func (m *mockPostRepo) AdmittedURIsForViewer(context.Context, []string, string) (map[string]bool, error) { + return map[string]bool{}, nil +} + // hideFromHeader makes the visibility predicate refuse this post, as it does for // any non-accepted admission state when the viewer is not the author. func (m *mockPostRepo) hideFromHeader(uri string) { @@ -2014,7 +2022,7 @@ func TestBuildCommentView_ValidEmbedDeserialization(t *testing.T) { communityRepo := newMockCommunityRepo() postURI := "at://did:plc:post123/app.bsky.feed.post/test" - embedJSON := `{"$type":"app.bsky.embed.images","images":[{"alt":"test","image":{"$type":"blob","ref":"bafytest"}}]}` + embedJSON := `{"$type":"social.coves.embed.images","images":[{"alt":"test","image":{"$type":"blob","ref":"bafytest"}}]}` comment := createTestComment("at://did:plc:commenter123/comment/1", "did:plc:commenter123", "commenter.test", postURI, postURI, 0) comment.Embed = &embedJSON @@ -2026,7 +2034,7 @@ func TestBuildCommentView_ValidEmbedDeserialization(t *testing.T) { assert.NotNil(t, result.Embed) embedMap, ok := result.Embed.(map[string]interface{}) assert.True(t, ok) - assert.Equal(t, "app.bsky.embed.images", embedMap["$type"]) + assert.Equal(t, "social.coves.embed.images", embedMap["$type"]) } func TestBuildCommentRecord_ValidLabelsDeserialization(t *testing.T) { @@ -2109,7 +2117,7 @@ func TestBuildCommentView_EmptyStringVsNilHandling(t *testing.T) { { name: "Valid JSON strings", facetsValue: strPtr(`[]`), - embedValue: strPtr(`{}`), + embedValue: strPtr(`{"$type":"social.coves.embed.post","post":{"uri":"at://did:plc:post123/social.coves.community.postv2/test","cid":"bafypost"}}`), labelsValue: strPtr(`{"$type":"com.atproto.label.defs#selfLabels","values":[]}`), expectFacetsNil: false, expectEmbedNil: false, diff --git a/internal/core/comments/post_removal_read_paths_integration_test.go b/internal/core/comments/post_removal_read_paths_integration_test.go new file mode 100644 index 0000000..0ff173c --- /dev/null +++ b/internal/core/comments/post_removal_read_paths_integration_test.go @@ -0,0 +1,194 @@ +//go:build integration + +package comments_test + +import ( + "database/sql" + "testing" + "time" + + "Coves/internal/core/comments" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func postRemovalComment(t *testing.T, db *sql.DB, commenterDID, postURI, postCID, content string, createdAt time.Time) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + commenterDID + "/" + moderation.CommentCollection + "/" + rkey + _, err := db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $5, $6, $7, $8) + `, uri, moderatedCommentCID, rkey, commenterDID, postURI, postCID, content, createdAt) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), `UPDATE posts SET comment_count = comment_count + 1 WHERE uri = $1`, postURI) + require.NoError(t, err) + return uri +} + +func postRemovalActorComments(t *testing.T, service comments.Service, commenterDID, communityDID string, limit int, cursor *string) *comments.GetActorCommentsResponse { + t.Helper() + response, err := service.GetActorComments(t.Context(), &comments.GetActorCommentsRequest{ + ActorDID: commenterDID, Community: communityDID, Limit: limit, Cursor: cursor, + }) + require.NoError(t, err) + return response +} + +func postRemovalFixture(t *testing.T) (*sql.DB, comments.Service, moderation.Service, moderation.StrongRef, string, string, string, string, string) { + t.Helper() + db, commentService, moderationService, postURI, _, authorDID := moderationThreadFixture(t) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + communityDID := post.CommunityDID + visibleURI := fixtures.Post(t, db, communityDID, authorDID, "unremoved companion thread", 0, time.Now()) + visiblePost, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), visibleURI) + require.NoError(t, err) + commenterName := testkit.UniqueIDWithPrefix(t, "rootcommenter") + commenterDID := fixtures.DID(commenterName) + fixtures.User(t, db, commenterName+".test", commenterDID) + first := postRemovalComment(t, db, commenterDID, postURI, post.CID, "first original comment", time.Now().Add(-time.Minute)) + visible := postRemovalComment(t, db, commenterDID, visibleURI, visiblePost.CID, "visible companion comment", time.Now().Add(-2*time.Minute)) + second := postRemovalComment(t, db, commenterDID, postURI, post.CID, "second original comment", time.Now().Add(-3*time.Minute)) + return db, commentService, moderationService, moderation.StrongRef{URI: postURI, CID: post.CID}, communityDID, commenterDID, first, visible, second +} + +func removeFixturePost(t *testing.T, service moderation.Service, subject moderation.StrongRef) *moderation.MutationResult { + t.Helper() + removed, err := service.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "rootadmina")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-root", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + return removed +} + +func TestPostRemovalGetCommentsHidesRootForEveryViewer(t *testing.T) { + db, commentService, moderationService, subject, _, _, _, _, _ := postRemovalFixture(t) + baseline, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{PostURI: subject.URI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + require.Len(t, baseline.Comments, 3) + var authorDID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT author_did FROM posts WHERE uri = $1`, subject.URI).Scan(&authorDID)) + removeFixturePost(t, moderationService, subject) + for _, viewer := range []struct { + name string + did *string + }{ + {name: "anonymous"}, + {name: "post author", did: &authorDID}, + } { + t.Run(viewer.name, func(t *testing.T) { + missingURI := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + testkit.TID() + _, missingErr := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{ + PostURI: missingURI, ViewerDID: viewer.did, Sort: "new", Depth: 1, Limit: 10, + }) + require.ErrorIs(t, missingErr, comments.ErrRootNotFound) + _, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{ + PostURI: subject.URI, ViewerDID: viewer.did, Sort: "new", Depth: 1, Limit: 10, + }) + require.ErrorIs(t, err, comments.ErrRootNotFound) + }) + } +} + +func TestPostRemovalActorCommentsFiltersBeforePagination(t *testing.T) { + _, commentService, moderationService, subject, communityDID, commenterDID, first, visible, second := postRemovalFixture(t) + baseline := postRemovalActorComments(t, commentService, commenterDID, "", 10, nil) + require.Len(t, baseline.Comments, 3) + require.Equal(t, []string{first, visible, second}, []string{baseline.Comments[0].URI, baseline.Comments[1].URI, baseline.Comments[2].URI}) + removeFixturePost(t, moderationService, subject) + for _, filter := range []struct{ name, community string }{{"all communities", ""}, {"community filtered", communityDID}} { + t.Run(filter.name, func(t *testing.T) { + whole := postRemovalActorComments(t, commentService, commenterDID, filter.community, 10, nil) + require.Len(t, whole.Comments, 1, "removed-root comments must be excluded before the page is selected") + assert.Equal(t, visible, whole.Comments[0].URI) + assert.Nil(t, whole.Cursor) + page := postRemovalActorComments(t, commentService, commenterDID, filter.community, 1, nil) + require.Len(t, page.Comments, 1, "page one must advance past the newer removed-root comment") + assert.Equal(t, visible, page.Comments[0].URI) + assert.Nil(t, page.Cursor, "the older removed-root comment must not leak a continuation cursor") + }) + } +} + +func TestPostRemovalRestoreServesIndexedThreadAndActorComments(t *testing.T) { + db, commentService, moderationService, subject, _, commenterDID, first, visible, second := postRemovalFixture(t) + before, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{PostURI: subject.URI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + baselinePost, ok := before.Post.(*posts.PostView) + require.True(t, ok) + require.NotNil(t, baselinePost.Stats) + baselineCommentCount := baselinePost.Stats.CommentCount + removed := removeFixturePost(t, moderationService, subject) + during := postRemovalComment(t, db, commenterDID, subject.URI, subject.CID, "indexed while root removed", time.Now()) + voterDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "rootvoter")) + voteRkey := testkit.TID() + _, err = db.ExecContext(t.Context(), ` + INSERT INTO votes (uri, cid, rkey, voter_did, subject_uri, subject_cid, direction, created_at) + VALUES ($1, $2, $3, $4, $5, $6, 'up', NOW()) + `, "at://"+voterDID+"/social.coves.interaction.vote/"+voteRkey, moderatedCommentCID, voteRkey, voterDID, first, moderatedCommentCID) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), `UPDATE comments SET upvote_count = upvote_count + 1, score = score + 1 WHERE uri = $1`, first) + require.NoError(t, err) + restored, err := moderationService.RestoreContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "rootadminb")), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: removed.State.Version, + IdempotencyKey: "restore-root", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + thread, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{PostURI: subject.URI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + restoredPost, ok := thread.Post.(*posts.PostView) + require.True(t, ok) + require.NotNil(t, restoredPost.Stats) + assert.Equal(t, baselineCommentCount+1, restoredPost.Stats.CommentCount, + "a comment indexed during removal must contribute to the restored post's count") + for _, uri := range []string{first, second, during} { + view := moderationThreadComment(t, thread, uri).Comment + require.NotNil(t, view.Record) + if uri == first { + assert.Equal(t, 1, view.Stats.Upvotes, "vote indexed during removal must be served") + } + } + assert.Equal(t, "indexed while root removed", moderationThreadComment(t, thread, during).Comment.Record.(*comments.CommentRecord).Content) + actor := postRemovalActorComments(t, commentService, commenterDID, "", 10, nil) + require.Len(t, actor.Comments, 4) + assert.Equal(t, []string{during, first, visible, second}, []string{actor.Comments[0].URI, actor.Comments[1].URI, actor.Comments[2].URI, actor.Comments[3].URI}) +} + +// The profile's commentCount must agree with the list actor.getComments +// serves: both exclude comments under an instance-removed root and comments +// that are themselves instance-removed. +func TestPostRemovalProfileCommentCountMatchesActorComments(t *testing.T) { + db, commentService, moderationService, subject, _, commenterDID, _, visible, _ := postRemovalFixture(t) + var visibleRootURI, visibleRootCID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT root_uri, root_cid FROM comments WHERE uri = $1`, visible).Scan(&visibleRootURI, &visibleRootCID)) + directlyRemoved := postRemovalComment(t, db, commenterDID, visibleRootURI, visibleRootCID, "comment removed directly", time.Now().Add(-4*time.Minute)) + userRepo := postgres.NewUserRepository(db) + baseline, err := userRepo.GetProfileStats(t.Context(), commenterDID) + require.NoError(t, err) + require.Equal(t, 4, baseline.CommentCount, "fixture: four comments before any removal") + + removeFixturePost(t, moderationService, subject) + removedComment, err := moderationService.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "commentadmin")), moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: directlyRemoved, CID: moderatedCommentCID}, ExpectedVersion: "v0", + IdempotencyKey: "remove-comment", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removedComment.Outcome) + + listed := postRemovalActorComments(t, commentService, commenterDID, "", 50, nil) + require.Len(t, listed.Comments, 1) + assert.Equal(t, visible, listed.Comments[0].URI) + stats, err := userRepo.GetProfileStats(t.Context(), commenterDID) + require.NoError(t, err) + assert.Equal(t, len(listed.Comments), stats.CommentCount, + "profile commentCount must equal what actor.getComments lists: removed-root and removed comments are excluded from both") +} diff --git a/internal/core/embeds/post_blob_cids_test.go b/internal/core/embeds/post_blob_cids_test.go new file mode 100644 index 0000000..ceda365 --- /dev/null +++ b/internal/core/embeds/post_blob_cids_test.go @@ -0,0 +1,63 @@ +package embeds + +import ( + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multibase" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPostBlobCIDs(t *testing.T) { + parsed, err := cid.Decode(testCID) + require.NoError(t, err) + base58Alias, err := parsed.StringOfBase(multibase.Base58BTC) + require.NoError(t, err) + legacyBlob := map[string]interface{}{"cid": secondTestCID, "mimeType": "image/png"} + + for _, test := range []struct { + name string + embed map[string]interface{} + want []string + }{ + {name: "nil embed", embed: nil}, + {name: "empty embed", embed: map[string]interface{}{}}, + {name: "images retain first-seen order", embed: imagesEmbed(imageEntry(blobRef(testCID)), imageEntry(blobRef(secondTestCID))), want: []string{testCID, secondTestCID}}, + {name: "video thumbnail is proxied but video is not", embed: map[string]interface{}{ + "$type": TypeVideo, "video": blobRef(testCID), "thumbnail": blobRef(secondTestCID), + }, want: []string{secondTestCID}}, + {name: "video without thumbnail has no proxy blob", embed: map[string]interface{}{"$type": TypeVideo, "video": blobRef(testCID)}}, + {name: "external thumbnail and gallery", embed: map[string]interface{}{ + "$type": TypeExternal, "external": map[string]interface{}{ + "thumb": blobRef(testCID), "images": []interface{}{imageEntry(legacyBlob)}, + }, + }, want: []string{testCID, secondTestCID}}, + {name: "legacy top-level cid", embed: imagesEmbed(imageEntry(legacyBlob)), want: []string{secondTestCID}}, + {name: "noncanonical CID becomes canonical", embed: imagesEmbed(imageEntry(blobRef(base58Alias))), want: []string{testCID}}, + {name: "duplicate encodings keep first occurrence", embed: imagesEmbed( + imageEntry(blobRef(secondTestCID)), imageEntry(blobRef(testCID)), + imageEntry(blobRef(base58Alias)), imageEntry(legacyBlob), + ), want: []string{secondTestCID, testCID}}, + {name: "malformed image siblings are skipped", embed: imagesEmbed( + "not an image", imageEntry("not a blob"), imageEntry(map[string]interface{}{"ref": map[string]interface{}{"$link": 5}}), + imageEntry(blobRef("bafynotacid")), imageEntry(blobRef(testCID)), + ), want: []string{testCID}}, + {name: "malformed external siblings are skipped", embed: map[string]interface{}{ + "$type": TypeExternal, "external": map[string]interface{}{ + "thumb": map[string]interface{}{"ref": map[string]interface{}{}}, + "images": []interface{}{imageEntry(blobRef(testCID)), nil, imageEntry(blobRef("bad cid")), imageEntry(legacyBlob)}, + }, + }, want: []string{testCID, secondTestCID}}, + {name: "quote embeds do not expose blobs", embed: map[string]interface{}{ + "$type": TypePost, "post": map[string]interface{}{"embed": imagesEmbed(imageEntry(blobRef(testCID)))}, + }}, + {name: "projected images are not indexed blobs", embed: map[string]interface{}{ + "$type": TypeImages + viewSuffix, "images": []interface{}{imageEntry(blobRef(testCID))}, + }}, + } { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, PostBlobCIDs(test.embed)) + }) + } +} diff --git a/internal/core/embeds/servable_embed_test.go b/internal/core/embeds/servable_embed_test.go new file mode 100644 index 0000000..950b6ba --- /dev/null +++ b/internal/core/embeds/servable_embed_test.go @@ -0,0 +1,254 @@ +package embeds + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// authorURL is an image-proxy URL an author typed into a record by hand. It +// names secondTestCID, so the served view must not carry it unless +// PostBlobCIDs also reports that CID for the same stored embed. +func authorURL(preset string) string { + return "https://img.coves.social/img/" + preset + "/plain/" + testDID + "/" + secondTestCID +} + +func copyEmbed(t *testing.T, stored interface{}) interface{} { + t.Helper() + encoded, err := json.Marshal(stored) + require.NoError(t, err) + var decoded interface{} + require.NoError(t, json.Unmarshal(encoded, &decoded)) + return decoded +} + +// servedProxyURLs collects every string in value that names the image proxy. +func servedProxyURLs(value interface{}) []string { + var urls []string + switch typed := value.(type) { + case string: + if strings.Contains(typed, "/img/") { + urls = append(urls, typed) + } + case map[string]interface{}: + for _, field := range typed { + urls = append(urls, servedProxyURLs(field)...) + } + case []interface{}: + for _, entry := range typed { + urls = append(urls, servedProxyURLs(entry)...) + } + } + return urls +} + +// serve runs the post read path's projection: scanPostView's ServableEmbed, +// then the handlers' HydrateView. +func serve(t *testing.T, stored interface{}) interface{} { + t.Helper() + served := ServableEmbed(copyEmbed(t, stored)) + if embed, ok := served.(map[string]interface{}); ok { + HydrateView(embed, testDID, testPDS) + } + return served +} + +func TestServableEmbed_ServesOnlyMediaURLsDerivedFromBlobs(t *testing.T) { + withProxy(t, "https://img.coves.social") + const blueskyURI = "at://did:plc:quoted/app.bsky.feed.post/3kquoted" + const covesURI = "at://did:plc:quoted/social.coves.community.postv2/3kquoted" + forgedResolved := map[string]interface{}{ + "text": "forged preview", "images": []interface{}{map[string]interface{}{ + "thumb": authorURL("content_preview"), "fullsize": authorURL("content_full"), + }}, + } + quoteOf := func(embedType, uri string) map[string]interface{} { + return map[string]interface{}{ + "$type": embedType, "post": map[string]interface{}{"uri": uri, "cid": testCID}, + "resolved": forgedResolved, "images": []interface{}{map[string]interface{}{"thumb": authorURL("content_preview")}}, + } + } + strongRef := func(uri string) map[string]interface{} { + return map[string]interface{}{"$type": TypePost, "post": map[string]interface{}{"uri": uri, "cid": testCID}} + } + + for _, test := range []struct { + name string + stored interface{} + want interface{} + }{ + { + name: "external thumb given as a URL string is dropped", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "title": "An article", "thumb": authorURL(presetEmbedThumbnail), + }}, + want: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", "title": "An article", + }}, + }, + { + name: "external thumb given as a non-blob value is dropped", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": []interface{}{authorURL(presetEmbedThumbnail)}, + }}, + want: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", + }}, + }, + { + name: "external gallery view URLs without a blob are dropped", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/gallery", "images": []interface{}{map[string]interface{}{ + "alt": "gallery", "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }}, + }}, + want: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/gallery", "images": []interface{}{map[string]interface{}{"alt": "gallery"}}, + }}, + }, + { + name: "images record entry with view URLs and no blob is dropped to its alt", + stored: imagesEmbed(map[string]interface{}{ + "alt": "no blob", "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }), + want: imagesEmbed(map[string]interface{}{"alt": "no blob"}), + }, + { + name: "images record entry with a blob serves only the blob's URLs", + stored: imagesEmbed(map[string]interface{}{ + "alt": "real", "image": blobRef(testCID), "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }), + want: map[string]interface{}{"$type": TypeImages + viewSuffix, "images": []interface{}{map[string]interface{}{ + "alt": "real", + "thumb": "https://img.coves.social/img/content_preview/plain/" + testDID + "/" + testCID, + "fullsize": "https://img.coves.social/img/content_full/plain/" + testDID + "/" + testCID, + }}}, + }, + { + name: "video given as URL strings is dropped", + stored: map[string]interface{}{ + "$type": TypeVideo, "alt": "clip", "video": "https://pds.example.com/video.mp4", "thumbnail": authorURL(presetContentPreview), + }, + want: map[string]interface{}{"$type": TypeVideo, "alt": "clip"}, + }, + { + name: "stored images#view is not served", + stored: map[string]interface{}{"$type": TypeImages + viewSuffix, "images": []interface{}{map[string]interface{}{ + "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }}}, + }, + { + name: "stored external#view is not served", + stored: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": authorURL(presetEmbedThumbnail), + }}, + }, + { + name: "stored video#view is not served", + stored: map[string]interface{}{"$type": TypeVideo + viewSuffix, "video": "https://pds.example.com/video.mp4", "thumbnail": authorURL(presetContentPreview)}, + }, + { + name: "a type outside the post embed union is not served", + stored: map[string]interface{}{"$type": "app.bsky.embed.images#view", "images": []interface{}{map[string]interface{}{ + "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }}}, + }, + {name: "an untyped embed is not served", stored: map[string]interface{}{"thumb": authorURL(presetEmbedThumbnail)}}, + {name: "a non-object embed is not served", stored: []interface{}{authorURL(presetEmbedThumbnail)}}, + {name: "a string embed is not served", stored: authorURL(presetEmbedThumbnail)}, + {name: "Coves quote record keeps only its strongRef", stored: quoteOf(TypePost, covesURI), want: strongRef(covesURI)}, + {name: "Coves quote view keeps only its strongRef", stored: quoteOf(TypePost+viewSuffix, covesURI), want: strongRef(covesURI)}, + { + name: "Bluesky quote view drops its stored resolved for server resolution", + stored: quoteOf(TypePost+viewSuffix, blueskyURI), + want: strongRef(blueskyURI), + }, + {name: "Bluesky quote record keeps only its strongRef", stored: quoteOf(TypePost, blueskyURI), want: strongRef(blueskyURI)}, + { + name: "Bluesky collection forged after a Coves collection keeps only its strongRef", + stored: quoteOf(TypePost+viewSuffix, "at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted"), + want: strongRef("at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted"), + }, + { + name: "Bluesky collection forged in a fragment keeps only its strongRef", + stored: quoteOf(TypePost+viewSuffix, covesURI+"#/app.bsky.feed.post/3k"), + want: strongRef(covesURI + "#/app.bsky.feed.post/3k"), + }, + { + name: "Bluesky collection forged as the authority keeps only its strongRef", + stored: quoteOf(TypePost+viewSuffix, "at://app.bsky.feed.post/social.coves.community.postv2/3kquoted"), + want: strongRef("at://app.bsky.feed.post/social.coves.community.postv2/3kquoted"), + }, + { + name: "blob-backed external thumb is served through the proxy", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": blobRef(testCID), + }}, + want: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": "https://img.coves.social/img/embed_thumbnail/plain/" + testDID + "/" + testCID, + }}, + }, + } { + t.Run(test.name, func(t *testing.T) { + served := serve(t, test.stored) + if test.want == nil { + assert.Nil(t, served) + } else { + assert.Equal(t, test.want, served) + } + + // The invariant the moderation blocks rely on: every proxy URL the + // view serves names a CID PostBlobCIDs reports for the stored embed. + stored, _ := copyEmbed(t, test.stored).(map[string]interface{}) + blocked := PostBlobCIDs(stored) + for _, url := range servedProxyURLs(served) { + covered := false + for _, cid := range blocked { + if strings.HasSuffix(url, "/"+testDID+"/"+cid) { + covered = true + } + } + assert.True(t, covered, "served proxy URL %s has no block-derivable CID in %v", url, blocked) + } + }) + } +} + +func TestServableEmbed_LeavesTheStoredRecordEmbedIntact(t *testing.T) { + stored := map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": authorURL(presetEmbedThumbnail), + }} + encoded, err := json.Marshal(stored) + require.NoError(t, err) + var decoded interface{} + require.NoError(t, json.Unmarshal(encoded, &decoded)) + + ServableEmbed(decoded) + + // ServableEmbed may edit the value it is handed; the record path decodes + // its own copy, so the bytes it serves are untouched. + var record interface{} + require.NoError(t, json.Unmarshal(encoded, &record)) + assert.Equal(t, stored, record) +} + +func TestIsBlueskyPostURI(t *testing.T) { + for uri, want := range map[string]bool{ + "at://did:plc:quoted/app.bsky.feed.post/3kquoted": true, + "at://quoted.example.com/app.bsky.feed.post/3kquoted": true, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted": false, + "at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted": false, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted#/app.bsky.feed.post/1": false, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted?/app.bsky.feed.post/1": false, + "at://app.bsky.feed.post/social.coves.community.postv2/3kquoted": false, + "at://did:plc:quoted/app.bsky.feed.post": false, + "at://did:plc:quoted/app.bsky.feed.post/": false, + "https://did:plc:quoted/app.bsky.feed.post/3kquoted": false, + "": false, + } { + assert.Equal(t, want, IsBlueskyPostURI(uri), uri) + } +} diff --git a/internal/core/embeds/view.go b/internal/core/embeds/view.go index 9f9412d..cc4c604 100644 --- a/internal/core/embeds/view.go +++ b/internal/core/embeds/view.go @@ -28,6 +28,7 @@ package embeds import ( "log/slog" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/ipfs/go-cid" "Coves/internal/core/blobs" @@ -84,8 +85,8 @@ type mutation func() // idempotent because a projected embed's $type is a #view type, which this // function does not act on. // -// social.coves.embed.post carries no blobs; it is projected to its own #view by -// posts.TransformPostEmbeds, which resolves the quoted record. +// social.coves.embed.post carries no blobs; Bluesky quotes are resolved by +// posts.TransformPostEmbeds, while Coves quotes retain only their strongRef. func HydrateView(embed map[string]interface{}, ownerDID, ownerPDSURL string) { if embed == nil { return @@ -186,8 +187,7 @@ func HydrateCommentView(embed map[string]interface{}, ownerDID, ownerPDSURL stri return } - // TypePost carries no blobs and is projected by posts.TransformPostEmbeds, - // which resolves the quoted record; anything other than images is outside + // TypePost carries no blobs; anything other than images is outside // the comment union entirely. if embedType, _ := embed["$type"].(string); embedType != TypeImages { return @@ -196,6 +196,132 @@ func HydrateCommentView(embed map[string]interface{}, ownerDID, ownerPDSURL stri HydrateView(embed, ownerDID, ownerPDSURL) } +// ServableEmbed reduces a stored embed to what a post or comment view may +// serve, and runs before HydrateView or HydrateCommentView projects it. It is +// the serving half of the rule moderation blocks depend on: the only media URLs +// a view carries are the ones HydrateView derives from blob references, which +// are exactly the CIDs PostBlobCIDs and CommentImageCIDs report for blocking. +// +// The stored embed is the author's record, and nothing on the firehose path +// validates it, so a URL string in it is author-written. Served verbatim it +// would point readers at an image no removal blocks, including one under the +// author's own DID that an illegal-content removal must refuse. The AppView +// itself never stores a media URL: post create rejects a string thumb, the +// unfurl path uploads thumbnails as blobs, and posts.TransformPostEmbeds +// resolves Bluesky quote previews at serve time, after this runs. +// +// So: +// - images, video and external keep their record shape, minus any field the +// record declares as a blob whose value is not an object (image, thumb, +// video, thumbnail) and minus the view-only thumb and fullsize on image +// entries, including an external gallery's. +// - A quote, record or #view, is rebuilt to its strongRef; see projectQuote. +// - Anything else is not served and yields nil: a stored #view of a media +// type, a type outside the record union, an untyped or non-object embed. +// +// It edits the embed it is given. Callers pass a freshly decoded copy; the +// verbatim record decodes its own and keeps every byte. +func ServableEmbed(stored interface{}) interface{} { + embed, isObject := stored.(map[string]interface{}) + if !isObject { + return nil + } + + switch embedType, _ := embed["$type"].(string); embedType { + case TypeImages: + dropViewImageURLs(embed["images"]) + case TypeVideo: + dropNonBlob(embed, "video") + dropNonBlob(embed, "thumbnail") + case TypeExternal: + dropNonBlob(embed, "external") + if external, isObject := embed["external"].(map[string]interface{}); isObject { + dropNonBlob(external, "thumb") + dropViewImageURLs(external["images"]) + } + case TypePost, TypePost + viewSuffix: + return projectQuote(embed) + default: + return nil + } + return embed +} + +// dropNonBlob deletes object[field] when it is present and not an object. A +// blob reference is always an object, so anything else in a blob position is +// an author-written value the view must not carry. +func dropNonBlob(object map[string]interface{}, field string) { + value, present := object[field] + if !present { + return + } + if _, isObject := value.(map[string]interface{}); !isObject { + delete(object, field) + } +} + +// dropViewImageURLs removes the view-only URL fields from every image entry in +// a record's image list. HydrateView writes thumb and fullsize from the entry's +// blob when it projects; left in place they would be served whenever it +// cannot. +func dropViewImageURLs(value interface{}) { + images, _ := value.([]interface{}) + for _, entry := range images { + image, isObject := entry.(map[string]interface{}) + if !isObject { + continue + } + dropNonBlob(image, "image") + delete(image, "thumb") + delete(image, "fullsize") + } +} + +// projectQuote rebuilds a quote embed, record or #view, to its strongRef. +// +// D-QUOTES: Coves-quoted posts are not hydrated server-side. The served quote +// is the strongRef only, so a removed quoted post leaves no copied content; +// post.get on its URI returns #moderatedPost. A Bluesky quote is rebuilt the +// same way: posts.TransformPostEmbeds resolves its preview at serve time from +// the record type, so a stored resolved object, which only an author can have +// written, is never served in its place. +// +// Decision: record.embed is not projected. The lexicon serves record as the +// quoter's own authored record verbatim, so its embed keeps every byte the +// quoter wrote, including preview fields it forged. None of that is +// AppView-held content of the quoted post; the AppView adds resolved only at +// serve time, only for Bluesky URIs, and only to the top-level embed. +func projectQuote(embed map[string]interface{}) map[string]interface{} { + projected := map[string]interface{}{"$type": TypePost} + post, _ := embed["post"].(map[string]interface{}) + strongRef := make(map[string]interface{}, 2) + if uri, ok := post["uri"].(string); ok && uri != "" { + strongRef["uri"] = uri + } + if cid, ok := post["cid"].(string); ok && cid != "" { + strongRef["cid"] = cid + } + if len(strongRef) > 0 { + projected["post"] = strongRef + } + return projected +} + +// blueskyPostCollection is the NSID of a Bluesky post record. +const blueskyPostCollection = "app.bsky.feed.post" + +// IsBlueskyPostURI reports whether uri is a syntactically valid AT-URI whose +// collection segment is app.bsky.feed.post and which names a record. The +// substring elsewhere in a URI (a Coves collection's record key path, a +// fragment, a handle authority) does not make it a Bluesky post. +func IsBlueskyPostURI(uri string) bool { + parsed, err := syntax.ParseATURI(uri) + if err != nil { + return false + } + return parsed.Collection().String() == blueskyPostCollection && parsed.RecordKey() != "" +} + // CommentImageCIDs returns the canonical CIDs of every image blob that // HydrateCommentView can serve through the image proxy for a comment embed. // Moderation blocks exactly these, so blocking and serving cannot drift apart. @@ -384,3 +510,44 @@ func blobCID(value interface{}) string { return "" } + +// PostBlobCIDs returns the canonical, first-seen CIDs of post blobs served +// through the image proxy. Malformed entries are skipped; video blobs are +// served directly by the PDS, so only their thumbnails are included. +func PostBlobCIDs(embed map[string]interface{}) []string { + var cids []string + seen := make(map[string]bool) + addBlob := func(value interface{}) { + parsed, err := cid.Decode(blobCID(value)) + if err != nil { + return + } + canonical := parsed.String() + if !seen[canonical] { + seen[canonical] = true + cids = append(cids, canonical) + } + } + addImages := func(value interface{}) { + images, _ := value.([]interface{}) + for _, entry := range images { + image, ok := entry.(map[string]interface{}) + if ok { + addBlob(image["image"]) + } + } + } + + switch embedType, _ := embed["$type"].(string); embedType { + case TypeImages: + addImages(embed["images"]) + case TypeVideo: + addBlob(embed["thumbnail"]) + case TypeExternal: + if external, ok := embed["external"].(map[string]interface{}); ok { + addBlob(external["thumb"]) + addImages(external["images"]) + } + } + return cids +} diff --git a/internal/core/moderation/fake_store_test.go b/internal/core/moderation/fake_store_test.go index 3b9af32..3f3ba7b 100644 --- a/internal/core/moderation/fake_store_test.go +++ b/internal/core/moderation/fake_store_test.go @@ -22,6 +22,7 @@ type inMemoryModerationIdempotencyKey struct { type inMemoryModerationState struct { indexedComments map[string]moderation.IndexedComment + indexedPosts map[string]moderation.IndexedPost versions map[string]int64 actions map[string]moderation.Action activeRemovals map[inMemoryModerationDecisionKey]string @@ -33,6 +34,7 @@ type inMemoryModerationState struct { func (state inMemoryModerationState) copy() inMemoryModerationState { working := inMemoryModerationState{ indexedComments: make(map[string]moderation.IndexedComment, len(state.indexedComments)), + indexedPosts: make(map[string]moderation.IndexedPost, len(state.indexedPosts)), versions: make(map[string]int64, len(state.versions)), actions: make(map[string]moderation.Action, len(state.actions)), activeRemovals: make(map[inMemoryModerationDecisionKey]string, len(state.activeRemovals)), @@ -44,6 +46,10 @@ func (state inMemoryModerationState) copy() inMemoryModerationState { comment.ImageCIDs = append([]string(nil), comment.ImageCIDs...) working.indexedComments[key] = comment } + for key, post := range state.indexedPosts { + post.BlobCIDs = append([]string(nil), post.BlobCIDs...) + working.indexedPosts[key] = post + } for key, version := range state.versions { working.versions[key] = version } @@ -78,6 +84,7 @@ func newInMemoryModerationStore(now time.Time) *inMemoryModerationStore { now: now, state: inMemoryModerationState{ indexedComments: make(map[string]moderation.IndexedComment), + indexedPosts: make(map[string]moderation.IndexedPost), versions: make(map[string]int64), actions: make(map[string]moderation.Action), activeRemovals: make(map[inMemoryModerationDecisionKey]string), @@ -161,6 +168,15 @@ func (transaction *inMemoryModerationTransaction) ReadIndexedComment(_ context.C return &comment, nil } +func (transaction *inMemoryModerationTransaction) ReadIndexedPost(_ context.Context, subjectURI string) (*moderation.IndexedPost, error) { + post, exists := transaction.state.indexedPosts[subjectURI] + if !exists { + return nil, moderation.ErrSubjectNotIndexed + } + post.BlobCIDs = append([]string(nil), post.BlobCIDs...) + return &post, nil +} + func (transaction *inMemoryModerationTransaction) GetAction(_ context.Context, actionID string) (*moderation.Action, error) { action, exists := transaction.state.actions[actionID] if !exists { diff --git a/internal/core/moderation/indexed_subject.go b/internal/core/moderation/indexed_subject.go new file mode 100644 index 0000000..cba65d5 --- /dev/null +++ b/internal/core/moderation/indexed_subject.go @@ -0,0 +1,55 @@ +package moderation + +import ( + "context" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type indexedSubject struct { + URI string + CID string + Collection string + AuthorDeleted bool + OwnerDID string + CommunityDID string + BlobCIDs []string +} + +type indexedSubjectReader interface { + ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + ReadIndexedPost(ctx context.Context, subjectURI string) (*IndexedPost, error) +} + +func readIndexedSubject(ctx context.Context, reader indexedSubjectReader, subjectURI string) (*indexedSubject, error) { + uri, err := syntax.ParseATURI(subjectURI) + if err != nil { + return nil, fmt.Errorf("%w: invalid subject URI: %w", ErrInvalidSubject, err) + } + collection := uri.Collection().String() + switch collection { + case CommentCollection: + comment, err := reader.ReadIndexedComment(ctx, subjectURI) + if err != nil || comment == nil { + return nil, err + } + return &indexedSubject{ + URI: comment.URI, CID: comment.CID, Collection: collection, + AuthorDeleted: comment.AuthorDeleted, OwnerDID: comment.OwnerDID, + CommunityDID: comment.CommunityDID, BlobCIDs: comment.ImageCIDs, + }, nil + case PostV2Collection, LegacyPostCollection: + post, err := reader.ReadIndexedPost(ctx, subjectURI) + if err != nil || post == nil { + return nil, err + } + return &indexedSubject{ + URI: post.URI, CID: post.CID, Collection: collection, + AuthorDeleted: post.AuthorDeleted, OwnerDID: post.OwnerDID, + CommunityDID: post.CommunityDID, BlobCIDs: post.BlobCIDs, + }, nil + default: + return nil, fmt.Errorf("%w: unsupported subject collection", ErrInvalidSubject) + } +} diff --git a/internal/core/moderation/media.go b/internal/core/moderation/media.go index d2cc529..30d0ccf 100644 --- a/internal/core/moderation/media.go +++ b/internal/core/moderation/media.go @@ -6,10 +6,11 @@ import ( "log/slog" ) -// MediaTransaction provides the operations needed to reconcile a comment's images. +// MediaTransaction provides the operations needed to reconcile a subject's images. type MediaTransaction interface { ActiveRemoval(ctx context.Context, authorityDID, subjectURI string) (*Action, error) ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + ReadIndexedPost(ctx context.Context, subjectURI string) (*IndexedPost, error) // InsertNewMediaBlocks inserts only blocks not already active for the action // and returns the blocks it inserted. InsertNewMediaBlocks(ctx context.Context, blocks []MediaBlock) ([]MediaBlock, error) @@ -41,11 +42,30 @@ func (r *MediaReconciler) ReconcileTx(ctx context.Context, tx *sql.Tx, subjectUR if err != nil || action == nil { return nil, err } - comment, err := bound.ReadIndexedComment(ctx, subjectURI) + subject, err := readIndexedSubject(ctx, bound, subjectURI) if err != nil { return nil, err } - return bound.InsertNewMediaBlocks(ctx, imageMediaBlocks(comment, action)) + if subject == nil { + return nil, ErrSubjectNotIndexed + } + return bound.InsertNewMediaBlocks(ctx, imageMediaBlocks(subject, action)) +} + +// ReconcileIncomingTx blocks blobs of incoming content the consumer did not +// index, such as a recreate of a deleted post whose tombstone is kept. The +// owner's repository still serves those blobs, and the stored row does not +// name them, so the caller passes the owner and CIDs from the incoming record. +func (r *MediaReconciler) ReconcileIncomingTx(ctx context.Context, tx *sql.Tx, subjectURI, ownerDID string, blobCIDs []string) ([]MediaBlock, error) { + if len(blobCIDs) == 0 { + return nil, nil + } + bound := r.binder.BindTransaction(tx) + action, err := bound.ActiveRemoval(ctx, r.instanceDID, subjectURI) + if err != nil || action == nil { + return nil, err + } + return bound.InsertNewMediaBlocks(ctx, imageMediaBlocks(&indexedSubject{OwnerDID: ownerDID, BlobCIDs: blobCIDs}, action)) } // Purge removes cached bytes of newly blocked blobs after commit. @@ -76,15 +96,15 @@ func purgeMediaBlocks(purger MediaPurger, blocks []MediaBlock) { } } -func imageMediaBlocks(comment *IndexedComment, action *Action) []MediaBlock { +func imageMediaBlocks(subject *indexedSubject, action *Action) []MediaBlock { var blocks []MediaBlock seen := make(map[string]bool) - for _, cid := range comment.ImageCIDs { + for _, cid := range subject.BlobCIDs { if seen[cid] { continue } seen[cid] = true - blocks = append(blocks, MediaBlock{OwnerDID: comment.OwnerDID, BlobCID: cid, ActionID: action.ID}) + blocks = append(blocks, MediaBlock{OwnerDID: subject.OwnerDID, BlobCID: cid, ActionID: action.ID}) if action.Reason == illegalContentReason { blocks = append(blocks, MediaBlock{BlobCID: cid, ActionID: action.ID}) } diff --git a/internal/core/moderation/mutation_validation.go b/internal/core/moderation/mutation_validation.go index 4202090..3630b01 100644 --- a/internal/core/moderation/mutation_validation.go +++ b/internal/core/moderation/mutation_validation.go @@ -31,9 +31,14 @@ func validateMutationFields(key, expectedVersion, reason, privateNote string) er return nil } -func validCommentStrongRef(ref StrongRef) bool { +func validContentStrongRef(ref StrongRef) bool { uri, err := syntax.ParseATURI(ref.URI) - if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" || uri.Collection().String() != CommentCollection { + if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" { + return false + } + switch uri.Collection().String() { + case CommentCollection, PostV2Collection, LegacyPostCollection: + default: return false } _, err = syntax.ParseCID(ref.CID) diff --git a/internal/core/moderation/post_rules_test.go b/internal/core/moderation/post_rules_test.go new file mode 100644 index 0000000..1cd4567 --- /dev/null +++ b/internal/core/moderation/post_rules_test.go @@ -0,0 +1,287 @@ +package moderation_test + +import ( + "context" + "database/sql" + "testing" + + "Coves/internal/core/moderation" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + postRulesAuthorDID = "did:plc:postauthor" + postRulesCommunityDID = "did:plc:postcommunity" + postRulesCID = "bafyreipostversion" + postRulesURI = "at://did:plc:postauthor/social.coves.community.postv2/3kabc" + legacyPostRulesURI = "at://did:plc:postcommunity/social.coves.community.post/3kabc" +) + +func newPostRulesScenario(uri, ownerDID string, blobCIDs []string) removeRulesScenario { + scenario := newRemoveRulesScenario() + scenario.store.state.indexedPosts[uri] = moderation.IndexedPost{ + URI: uri, CID: postRulesCID, OwnerDID: ownerDID, + CommunityDID: postRulesCommunityDID, BlobCIDs: blobCIDs, + } + scenario.reader.record = &moderation.IndexedRecord{URI: uri, CID: postRulesCID} + scenario.request.Subject = moderation.StrongRef{URI: uri, CID: postRulesCID} + return scenario +} + +func TestRemovePostContentRecordsCollectionAndBlocksMedia(t *testing.T) { + for _, test := range []struct { + name, uri, collection, ownerDID, reason string + blobCIDs []string + wantBlocks []moderation.MediaBlock + wantOwnerPurges []removeRulesOwnerPurge + wantBlobPurges []string + }{ + { + name: "author-owned postv2", uri: postRulesURI, collection: moderation.PostV2Collection, + ownerDID: postRulesAuthorDID, reason: removeRulesSpam, + blobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + wantBlocks: []moderation.MediaBlock{ + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage}, + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesSecondImage}, + }, + wantOwnerPurges: []removeRulesOwnerPurge{ + {postRulesAuthorDID, removeRulesFirstImage}, {postRulesAuthorDID, removeRulesSecondImage}, + }, + }, + { + name: "community-owned legacy post", uri: legacyPostRulesURI, collection: moderation.LegacyPostCollection, + ownerDID: postRulesCommunityDID, reason: removeRulesSpam, + blobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + wantBlocks: []moderation.MediaBlock{ + {OwnerDID: postRulesCommunityDID, BlobCID: removeRulesFirstImage}, + {OwnerDID: postRulesCommunityDID, BlobCID: removeRulesSecondImage}, + }, + wantOwnerPurges: []removeRulesOwnerPurge{ + {postRulesCommunityDID, removeRulesFirstImage}, {postRulesCommunityDID, removeRulesSecondImage}, + }, + }, + { + name: "illegal content also blocks every owner", uri: postRulesURI, collection: moderation.PostV2Collection, + ownerDID: postRulesAuthorDID, reason: removeRulesIllegal, + blobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + wantBlocks: []moderation.MediaBlock{ + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage}, + {BlobCID: removeRulesFirstImage}, + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesSecondImage}, + {BlobCID: removeRulesSecondImage}, + }, + wantOwnerPurges: []removeRulesOwnerPurge{ + {postRulesAuthorDID, removeRulesFirstImage}, {postRulesAuthorDID, removeRulesSecondImage}, + }, + wantBlobPurges: []string{removeRulesFirstImage, removeRulesSecondImage}, + }, + { + name: "post with no blobs", uri: postRulesURI, collection: moderation.PostV2Collection, + ownerDID: postRulesAuthorDID, reason: removeRulesSpam, + }, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newPostRulesScenario(test.uri, test.ownerDID, test.blobCIDs) + scenario.request.Reason = test.reason + scenario.purger.onPurge = func() { + require.NotEmpty(t, scenario.store.state.activeRemovals, "purge must follow the committed decision") + require.NotEmpty(t, scenario.store.state.mediaBlocks, "purge must follow committed block insertion") + } + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + assert.Equal(t, test.collection, result.Action.SubjectCollection) + assert.Equal(t, postRulesCommunityDID, result.Action.SubjectCommunityDID) + assert.Equal(t, *result.Action, scenario.store.state.actions[result.Action.ID]) + assert.Equal(t, result.Action.ID, scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, test.uri}]) + wantBlocks := make(map[moderation.MediaBlock]bool) + for _, block := range test.wantBlocks { + block.ActionID = result.Action.ID + wantBlocks[block] = true + } + assert.Equal(t, wantBlocks, scenario.store.state.mediaBlocks) + assert.Equal(t, test.wantOwnerPurges, scenario.purger.ownerPurges) + assert.Equal(t, test.wantBlobPurges, scenario.purger.blobPurges) + assert.Equal(t, len(test.blobCIDs) > 0, containsModerationWrite(scenario.store.writeCalls, "InsertMediaBlocks")) + }) + } +} + +func containsModerationWrite(writes []string, wanted string) bool { + for _, write := range writes { + if write == wanted { + return true + } + } + return false +} + +func TestRemovePostContentChecksIndexedRecord(t *testing.T) { + for _, test := range []struct { + name, requestedCID string + deleted, missing bool + wantError error + }{ + {name: "author-deleted post ignores stale strongRef", requestedCID: "bafyreistalepostversion", deleted: true}, + {name: "present post refuses stale CID", requestedCID: "bafyreistalepostversion", wantError: moderation.ErrContentChanged}, + {name: "never-indexed post", requestedCID: postRulesCID, missing: true, wantError: moderation.ErrSubjectNotFound}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newPostRulesScenario(postRulesURI, postRulesAuthorDID, nil) + scenario.request.Subject.CID = test.requestedCID + post := scenario.store.state.indexedPosts[postRulesURI] + post.AuthorDeleted = test.deleted + scenario.store.state.indexedPosts[postRulesURI] = post + scenario.reader.record.Deleted = test.deleted + if test.missing { + delete(scenario.store.state.indexedPosts, postRulesURI) + scenario.reader.record = nil + scenario.reader.err = moderation.ErrSubjectNotIndexed + } + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + if test.wantError != nil { + require.ErrorIs(t, err, test.wantError) + assert.Nil(t, result) + assertRemoveRulesNoWrites(t, scenario) + return + } + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + assert.Equal(t, postRulesCID, result.Action.ObservedCID) + assert.Equal(t, moderation.RecordStateDeleted, result.State.RecordState) + assert.Nil(t, result.State.CurrentSubject) + }) + } +} + +func TestRestorePostContentReviewsCurrentPostAndDeactivatesBlocks(t *testing.T) { + for _, test := range []struct { + name string + noReview bool + deleted bool + staleCID bool + wantError error + }{ + {name: "present post requires review", noReview: true, wantError: moderation.ErrInvalidRequest}, + {name: "present post refuses stale review", staleCID: true, wantError: moderation.ErrContentChanged}, + {name: "present post accepts current review"}, + {name: "author-deleted post needs no review", deleted: true, noReview: true}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newPostRulesScenario(postRulesURI, postRulesAuthorDID, []string{removeRulesFirstImage}) + post := scenario.store.state.indexedPosts[postRulesURI] + post.AuthorDeleted = test.deleted + scenario.store.state.indexedPosts[postRulesURI] = post + scenario.reader.record.Deleted = test.deleted + removal := moderation.Action{ + ID: "post-removal", Action: moderation.ActionRemove, ActorDID: removeRulesAdminDID, + AuthorityDID: removeRulesInstanceDID, ScopeKind: moderation.ScopeInstance, + SubjectURI: postRulesURI, SubjectCollection: moderation.PostV2Collection, + SubjectCommunityDID: postRulesCommunityDID, ObservedCID: postRulesCID, + } + scenario.store.state.actions[removal.ID] = removal + scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, postRulesURI}] = removal.ID + scenario.store.state.versions[postRulesURI] = 1 + block := moderation.MediaBlock{OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage, ActionID: removal.ID} + scenario.store.state.mediaBlocks[block] = true + request := moderation.RestoreContentRequest{ + ActionID: removal.ID, ExpectedVersion: "v1", IdempotencyKey: "restore-post", + Reason: removeRulesSpam, ReviewedSubject: &moderation.StrongRef{URI: postRulesURI, CID: postRulesCID}, + } + if test.noReview { + request.ReviewedSubject = nil + } + if test.staleCID { + request.ReviewedSubject.CID = "bafyreistalepostversion" + } + before := scenario.store.state.copy() + result, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, request) + if test.wantError != nil { + require.ErrorIs(t, err, test.wantError) + assert.Nil(t, result) + assert.Equal(t, before, scenario.store.state) + return + } + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + assert.Equal(t, moderation.ActionRestore, result.Action.Action) + assert.Equal(t, removal.ID, result.Action.ReversesActionID) + assert.Equal(t, moderation.PostV2Collection, result.Action.SubjectCollection) + assert.Equal(t, postRulesCommunityDID, result.Action.SubjectCommunityDID) + assert.Equal(t, "v2", result.State.Version) + assert.Equal(t, moderation.ModerationStateClear, result.State.Moderation.State) + assert.Empty(t, scenario.store.state.activeRemovals) + assert.False(t, scenario.store.state.mediaBlocks[block], "restore deactivates its removal's blob block") + if test.deleted { + assert.Equal(t, moderation.RecordStateDeleted, result.State.RecordState) + assert.Nil(t, result.State.CurrentSubject) + } else { + assert.Equal(t, moderation.RecordStatePresent, result.State.RecordState) + assert.Equal(t, &moderation.StrongRef{URI: postRulesURI, CID: postRulesCID}, result.State.CurrentSubject) + } + }) + } +} + +type postRulesMediaTransaction struct { + post moderation.IndexedPost + action moderation.Action + calls []string + blocks []moderation.MediaBlock +} + +func (transaction *postRulesMediaTransaction) ActiveRemoval(context.Context, string, string) (*moderation.Action, error) { + transaction.calls = append(transaction.calls, "ActiveRemoval") + return &transaction.action, nil +} + +func (transaction *postRulesMediaTransaction) ReadIndexedComment(context.Context, string) (*moderation.IndexedComment, error) { + transaction.calls = append(transaction.calls, "ReadIndexedComment") + return &moderation.IndexedComment{}, nil +} + +func (transaction *postRulesMediaTransaction) ReadIndexedPost(context.Context, string) (*moderation.IndexedPost, error) { + transaction.calls = append(transaction.calls, "ReadIndexedPost") + return &transaction.post, nil +} + +func (transaction *postRulesMediaTransaction) InsertNewMediaBlocks(_ context.Context, blocks []moderation.MediaBlock) ([]moderation.MediaBlock, error) { + transaction.calls = append(transaction.calls, "InsertNewMediaBlocks") + transaction.blocks = append(transaction.blocks, blocks...) + return blocks, nil +} + +type postRulesMediaBinder struct{ bound *postRulesMediaTransaction } + +func (binder postRulesMediaBinder) BindTransaction(*sql.Tx) moderation.MediaTransaction { + return binder.bound +} + +func TestReconcileRemovedPostMediaReadsIndexedPost(t *testing.T) { + bound := &postRulesMediaTransaction{ + post: moderation.IndexedPost{ + URI: postRulesURI, CID: postRulesCID, OwnerDID: postRulesAuthorDID, + BlobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + }, + action: moderation.Action{ID: "post-removal", Reason: removeRulesSpam}, + } + reconciler := moderation.NewMediaReconciler(postRulesMediaBinder{bound}, removeRulesInstanceDID, nil) + blocks, err := reconciler.ReconcileTx(t.Context(), nil, postRulesURI) + require.Equal(t, []string{"ActiveRemoval", "ReadIndexedPost", "InsertNewMediaBlocks"}, bound.calls) + require.NoError(t, err) + want := []moderation.MediaBlock{ + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage, ActionID: bound.action.ID}, + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesSecondImage, ActionID: bound.action.ID}, + } + assert.Equal(t, want, bound.blocks) + assert.Equal(t, want, blocks) +} + +var _ moderation.MediaTransaction = (*postRulesMediaTransaction)(nil) diff --git a/internal/core/moderation/remove.go b/internal/core/moderation/remove.go index 55c0d77..5ea4d39 100644 --- a/internal/core/moderation/remove.go +++ b/internal/core/moderation/remove.go @@ -23,12 +23,10 @@ var removeReasons = map[string]struct{}{ func validateRemoveRequest(request RemoveContentRequest) error { uri, err := syntax.ParseATURI(request.Subject.URI) if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" { - return fmt.Errorf("%w: expected a comment record URI with a DID authority", ErrInvalidSubject) + return fmt.Errorf("%w: expected a content record URI with a DID authority", ErrInvalidSubject) } switch uri.Collection().String() { - case PostV2Collection, LegacyPostCollection: - return fmt.Errorf("%w: post removal is unsupported", ErrInvalidSubject) - case CommentCollection: + case CommentCollection, PostV2Collection, LegacyPostCollection: default: return fmt.Errorf("%w: unsupported subject collection", ErrInvalidSubject) } @@ -85,17 +83,17 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re if request.ExpectedVersion != versionToken(version) { return fail(ErrStateConflict) } - comment, err := tx.ReadIndexedComment(ctx, request.Subject.URI) + subject, err := readIndexedSubject(ctx, tx, request.Subject.URI) if errors.Is(err, ErrSubjectNotIndexed) { return fail(ErrSubjectNotFound) } if err != nil { return unavailable(err) } - if comment == nil { - return unavailable(errors.New("indexed comment missing")) + if subject == nil { + return unavailable(errors.New("indexed subject missing")) } - if !comment.AuthorDeleted && comment.CID != request.Subject.CID { + if !subject.AuthorDeleted && subject.CID != request.Subject.CID { return fail(ErrContentChanged) } active, err := tx.ActiveRemoval(ctx, s.config.InstanceDID, request.Subject.URI) @@ -104,10 +102,10 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re } recordState := RecordStatePresent var current *StrongRef - if comment.AuthorDeleted { + if subject.AuthorDeleted { recordState = RecordStateDeleted } else { - current = &StrongRef{URI: comment.URI, CID: comment.CID} + current = &StrongRef{URI: subject.URI, CID: subject.CID} } if active != nil { state, err := newSubjectState(request.Subject.URI, version, recordState, current, active, s.config.InstanceDID) @@ -119,8 +117,8 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re action, err := tx.InsertAction(ctx, Action{ ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, ScopeKind: ScopeInstance, SubjectURI: request.Subject.URI, - SubjectCollection: CommentCollection, SubjectCommunityDID: comment.CommunityDID, - ObservedCID: comment.CID, Action: ActionRemove, Reason: request.Reason, + SubjectCollection: subject.Collection, SubjectCommunityDID: subject.CommunityDID, + ObservedCID: subject.CID, Action: ActionRemove, Reason: request.Reason, PrivateNote: request.PrivateNote, Origin: OriginLocal, CreatedAt: now, }) if err != nil { @@ -135,7 +133,7 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re if err := tx.SetSubjectVersion(ctx, request.Subject.URI, version+1); err != nil { return unavailable(err) } - newlyBlocked = imageMediaBlocks(comment, action) + newlyBlocked = imageMediaBlocks(subject, action) if len(newlyBlocked) > 0 { if err := tx.InsertMediaBlocks(ctx, newlyBlocked); err != nil { return unavailable(err) diff --git a/internal/core/moderation/remove_rules_test.go b/internal/core/moderation/remove_rules_test.go index 93abeaa..ad9e03c 100644 --- a/internal/core/moderation/remove_rules_test.go +++ b/internal/core/moderation/remove_rules_test.go @@ -33,14 +33,21 @@ type removeRulesOwnerPurge struct { type removeRulesPurger struct { ownerPurges []removeRulesOwnerPurge blobPurges []string + onPurge func() } func (purger *removeRulesPurger) PurgeOwnerBlob(ownerDID, blobCID string) error { + if purger.onPurge != nil { + purger.onPurge() + } purger.ownerPurges = append(purger.ownerPurges, removeRulesOwnerPurge{ownerDID, blobCID}) return nil } func (purger *removeRulesPurger) PurgeBlob(blobCID string) error { + if purger.onPurge != nil { + purger.onPurge() + } purger.blobPurges = append(purger.blobPurges, blobCID) return nil } diff --git a/internal/core/moderation/remove_validation_test.go b/internal/core/moderation/remove_validation_test.go index ec83006..ed20882 100644 --- a/internal/core/moderation/remove_validation_test.go +++ b/internal/core/moderation/remove_validation_test.go @@ -34,8 +34,7 @@ func TestRemoveContentRejectsUnsupportedSubjectsReasonsAndOversizedInputs(t *tes change func(*moderation.RemoveContentRequest) want error }{ - {name: "author-owned post removal is unsupported", collection: moderation.PostV2Collection, want: moderation.ErrInvalidSubject}, - {name: "legacy post removal is unsupported", collection: moderation.LegacyPostCollection, want: moderation.ErrInvalidSubject}, + {name: "non-content collection is unsupported", collection: "app.bsky.feed.post", want: moderation.ErrInvalidSubject}, {name: "unrecognized reason token", change: func(request *moderation.RemoveContentRequest) { request.Reason = "social.coves.moderation.defs#reasonCsam" }, want: moderation.ErrUnsupportedReason}, @@ -81,9 +80,6 @@ func TestRemoveContentRejectsUnsupportedSubjectsReasonsAndOversizedInputs(t *tes result, err := service.RemoveContent(t.Context(), actorDID, request) require.ErrorIs(t, err, test.want) assert.Nil(t, result) - if test.want == moderation.ErrInvalidSubject { - assert.Contains(t, err.Error(), "post", "post removal must be explicitly unsupported") - } assert.Empty(t, store.writeCalls, "validation must not attempt a mutation") assert.Empty(t, store.state.actions) assert.Empty(t, store.state.activeRemovals) diff --git a/internal/core/moderation/restore.go b/internal/core/moderation/restore.go index a86e204..0b6adb2 100644 --- a/internal/core/moderation/restore.go +++ b/internal/core/moderation/restore.go @@ -14,8 +14,8 @@ func validateRestoreRequest(request RestoreContentRequest) error { if err := validateMutationFields(request.IdempotencyKey, request.ExpectedVersion, request.Reason, request.PrivateNote); err != nil { return err } - if request.ReviewedSubject != nil && !validCommentStrongRef(*request.ReviewedSubject) { - return fmt.Errorf("%w: reviewedSubject must be a comment strongRef", ErrInvalidRequest) + if request.ReviewedSubject != nil && !validContentStrongRef(*request.ReviewedSubject) { + return fmt.Errorf("%w: reviewedSubject must be a content strongRef", ErrInvalidRequest) } return nil } @@ -94,7 +94,7 @@ func (s *service) restoreContent(ctx context.Context, actorDID string, request R if request.ReviewedSubject != nil && reviewedURI != removal.SubjectURI { return fail(fmt.Errorf("%w: reviewedSubject URI differs from removal subject", ErrInvalidRequest)) } - comment, err := tx.ReadIndexedComment(ctx, removal.SubjectURI) + subject, err := readIndexedSubject(ctx, tx, removal.SubjectURI) if err != nil && !errors.Is(err, ErrSubjectNotIndexed) { return unavailable(err) } @@ -102,19 +102,19 @@ func (s *service) restoreContent(ctx context.Context, actorDID string, request R var current *StrongRef var observedCID string if err == nil { - if comment == nil { - return unavailable(errors.New("indexed comment lookup returned no comment")) + if subject == nil { + return unavailable(errors.New("indexed subject lookup returned no subject")) } - observedCID = comment.CID - if comment.AuthorDeleted { + observedCID = subject.CID + if subject.AuthorDeleted { recordState = RecordStateDeleted } else { recordState = RecordStatePresent - current = &StrongRef{URI: comment.URI, CID: comment.CID} + current = &StrongRef{URI: subject.URI, CID: subject.CID} if request.ReviewedSubject == nil { - return fail(fmt.Errorf("%w: reviewedSubject is required for a present comment", ErrInvalidRequest)) + return fail(fmt.Errorf("%w: reviewedSubject is required for present content", ErrInvalidRequest)) } - if reviewedCID != comment.CID { + if reviewedCID != subject.CID { return fail(ErrContentChanged) } } diff --git a/internal/core/moderation/store.go b/internal/core/moderation/store.go index 8a5281a..cd49ad0 100644 --- a/internal/core/moderation/store.go +++ b/internal/core/moderation/store.go @@ -57,6 +57,20 @@ type IndexedComment struct { ImageCIDs []string } +// IndexedPost is the indexed post row a mutation inspects, read under a share +// lock so consumer writes serialize against the CID check. +type IndexedPost struct { + URI string + CID string + AuthorDeleted bool + // OwnerDID is the repository holding the post's blobs: the author for + // postv2, the community for legacy posts. + OwnerDID string + CommunityDID string + // BlobCIDs are the canonical CIDs of the post's proxy-served blobs. + BlobCIDs []string +} + // MediaBlock suppresses Coves-served bytes of a blob. An empty OwnerDID // blocks the CID for every owner. type MediaBlock struct { @@ -102,6 +116,8 @@ type Transaction interface { LockSubject(ctx context.Context, subjectURI string) (int64, error) // ReadIndexedComment returns ErrSubjectNotIndexed for a never-indexed URI. ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + // ReadIndexedPost returns ErrSubjectNotIndexed for a never-indexed URI. + ReadIndexedPost(ctx context.Context, subjectURI string) (*IndexedPost, error) // GetAction returns ErrDecisionNotFound for an unknown id. GetAction(ctx context.Context, actionID string) (*Action, error) // ActiveRemoval returns the active removal action, or nil. diff --git a/internal/core/posts/blob_transform.go b/internal/core/posts/blob_transform.go index 3ed2a4c..893c6f1 100644 --- a/internal/core/posts/blob_transform.go +++ b/internal/core/posts/blob_transform.go @@ -115,16 +115,9 @@ func TransformPostEmbeds(ctx context.Context, postView *PostView, blueskyService return } - // Only process app.bsky.feed.post URIs (Bluesky posts) - // Format: at://did:plc:xxx/app.bsky.feed.post/abc123 - if len(atURI) < 20 || atURI[:5] != "at://" { - log.Printf("[DEBUG] [TRANSFORM-EMBED] Skipping: invalid AT-URI format: %s", atURI) - return - } - - // Simple check for app.bsky.feed.post collection - // We don't want to process other types of embeds (e.g., Coves posts) - if !strings.Contains(atURI, "/app.bsky.feed.post/") { + // Only a URI whose collection segment is app.bsky.feed.post is a Bluesky + // post; the substring elsewhere (e.g. in a Coves post's path) is not. + if !embeds.IsBlueskyPostURI(atURI) { log.Printf("[DEBUG] [TRANSFORM-EMBED] Skipping: not a Bluesky post (URI: %s)", atURI) return } diff --git a/internal/core/posts/blob_transform_quote_uri_test.go b/internal/core/posts/blob_transform_quote_uri_test.go new file mode 100644 index 0000000..99cedd9 --- /dev/null +++ b/internal/core/posts/blob_transform_quote_uri_test.go @@ -0,0 +1,54 @@ +package posts + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "Coves/internal/core/blueskypost" +) + +// recordingBlueskyResolver records every URI it is asked to resolve. +type recordingBlueskyResolver struct{ resolved []string } + +func (r *recordingBlueskyResolver) ResolvePost(_ context.Context, uri string) (*blueskypost.BlueskyPostResult, error) { + r.resolved = append(r.resolved, uri) + return &blueskypost.BlueskyPostResult{URI: uri, Text: "resolved"}, nil +} + +func (r *recordingBlueskyResolver) ParseBlueskyURL(context.Context, string) (string, error) { + return "", nil +} + +func (r *recordingBlueskyResolver) IsBlueskyURL(string) bool { return false } + +// Only a URI whose collection segment is app.bsky.feed.post is sent to the +// Bluesky resolver; the substring elsewhere in a URI is not a Bluesky post. +func TestTransformPostEmbeds_ResolvesOnlyBlueskyCollectionURIs(t *testing.T) { + for uri, wantResolved := range map[string]bool{ + "at://did:plc:quoted/app.bsky.feed.post/3kquoted": true, + "at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted": false, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted#/app.bsky.feed.post/1": false, + "at://app.bsky.feed.post/social.coves.community.postv2/3kquoted": false, + } { + t.Run(uri, func(t *testing.T) { + resolver := &recordingBlueskyResolver{} + embed := map[string]interface{}{ + "$type": "social.coves.embed.post", "post": map[string]interface{}{"uri": uri, "cid": "bafyquoted"}, + } + TransformPostEmbeds(t.Context(), &PostView{Embed: embed}, resolver) + + if wantResolved { + assert.Equal(t, []string{uri}, resolver.resolved) + assert.Equal(t, "social.coves.embed.post#view", embed["$type"]) + assert.Contains(t, embed, "resolved") + return + } + assert.Empty(t, resolver.resolved, "a non-Bluesky collection must not reach the Bluesky resolver") + assert.Equal(t, map[string]interface{}{ + "$type": "social.coves.embed.post", "post": map[string]interface{}{"uri": uri, "cid": "bafyquoted"}, + }, embed) + }) + } +} diff --git a/internal/core/posts/interfaces.go b/internal/core/posts/interfaces.go index a2c0dbc..05d3991 100644 --- a/internal/core/posts/interfaces.go +++ b/internal/core/posts/interfaces.go @@ -113,6 +113,8 @@ type Repository interface { // GetRawIndexedRowsByURIs is the batched GetRawIndexedRow: same ungated raw // rows, one round trip. THE SAME DANGER APPLIES — read the banner above // before calling it. URIs with no indexed row are absent from the map. + // Rows also carry community handle/name when available; a missing community + // row does not remove an indexed post from the result. // // It exists because the post.get removal path needs the community and // soft-delete state of every absent URI in a caller-supplied batch, and @@ -120,6 +122,15 @@ type Repository interface { // list the caller controls. GetRawIndexedRowsByURIs(ctx context.Context, uris []string) (map[string]*Post, error) + // ActiveRemovalsByURIs returns the active removal sources of each URI that has any. + ActiveRemovalsByURIs(ctx context.Context, uris []string) (map[string][]RemovalSource, error) + + // AdmittedURIsForViewer returns the URIs whose posts pass the viewer-bound + // admission rule of the read-path visibility predicate, ignoring active + // moderation removals. viewerDID is "" for an anonymous read. post.get uses + // it so a removal tombstone never discloses a post the viewer could not see. + AdmittedURIsForViewer(ctx context.Context, uris []string, viewerDID string) (map[string]bool, error) + // GetViewsByURIs retrieves full post views (with author + community joins) for a // set of canonical DID-based AT-URIs. Returns a map keyed by URI; missing or // soft-deleted posts are simply absent from the map. Backs social.coves.community.post.get. diff --git a/internal/core/posts/post.go b/internal/core/posts/post.go index bd54deb..f22eefe 100644 --- a/internal/core/posts/post.go +++ b/internal/core/posts/post.go @@ -1,6 +1,7 @@ package posts import ( + "encoding/json" "time" ) @@ -34,11 +35,14 @@ type Post struct { RKey string `json:"rkey" db:"rkey"` URI string `json:"uri" db:"uri"` AuthorDID string `json:"authorDid" db:"author_did"` - ID int64 `json:"id" db:"id"` - UpvoteCount int `json:"upvoteCount" db:"upvote_count"` - DownvoteCount int `json:"downvoteCount" db:"downvote_count"` - Score int `json:"score" db:"score"` - CommentCount int `json:"commentCount" db:"comment_count"` + // Set by the batched raw read for content-free moderation tombstones. + CommunityHandle string `json:"-"` + CommunityName string `json:"-"` + ID int64 `json:"id" db:"id"` + UpvoteCount int `json:"upvoteCount" db:"upvote_count"` + DownvoteCount int `json:"downvoteCount" db:"downvote_count"` + Score int `json:"score" db:"score"` + CommentCount int `json:"commentCount" db:"comment_count"` // Bridge-asserted origin-platform vote aggregates for federated/bridged content. // Populated from the record's bridgedStats field; kept separate from native votes. @@ -193,17 +197,61 @@ type RemovedPost struct { Code string `json:"code,omitempty"` } +// RemovalSource attributes one active removal decision on a post. +type RemovalSource struct { + AuthorityDID string + ScopeKind string +} + +// ModerationView is a post's public removal state +// (social.coves.moderation.defs#moderationView). +type ModerationView struct { + State string `json:"state"` + Sources []ModerationSourceView `json:"sources,omitempty"` +} + +// ModerationSourceView attributes a removal (social.coves.moderation.defs#sourceView). +type ModerationSourceView struct { + AuthorityDID string `json:"authorityDid"` + Scope ModerationScopeView `json:"scope"` +} + +// ModerationScopeView is a removal's scope (social.coves.moderation.defs#scopeView). +type ModerationScopeView struct { + Kind string `json:"kind"` +} + +// ModeratedPost is the content-free social.coves.community.post.defs#moderatedPost +// tombstone for a post under an instance-scoped removal. +type ModeratedPost struct { + Moderation *ModerationView `json:"moderation"` + Community *CommunityRef `json:"community,omitempty"` + URI string `json:"uri"` + AuthorDID string `json:"authorDid,omitempty"` +} + +// MarshalJSON identifies this tombstone as the moderatedPost union member. +func (p ModeratedPost) MarshalJSON() ([]byte, error) { + type wirePost ModeratedPost + return json.Marshal(struct { + Type string `json:"$type"` + wirePost + }{Type: "social.coves.community.post.defs#moderatedPost", wirePost: wirePost(p)}) +} + // PostResult is one ordered element of a GetPosts response. Exactly one of Post, -// Blocked, Removed, or NotFound is set: Post when the post was found and visible -// to the viewer, Blocked when the viewer has blocked the author, Removed when the -// post's own community removed it, NotFound when the URI could not be resolved. +// Blocked, Removed, Moderated, or NotFound is set: Post when visible, Blocked +// when the viewer has blocked the author, Removed when the post's own community +// removed it, Moderated when an active instance removal hides it, and NotFound +// when the URI could not be resolved or the author deleted it. // Construct results via the result helpers so the const discriminators // (notFound/blocked/removed == true) cannot be left unset. type PostResult struct { - Post *PostView - Blocked *BlockedPost - Removed *RemovedPost - NotFound *NotFoundPost + Post *PostView + Blocked *BlockedPost + Removed *RemovedPost + NotFound *NotFoundPost + Moderated *ModeratedPost } // foundResult builds a found (postView) union member. @@ -222,6 +270,25 @@ func removedResult(uri, code string) *PostResult { return &PostResult{Removed: &RemovedPost{URI: uri, Removed: true, Code: code}} } +// moderatedResult builds a content-free removal tombstone from indexed metadata. +func moderatedResult(post *Post, sources []RemovalSource) *PostResult { + viewSources := make([]ModerationSourceView, 0, len(sources)) + for _, source := range sources { + viewSources = append(viewSources, ModerationSourceView{ + AuthorityDID: source.AuthorityDID, + Scope: ModerationScopeView{Kind: source.ScopeKind}, + }) + } + result := &ModeratedPost{ + URI: post.URI, AuthorDID: post.AuthorDID, + Moderation: &ModerationView{State: "removed", Sources: viewSources}, + } + if post.CommunityDID != "" && post.CommunityName != "" { + result.Community = &CommunityRef{DID: post.CommunityDID, Handle: post.CommunityHandle, Name: post.CommunityName} + } + return &PostResult{Moderated: result} +} + // blockedByAuthorResult builds a blockedPost union member (blockedBy "author") with its // const discriminator set. func blockedByAuthorResult(uri, authorDID string) *PostResult { @@ -233,10 +300,9 @@ func blockedByAuthorResult(uri, authorDID string) *PostResult { }} } -// GetPost returns the underlying PostView (nil for blocked/not-found results), -// satisfying the viewer-state enrichment helper's FeedPostProvider interface. Blocked -// and not-found results carry no PostView, so they are skipped by viewer enrichment -// and embed transforms. +// GetPost returns the underlying PostView (nil for tombstones), satisfying the +// viewer-state enrichment helper's FeedPostProvider interface. Content-free +// results are skipped by viewer enrichment and embed transforms. func (r *PostResult) GetPost() *PostView { return r.Post } @@ -247,7 +313,7 @@ func (r *PostResult) GetPost() *PostView { // an assembly bug; reporting it (rather than silently picking one by priority) is the // whole point of this guard, so a mis-assembled result surfaces as an internal error // instead of emitting a null or ambiguous array entry that violates the lexicon's union -// (postView | blockedPost | notFoundPost). +// (postView | blockedPost | removedPost | moderatedPost | notFoundPost). func (r *PostResult) Member() (interface{}, bool) { var member interface{} count := 0 @@ -263,6 +329,10 @@ func (r *PostResult) Member() (interface{}, bool) { member = r.Removed count++ } + if r.Moderated != nil { + member = r.Moderated + count++ + } if r.NotFound != nil { member = r.NotFound count++ diff --git a/internal/core/posts/service.go b/internal/core/posts/service.go index e15652b..bfa5f9b 100644 --- a/internal/core/posts/service.go +++ b/internal/core/posts/service.go @@ -1307,7 +1307,7 @@ const MaxGetPostsURIs = 25 // 1. Validate the URI count (1..25) and that every URI is a well-formed DID-based URI. // A malformed or handle-based URI is a client error -> InvalidRequest, not a silent miss. // 2. Batch fetch views for the (deduped) URIs. -// 3. Assemble results in request order; valid-but-absent URIs become notFoundPost. +// 3. Resolve removal tombstones for absent URIs, then assemble in request order. // // Viewer state (vote) and embed/blob transforms are applied by the handler layer. func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*PostResult, error) { @@ -1347,13 +1347,36 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos return nil, fmt.Errorf("failed to fetch post views: %w", err) } - // 3. Assemble results in request order. A visible view is a postView; an - // absent URI is a notFoundPost — UNLESS its own community removed it, in - // which case it becomes a #removedPost tombstone carrying the removal code - // (PRD §3.4/§6.2). The visibility predicate hides a removed post from - // GetViewsByURIs exactly as it hides a pending one, so the removal is - // recovered here from the admission row rather than from the (absent) view. - removed, err := s.removedMarkers(ctx, req.URIs, views) + // 3. Resolve absent URIs in batches. Raw rows establish that a post was + // indexed and has not been withdrawn by its author; neither the raw content + // nor the admission row can override an active instance removal. + absent := make([]string, 0, len(unique)) + for _, uri := range unique { + if views[uri] == nil { + absent = append(absent, uri) + } + } + var postsByURI map[string]*Post + var removals map[string][]RemovalSource + var admitted map[string]bool + if len(absent) != 0 { + postsByURI, err = s.repo.GetRawIndexedRowsByURIs(ctx, absent) + if err != nil { + return nil, fmt.Errorf("failed to resolve removal state for post.get: %w", err) + } + removals, err = s.repo.ActiveRemovalsByURIs(ctx, absent) + if err != nil { + return nil, fmt.Errorf("failed to fetch active post removals: %w", err) + } + // An instance removal must not widen access: the #moderatedPost + // tombstone discloses the author and the community, so it is served + // only to a viewer the admission rule would have shown the post to. + admitted, err = s.repo.AdmittedURIsForViewer(ctx, absent, req.ViewerDID) + if err != nil { + return nil, fmt.Errorf("failed to resolve post admission for post.get: %w", err) + } + } + removed, err := s.removedMarkers(ctx, absent, postsByURI) if err != nil { return nil, err } @@ -1362,6 +1385,11 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos switch { case views[uri] != nil: results[i] = foundResult(views[uri]) + case postsByURI[uri] != nil && postsByURI[uri].DeletedAt == nil && len(removals[uri]) != 0 && (admitted[uri] || hasRemovedMarker(removed, uri)): + // A same-community removal marker already makes the post and its + // community public as #removedPost, so the instance tombstone may + // replace it without disclosing anything new. + results[i] = moderatedResult(postsByURI[uri], removals[uri]) default: if code, ok := removed[uri]; ok { results[i] = removedResult(uri, code) @@ -1383,8 +1411,15 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos return results, nil } -// removedMarkers returns, for the requested URIs absent from the visible view -// set, the removal code of any whose OWN community removed it — so post.get can +// hasRemovedMarker reports whether removedMarkers produced a same-community +// removal marker for uri. +func hasRemovedMarker(markers map[string]string, uri string) bool { + _, ok := markers[uri] + return ok +} + +// removedMarkers returns, for the absent URIs with indexed rows, the removal +// code of any whose OWN community removed it — so post.get can // serve a #removedPost tombstone (PRD §3.4) instead of collapsing a moderator // removal into an indistinguishable notFoundPost. The presence of a URI in the // returned map is the removed signal; the value is the code (possibly empty). @@ -1394,7 +1429,7 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos // That is a CONFIGURATION fact, known before any lookup runs, and it is the only // thing that silently degrades to notFound. // -// A LOOKUP FAILURE IS AN ERROR, NOT A NOTFOUND. Both lookups here used to be +// A LOOKUP FAILURE IS AN ERROR, NOT A NOTFOUND. The lookups used to be // best-effort: a database blip turned a standing removal into notFoundPost, so // the same request answered with a different union member depending on the // health of the database, and a client (or a moderator checking their own @@ -1402,27 +1437,9 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos // out". post.get answering 5xx is the honest response to "we do not know"; // silently downgrading the tombstone is not, and it is unfalsifiable from the // wire. Callers propagate the error. -func (s *postService) removedMarkers(ctx context.Context, uris []string, views map[string]*PostView) (map[string]string, error) { +func (s *postService) removedMarkers(ctx context.Context, absent []string, postsByURI map[string]*Post) (map[string]string, error) { markers := make(map[string]string) - if s.admissions == nil { - return markers, nil - } - - // Collect the absent URIs once (deduped), then resolve their admissions in a - // single batched lookup rather than one round-trip per URI. - seen := make(map[string]struct{}, len(uris)) - absent := make([]string, 0, len(uris)) - for _, uri := range uris { - if views[uri] != nil { - continue // visible — not a candidate for a tombstone - } - if _, done := seen[uri]; done { - continue - } - seen[uri] = struct{}{} - absent = append(absent, uri) - } - if len(absent) == 0 { + if s.admissions == nil || len(absent) == 0 { return markers, nil } @@ -1431,21 +1448,6 @@ func (s *postService) removedMarkers(ctx context.Context, uris []string, views m return nil, fmt.Errorf("failed to resolve removal state for post.get: %w", err) } - // The post rows are fetched in ONE batched round trip. Looping a per-URI - // lookup here was an N+1 on a public endpoint whose URI list the caller - // controls: 25 URIs (MaxGetPostsURIs) meant up to 25 sequential queries per - // request, all of them for URIs the visibility predicate had already refused. - // - // These are RAW rows on purpose — the predicate has already hidden every URI - // in `absent`, so a gated read would return nothing and there would be no - // removal to report. The raw row is used for exactly two facts, both checked - // below and neither of them content: which community owns the post, and - // whether its author withdrew it. - postsByURI, err := s.repo.GetRawIndexedRowsByURIs(ctx, absent) - if err != nil { - return nil, fmt.Errorf("failed to resolve removal state for post.get: %w", err) - } - for _, uri := range absent { // A removal is an admission-state change, not a soft delete, so the post // row still stands and its own community — the key the admission is scoped diff --git a/internal/core/posts/service_author_posts_test.go b/internal/core/posts/service_author_posts_test.go index 1a63063..1a95ab2 100644 --- a/internal/core/posts/service_author_posts_test.go +++ b/internal/core/posts/service_author_posts_test.go @@ -51,6 +51,14 @@ func (m *mockRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []str return out, nil } +func (m *mockRepository) ActiveRemovalsByURIs(context.Context, []string) (map[string][]RemovalSource, error) { + return map[string][]RemovalSource{}, nil +} + +func (m *mockRepository) AdmittedURIsForViewer(context.Context, []string, string) (map[string]bool, error) { + return map[string]bool{}, nil +} + func (m *mockRepository) GetViewsByURIs(ctx context.Context, uris []string, viewerDID string) (map[string]*PostView, error) { m.getViewsByURIsCalls++ m.gotViewsViewerDID = viewerDID diff --git a/internal/db/postgres/comment_repo.go b/internal/db/postgres/comment_repo.go index ede49fe..401a8d2 100644 --- a/internal/db/postgres/comment_repo.go +++ b/internal/db/postgres/comment_repo.go @@ -459,6 +459,7 @@ func (r *postgresCommentRepo) ListByCommenter(ctx context.Context, commenterDID // Used for user profile comment history (social.coves.actor.getComments) // Supports optional community filtering and returns next page cursor // Uses chronological ordering (newest first) with composite key cursor for stable pagination +// Excludes comments removed directly or under a removed root before pagination. func (r *postgresCommentRepo) ListByCommenterWithCursor(ctx context.Context, req comments.ListByCommenterRequest) ([]*comments.Comment, *string, error) { // Parse cursor for pagination cursorFilter, cursorValues, err := r.parseCommenterCursor(req.Cursor) @@ -523,7 +524,7 @@ func (r *postgresCommentRepo) ListByCommenterWithCursor(ctx context.Context, req AND c.deleted_at IS NULL AND NOT EXISTS ( SELECT 1 FROM moderation_decisions d - WHERE d.subject_uri = c.uri AND d.kind = 'removal' AND d.active + WHERE d.subject_uri IN (c.uri, c.root_uri) AND d.kind = 'removal' AND d.active ) %s %s diff --git a/internal/db/postgres/discover_hot_moderation_race_test.go b/internal/db/postgres/discover_hot_moderation_race_test.go new file mode 100644 index 0000000..199919a --- /dev/null +++ b/internal/db/postgres/discover_hot_moderation_race_test.go @@ -0,0 +1,143 @@ +//go:build integration + +package postgres + +import ( + "context" + "database/sql" + "testing" + "time" + + "Coves/internal/core/discover" + "Coves/tests/testkit" + + "github.com/stretchr/testify/require" +) + +// insertDiscoverHotRemoval uses the same action/decision shape as +// moderationTransaction.SetRemovalDecision. It avoids reading the posts view +// while the hydration-boundary gate holds its advisory lock. +func insertDiscoverHotRemoval(t *testing.T, db *sql.DB, uri string) { + t.Helper() + actionID := "hot-removal-" + testkit.UniqueID(t) + _, err := db.ExecContext(t.Context(), ` + INSERT INTO moderation_actions + (id, actor_did, authority_did, scope_kind, subject_uri, subject_collection, + action, reason, origin, created_at) + VALUES ($1, 'did:plc:hotmoderator', 'did:plc:hotinstance', 'instance', $2, + 'social.coves.community.post', 'remove', + 'social.coves.moderation.defs#reasonSpam', 'local', NOW()) + `, actionID, uri) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO moderation_decisions + (authority_did, scope_kind, scope_community_did, subject_uri, kind, value, active_action_id, active) + VALUES ('did:plc:hotinstance', 'instance', NULL, $2, 'removal', NULL, $1, TRUE) + `, actionID, uri) + require.NoError(t, err) +} + +func TestDiscoverRepo_HotRefillsAfterInstanceRemovalAtHydrationBoundary(t *testing.T) { + for _, timing := range []string{"before-revalidation", "after-revalidation-before-hydration"} { + t.Run(timing, func(t *testing.T) { + db := testkit.DB(t) + ctx := t.Context() + const ( + authorDID = "did:plc:hotmoderationauthor" + lockClass = 1_126_644_054 + lockID = 1 + ) + createTestUser(t, db, "hot-moderation-author.test", authorDID) + communities := []string{"did:plc:hotmoderationa", "did:plc:hotmoderationb", "did:plc:hotmoderationc"} + for i, did := range communities { + createTestCommunity(t, db, did, "hot-moderation-"+string(rune('a'+i))+".test", authorDID) + } + createdAt := time.Now().Add(-4 * time.Hour).Truncate(time.Second) + seed := func(communityDID, rkey string, score int) string { + t.Helper() + uri := seedFilterablePost(t, db, communityDID, authorDID, rkey, createdAt) + _, err := db.ExecContext(ctx, `UPDATE posts SET score = $2, upvote_count = $2 WHERE uri = $1`, uri, score) + require.NoError(t, err) + return uri + } + a1 := seed(communities[0], "hotmoda1", 1_000_000_000) + removedA2 := seed(communities[0], "hotmoda2", 1_000_000) + a3 := seed(communities[0], "hotmoda3", 2_000) + b1 := seed(communities[1], "hotmodb1", 5_000) + c1 := seed(communities[2], "hotmodc1", 100) + + repo := NewDiscoverRepository(db, "hot-moderation-race-secret").(*postgresDiscoverRepo) + repo.discoverHotWorkDeadline = 10 * time.Second + first, cursor, err := repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 1}) + require.NoError(t, err) + require.Equal(t, []string{a1}, discoverURIs(first)) + require.NotNil(t, cursor) + + type pageResult struct { + feed []*discover.FeedViewPost + cursor *string + err error + } + var second pageResult + if timing == "before-revalidation" { + insertDiscoverHotRemoval(t, db, removedA2) + second.feed, second.cursor, second.err = repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 2, Cursor: cursor}) + } else { + _, err := db.ExecContext(ctx, ` + CREATE TABLE hot_moderation_gate (candidate_uri TEXT PRIMARY KEY, lock_class INTEGER, lock_id INTEGER); + CREATE FUNCTION hot_moderation_wait(candidate_uri TEXT) RETURNS BOOLEAN + LANGUAGE plpgsql VOLATILE AS $$ + DECLARE gate hot_moderation_gate%ROWTYPE; + BEGIN + SELECT * INTO gate FROM hot_moderation_gate WHERE hot_moderation_gate.candidate_uri = $1; + IF FOUND THEN PERFORM pg_advisory_xact_lock(gate.lock_class, gate.lock_id); END IF; + RETURN TRUE; + END; + $$; + ALTER TABLE posts RENAME TO hot_moderation_posts; + CREATE VIEW posts WITH (security_barrier = true) AS + SELECT stored.* FROM hot_moderation_posts stored WHERE hot_moderation_wait(stored.uri); + `) + require.NoError(t, err) + _, err = db.ExecContext(ctx, `INSERT INTO hot_moderation_gate VALUES ($1, $2, $3)`, removedA2, lockClass, lockID) + require.NoError(t, err) + lockConnection, err := db.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lockConnection.Close()) }) + _, err = lockConnection.ExecContext(ctx, `SELECT pg_advisory_lock($1, $2)`, lockClass, lockID) + require.NoError(t, err) + locked := true + t.Cleanup(func() { + if locked { + _, unlockErr := lockConnection.ExecContext(context.Background(), `SELECT pg_advisory_unlock($1, $2)`, lockClass, lockID) + require.NoError(t, unlockErr) + } + }) + var holderPID int + require.NoError(t, lockConnection.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&holderPID)) + results := make(chan pageResult, 1) + go func() { + feed, next, requestErr := repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 2, Cursor: cursor}) + results <- pageResult{feed: feed, cursor: next, err: requestErr} + }() + waitForDiscoverHotAdvisoryLock(t, db, holderPID) + insertDiscoverHotRemoval(t, db, removedA2) + _, err = lockConnection.ExecContext(ctx, `SELECT pg_advisory_unlock($1, $2)`, lockClass, lockID) + require.NoError(t, err) + locked = false + second = <-results + } + + require.NoError(t, second.err) + require.Equal(t, []string{b1, a3}, discoverURIs(second.feed), "removal must refill without penalizing A's diversity slot") + require.NotNil(t, second.cursor) + third, end, err := repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 2, Cursor: second.cursor}) + require.NoError(t, err) + require.Equal(t, []string{c1}, discoverURIs(third)) + require.Nil(t, end) + all := append(append(discoverURIs(first), discoverURIs(second.feed)...), discoverURIs(third)...) + require.Equal(t, []string{a1, b1, a3, c1}, all, "every eligible snapshot candidate exactly once in checkpoint order") + require.NotContains(t, all, removedA2) + }) + } +} diff --git a/internal/db/postgres/moderation_post_integration_test.go b/internal/db/postgres/moderation_post_integration_test.go new file mode 100644 index 0000000..0e74646 --- /dev/null +++ b/internal/db/postgres/moderation_post_integration_test.go @@ -0,0 +1,359 @@ +//go:build integration + +package postgres_test + +import ( + "database/sql" + "fmt" + "testing" + + "Coves/internal/core/communityFeeds" + "Coves/internal/core/discover" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/core/timeline" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const moderationPostCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + +func indexedModerationPost(t *testing.T, db *sql.DB, collection, communityDID, authorDID, title, embed string) moderation.StrongRef { + t.Helper() + rkey := testkit.TID() + authority := authorDID + if collection == moderation.LegacyPostCollection { + authority = communityDID + } + subject := moderation.StrongRef{URI: "at://" + authority + "/" + collection + "/" + rkey, CID: moderationPostCID} + _, err := db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, $6, 'indexed post content', NULLIF($7, '')::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, authorDID, communityDID, title, embed) + require.NoError(t, err) + if collection == moderation.PostV2Collection { + _, err = db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, last_community_rev, last_community_op_rank, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, '3lqqqqqqqqqq2', 1, NOW(), NOW()) + `, communityDID, subject.URI, subject.CID) + require.NoError(t, err) + } + return subject +} + +func moderationPostActors(t *testing.T, db *sql.DB) (communityDID, authorDID string) { + t.Helper() + authorName := testkit.UniqueIDWithPrefix(t, "postowner") + authorDID = fixtures.DID(authorName) + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "postplace") + var err error + communityDID, err = fixtures.Community(t.Context(), db, communityName, "owner"+communityName) + require.NoError(t, err) + return communityDID, authorDID +} + +func removeIndexedModerationPost(t *testing.T, service moderation.Service, subject moderation.StrongRef) *moderation.MutationResult { + t.Helper() + result, err := service.RemoveContent(t.Context(), fixtures.DID("postremovaladmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-post", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err, "an indexed post must be readable inside the moderation transaction") + require.NotNil(t, result) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + return result +} + +func TestModerationPostRepositoryRemovalPersistence(t *testing.T) { + for _, scenario := range []struct { + name, collection, embed string + softDeleted bool + requestedCID string + wantCIDs []string + communityOwned bool + }{ + {name: "postv2 two images", collection: moderation.PostV2Collection, embed: moderationTwoImageEmbed(), wantCIDs: []string{moderationImageCIDOne, moderationImageCIDTwo}}, + {name: "legacy image belongs to community", collection: moderation.LegacyPostCollection, embed: fmt.Sprintf(`{"$type":"social.coves.embed.images","images":[{"image":{"ref":{"$link":"%s"}}}]}`, moderationImageCIDOne), wantCIDs: []string{moderationImageCIDOne}, communityOwned: true}, + {name: "postv2 without media", collection: moderation.PostV2Collection}, + {name: "author deleted postv2 with stale requested CID", collection: moderation.PostV2Collection, softDeleted: true, requestedCID: moderationImageCIDTwo}, + } { + t.Run(scenario.name, func(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + subject := indexedModerationPost(t, db, scenario.collection, communityDID, authorDID, "post to moderate", scenario.embed) + if scenario.softDeleted { + _, err := db.ExecContext(t.Context(), `UPDATE posts SET deleted_at = NOW() WHERE uri = $1`, subject.URI) + require.NoError(t, err) + } + requested := subject + if scenario.requestedCID != "" { + requested.CID = scenario.requestedCID + } + service := newPostgresModerationService(db) + removed := removeIndexedModerationPost(t, service, requested) + expectedState := moderation.RecordStatePresent + if scenario.softDeleted { + expectedState = moderation.RecordStateDeleted + } + assert.Equal(t, expectedState, removed.State.RecordState) + var collection, associatedCommunity, observedCID string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT subject_collection, subject_community_did, observed_cid + FROM moderation_actions WHERE id = $1 AND subject_uri = $2 + `, removed.Action.ID, subject.URI).Scan(&collection, &associatedCommunity, &observedCID)) + assert.Equal(t, scenario.collection, collection) + assert.Equal(t, communityDID, associatedCommunity) + assert.Equal(t, subject.CID, observedCID, "the indexed CID, even when the deleted subject was requested with a different CID") + var decisionCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_decisions WHERE subject_uri = $1 AND active`, subject.URI).Scan(&decisionCount)) + assert.Equal(t, 1, decisionCount) + var activeAction string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT active_action_id FROM moderation_decisions + WHERE subject_uri = $1 AND authority_did = $2 AND kind = 'removal' AND active + `, subject.URI, fixtures.InstanceDID()).Scan(&activeAction)) + assert.Equal(t, removed.Action.ID, activeAction) + + rows, err := db.QueryContext(t.Context(), `SELECT owner_did, blob_cid FROM moderation_media_blocks WHERE action_id = $1 AND active`, removed.Action.ID) + require.NoError(t, err) + blocks := map[string][]string{} + for rows.Next() { + var owner sql.NullString + var cid string + require.NoError(t, rows.Scan(&owner, &cid)) + require.True(t, owner.Valid, "spam must not create ownerless media blocks") + blocks[owner.String] = append(blocks[owner.String], cid) + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + owner := authorDID + if scenario.communityOwned { + owner = communityDID + } + if len(scenario.wantCIDs) == 0 { + assert.Empty(t, blocks) + } else { + require.Len(t, blocks, 1, "only the correct blob owner may be blocked") + assert.ElementsMatch(t, scenario.wantCIDs, blocks[owner]) + } + var originalContent, storedCID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT content, cid FROM posts WHERE uri = $1`, subject.URI).Scan(&originalContent, &storedCID)) + assert.Equal(t, "indexed post content", originalContent, "instance removal is an overlay") + assert.Equal(t, subject.CID, storedCID) + + state, err := service.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, expectedState, state.RecordState) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + }) + } +} + +func TestModerationPostRepositoryNeverIndexed(t *testing.T) { + db := testkit.DB(t) + service := newPostgresModerationService(db) + missing := moderation.StrongRef{URI: "at://" + fixtures.DID("neverindexedpost") + "/" + moderation.PostV2Collection + "/" + testkit.TID(), CID: moderationPostCID} + result, err := service.RemoveContent(t.Context(), fixtures.DID("postremovaladmin"), moderation.RemoveContentRequest{ + Subject: missing, ExpectedVersion: "v0", IdempotencyKey: "missing-post", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + assert.ErrorIs(t, err, moderation.ErrSubjectNotFound, "a post URI never indexed must not be moderated") + assert.Nil(t, result) + for _, table := range []string{"moderation_actions", "moderation_decisions", "moderation_media_blocks", "moderation_idempotency_keys"} { + var count int + require.NoError(t, db.QueryRowContext(t.Context(), "SELECT count(*) FROM "+table).Scan(&count)) + assert.Zero(t, count, table) + } +} + +func TestModerationPostRepositoryReconcilesEditedImagesInTransaction(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "edited moderated post", moderationTwoImageEmbed()) + removed := removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + tx, err := db.BeginTx(t.Context(), nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + digest, err := multihash.Sum([]byte("moderation edited post third image"), multihash.SHA2_256, -1) + require.NoError(t, err) + thirdCID := cid.NewCidV1(cid.Raw, digest).String() + thirdImage := fmt.Sprintf(`{"$type":"social.coves.embed.images","images":[{"image":{"ref":{"$link":"%s"}}},{"image":{"ref":{"$link":"%s"}}},{"image":{"ref":{"$link":"%s"}}}]}`, moderationImageCIDOne, moderationImageCIDTwo, thirdCID) + _, err = tx.ExecContext(t.Context(), `UPDATE posts SET embed = $1::jsonb WHERE uri = $2`, thirdImage, subject.URI) + require.NoError(t, err) + reconciler := moderation.NewMediaReconciler(postgres.NewModerationRepository(db), fixtures.InstanceDID(), nil) + blocks, err := reconciler.ReconcileTx(t.Context(), tx, subject.URI) + require.NoError(t, err, "the consumer's post media seam must read the edited post inside its transaction") + require.Len(t, blocks, 1) + assert.Equal(t, authorDID, blocks[0].OwnerDID) + assert.Equal(t, thirdCID, blocks[0].BlobCID) + require.NoError(t, tx.Commit()) + var count int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, authorDID, thirdCID).Scan(&count)) + assert.Equal(t, 1, count) +} + +func TestModerationPostRepositoryRemovalVisibilityAndRestore(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + viewerName := testkit.UniqueIDWithPrefix(t, "postviewer") + viewerDID := fixtures.DID(viewerName) + fixtures.User(t, db, viewerName+".test", viewerDID) + _, err := db.ExecContext(t.Context(), `INSERT INTO community_subscriptions (user_did, community_did, subscribed_at) VALUES ($1, $2, NOW())`, viewerDID, communityDID) + require.NoError(t, err) + title := "moderation visibility signal" + removed := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, title, "") + control := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, title, "") + postRepo := postgres.NewPostRepository(db) + feedRepo := postgres.NewCommunityFeedRepository(db, "moderation-post-visibility-secret") + discoverRepo := postgres.NewDiscoverRepository(db, "moderation-post-visibility-secret") + timelineRepo := postgres.NewTimelineRepository(db, "moderation-post-visibility-secret") + service := newPostgresModerationService(db) + + check := func(stage string, want []string, count int) { + t.Helper() + checks := []struct { + name string + read func(*testing.T) []string + }{ + {"post.get anonymous", func(t *testing.T) []string { + views, err := postRepo.GetViewsByURIs(t.Context(), []string{removed.URI, control.URI}, "") + require.NoError(t, err) + return moderationPostViewURIs(views) + }}, + {"post.get author", func(t *testing.T) []string { + views, err := postRepo.GetViewsByURIs(t.Context(), []string{removed.URI, control.URI}, authorDID) + require.NoError(t, err) + return moderationPostViewURIs(views) + }}, + {"comment header anonymous", func(t *testing.T) []string { return moderationPostHeaderURI(t, postRepo, removed.URI, control.URI, "") }}, + {"comment header author", func(t *testing.T) []string { + return moderationPostHeaderURI(t, postRepo, removed.URI, control.URI, authorDID) + }}, + {"actor.getPosts anonymous", func(t *testing.T) []string { + views, _, err := postRepo.GetByAuthor(t.Context(), posts.GetAuthorPostsRequest{ActorDID: authorDID, Limit: 50}) + require.NoError(t, err) + var uris []string + for _, view := range views { + uris = append(uris, view.URI) + } + return uris + }}, + {"actor.getPosts author", func(t *testing.T) []string { + views, _, err := postRepo.GetByAuthor(t.Context(), posts.GetAuthorPostsRequest{ActorDID: authorDID, ViewerDID: authorDID, Limit: 50}) + require.NoError(t, err) + var uris []string + for _, view := range views { + uris = append(uris, view.URI) + } + return uris + }}, + {"timeline subscriber", func(t *testing.T) []string { + feed, _, err := timelineRepo.GetTimeline(t.Context(), timeline.GetTimelineRequest{UserDID: viewerDID, Sort: "new", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}, + {"search matching title", func(t *testing.T) []string { + feed, _, err := feedRepo.SearchPosts(t.Context(), communityFeeds.SearchPostsRequest{Query: "moderation visibility", Community: communityDID, Sort: "relevance", Timeframe: "all", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}, + } + for _, sort := range []string{"hot", "new", "top"} { + checks = append(checks, struct { + name string + read func(*testing.T) []string + }{"community feed " + sort, func(t *testing.T) []string { + feed, _, err := feedRepo.GetCommunityFeed(t.Context(), communityFeeds.GetCommunityFeedRequest{Community: communityDID, Sort: sort, Timeframe: "all", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}) + } + for _, sort := range []string{"new", "top"} { + checks = append(checks, struct { + name string + read func(*testing.T) []string + }{"discover " + sort, func(t *testing.T) []string { + feed, _, err := discoverRepo.GetDiscover(t.Context(), discover.GetDiscoverRequest{Sort: sort, Timeframe: "all", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}) + } + for _, check := range checks { + t.Run(stage+"/"+check.name, func(t *testing.T) { + assert.ElementsMatch(t, want, check.read(t), "instance removal must hide the post from this read path, including its author") + }) + } + t.Run(stage+"/community postCount", func(t *testing.T) { + community, err := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()).GetByDID(t.Context(), communityDID) + require.NoError(t, err) + assert.Equal(t, count, community.PostCount) + }) + t.Run(stage+"/profile post count", func(t *testing.T) { + stats, err := postgres.NewUserRepository(db).GetProfileStats(t.Context(), authorDID) + require.NoError(t, err) + assert.Equal(t, count, stats.PostCount) + }) + } + check("before removal", []string{removed.URI, control.URI}, 2) + result := removeIndexedModerationPost(t, service, removed) + check("while removed", []string{control.URI}, 1) + restored, err := service.RestoreContent(t.Context(), fixtures.DID("postrestoreadmin"), moderation.RestoreContentRequest{ + ActionID: result.Action.ID, ReviewedSubject: &removed, ExpectedVersion: result.State.Version, + IdempotencyKey: "restore-post", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + check("after restore", []string{removed.URI, control.URI}, 2) +} + +func moderationPostViewURIs(views map[string]*posts.PostView) []string { + var uris []string + for uri := range views { + uris = append(uris, uri) + } + return uris +} + +func moderationPostHeaderURI(t *testing.T, repo *postgres.PostRepository, removed, control, viewer string) []string { + t.Helper() + var uris []string + for _, uri := range []string{removed, control} { + view, err := repo.VisibleHeaderView(t.Context(), uri, viewer) + require.NoError(t, err) + if view != nil { + uris = append(uris, view.URI) + } + } + return uris +} diff --git a/internal/db/postgres/moderation_post_tombstone_integration_test.go b/internal/db/postgres/moderation_post_tombstone_integration_test.go new file mode 100644 index 0000000..0387efd --- /dev/null +++ b/internal/db/postgres/moderation_post_tombstone_integration_test.go @@ -0,0 +1,269 @@ +//go:build integration + +package postgres_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "testing" + + "Coves/internal/core/communities" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func moderationPostGetService(db *sql.DB, repo posts.Repository) posts.Service { + communityRepo := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()) + communityService := communities.NewCommunityServiceWithPDSFactory( + communityRepo, testkit.Endpoints().PDS.BaseURL, fixtures.InstanceDID(), "", nil, nil, nil, + communities.PrivateHostOptions(true)..., + ) + return posts.NewPostService(repo, communityService, nil, nil, nil, nil, testkit.Endpoints().PDS.BaseURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(postgres.NewAdmissionRepository(db), nil), + ) +} + +func moderationPostTombstoneActors(t *testing.T, db *sql.DB) (communityDID, authorDID string) { + t.Helper() + communityDID, _ = moderationPostActors(t, db) + authorDID = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" + fixtures.User(t, db, "posttombstoneauthor.test", authorDID) + return communityDID, authorDID +} + +func restoreModerationPost(t *testing.T, service moderation.Service, subject moderation.StrongRef, removed *moderation.MutationResult) { + t.Helper() + restored, err := service.RestoreContent(t.Context(), fixtures.DID("postrestoreadmin"), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: removed.State.Version, + IdempotencyKey: "restore-post", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) +} + +func TestModerationPostActiveRemovalsByURIs(t *testing.T) { + db := testkit.DB(t) + repo := postgres.NewPostRepository(db) + communityDID, authorDID := moderationPostActors(t, db) + active := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "active decision", "") + restored := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "restored decision", "") + control := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "never removed", "") + service := newPostgresModerationService(db) + removeIndexedModerationPost(t, service, active) + previous, err := service.RemoveContent(t.Context(), fixtures.DID("postremovaladminb"), moderation.RemoveContentRequest{ + Subject: restored, ExpectedVersion: "v0", IdempotencyKey: "remove-restored-post", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err) + require.NotNil(t, previous) + require.NotNil(t, previous.Action) + restoreModerationPost(t, service, restored, previous) + + got, err := repo.ActiveRemovalsByURIs(t.Context(), []string{control.URI, active.URI, restored.URI, active.URI}) + require.NoError(t, err) + assert.Equal(t, map[string][]posts.RemovalSource{ + active.URI: {{AuthorityDID: fixtures.InstanceDID(), ScopeKind: "instance"}}, + }, got, "only active decisions may appear, even in a batch with restored and unremoved URIs") + empty, err := repo.ActiveRemovalsByURIs(t.Context(), nil) + require.NoError(t, err) + assert.NotNil(t, empty, "an empty lookup must return an empty map") + assert.Empty(t, empty) +} + +func TestModerationPostGetTombstone(t *testing.T) { + for _, scenario := range []struct { + name string + communityRemoved bool + deleted bool + }{ + {name: "accepted post"}, + {name: "instance removal overrides community removal", communityRemoved: true}, + {name: "author deletion overrides instance removal", deleted: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "private after removal", "") + if scenario.communityRemoved { + _, err := db.ExecContext(t.Context(), ` + UPDATE community_post_admissions SET status = 'removed', accepted_cid = NULL, + decision_code = 'rule-violation', decision_at = NOW(), updated_at = NOW() + WHERE community_did = $1 AND post_uri = $2 + `, communityDID, subject.URI) + require.NoError(t, err) + } + if scenario.deleted { + _, err := db.ExecContext(t.Context(), `UPDATE posts SET deleted_at = NOW() WHERE uri = $1`, subject.URI) + require.NoError(t, err) + } + moderationService := newPostgresModerationService(db) + postService := moderationPostGetService(db, postgres.NewPostRepository(db)) + if scenario.communityRemoved { + before, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + require.NoError(t, err) + require.Len(t, before, 1) + require.NotNil(t, before[0].Removed, "the community-removal fixture must serve #removedPost before instance removal") + } + removed := removeIndexedModerationPost(t, moderationService, subject) + for _, viewer := range []struct{ name, did string }{{"anonymous", ""}, {"author", authorDID}} { + t.Run(viewer.name, func(t *testing.T) { + results, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}, ViewerDID: viewer.did}) + require.NoError(t, err) + require.Len(t, results, 1) + if scenario.deleted { + require.NotNil(t, results[0].NotFound, "author deletion takes precedence over instance moderation") + assert.Nil(t, results[0].Moderated) + return + } + require.NotNil(t, results[0].Moderated, "instance removal must serve #moderatedPost instead of #notFoundPost or #removedPost") + assert.Nil(t, results[0].Post) + assert.Nil(t, results[0].Removed) + assert.Nil(t, results[0].NotFound) + assert.Nil(t, results[0].Blocked) + member, ok := results[0].Member() + assert.True(t, ok) + assert.Same(t, results[0].Moderated, member) + tombstone := results[0].Moderated + assert.Equal(t, subject.URI, tombstone.URI) + assert.Equal(t, authorDID, tombstone.AuthorDID) + require.NotNil(t, tombstone.Moderation) + assert.Equal(t, "removed", tombstone.Moderation.State) + require.Len(t, tombstone.Moderation.Sources, 1) + assert.Equal(t, fixtures.InstanceDID(), tombstone.Moderation.Sources[0].AuthorityDID) + assert.Equal(t, "instance", tombstone.Moderation.Sources[0].Scope.Kind) + require.NotNil(t, tombstone.Community) + assert.Equal(t, communityDID, tombstone.Community.DID) + var name, handle string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT name, handle FROM communities WHERE did = $1`, communityDID).Scan(&name, &handle)) + assert.Equal(t, name, tombstone.Community.Name) + assert.Equal(t, handle, tombstone.Community.Handle) + assert.Nil(t, tombstone.Community.Avatar) + assert.Nil(t, tombstone.Community.Origin) + assert.Empty(t, tombstone.Community.PDSURL) + }) + } + if scenario.communityRemoved { + restoreModerationPost(t, moderationService, subject, removed) + after, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + require.NoError(t, err) + require.Len(t, after, 1) + require.NotNil(t, after[0].Removed, "restoring the instance removal must reveal the community's #removedPost again") + assert.Equal(t, "rule-violation", after[0].Removed.Code) + assert.Nil(t, after[0].Moderated) + } + }) + } +} + +func TestModerationPostGetBatchPreservesOrder(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + uris := make([]string, 25) + var subject moderation.StrongRef + for i := range uris { + post := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, fmt.Sprintf("post %d", i), "") + uris[i] = post.URI + if i == 12 { + subject = post + } + } + removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + results, err := moderationPostGetService(db, postgres.NewPostRepository(db)).GetPosts(t.Context(), posts.GetPostsRequest{URIs: uris}) + require.NoError(t, err) + require.Len(t, results, len(uris)) + for i, result := range results { + if i == 12 { + require.NotNil(t, result.Moderated, "the removed post must occupy its original slot in the 25-URI batch") + assert.Equal(t, uris[i], result.Moderated.URI) + } else { + require.NotNil(t, result.Post, "the remaining 24 posts must still be #postView at index %d", i) + assert.Equal(t, uris[i], result.Post.URI) + } + } +} + +type failingPostRemovalRepository struct { + *postgres.PostRepository + err error +} + +func (r failingPostRemovalRepository) ActiveRemovalsByURIs(context.Context, []string) (map[string][]posts.RemovalSource, error) { + return nil, r.err +} + +func TestModerationPostGetRemovalLookupFailure(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "removed subject", "") + removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + lookupFailure := errors.New("removal lookup failed") + repo := failingPostRemovalRepository{PostRepository: postgres.NewPostRepository(db), err: lookupFailure} + results, err := moderationPostGetService(db, repo).GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + assert.ErrorIs(t, err, lookupFailure, "a failed instance-removal lookup must fail post.get, never return #notFoundPost") + assert.Nil(t, results) +} + +// An instance removal must not widen access: a post the viewer could not see +// before the removal (pending, rejected, or a postv2 its community never +// admitted) stays #notFoundPost for that viewer afterwards, instead of becoming +// a #moderatedPost that discloses its author and ties it to the community it +// names. The author, who could see it before, gets the tombstone. +func TestModerationPostGetTombstoneDoesNotWidenAccess(t *testing.T) { + for _, scenario := range []struct { + name, admissionSQL string + }{ + {name: "pending postv2", admissionSQL: ` + UPDATE community_post_admissions SET status = 'pending', accepted_cid = NULL, updated_at = NOW() + WHERE community_did = $1 AND post_uri = $2`}, + {name: "rejected postv2", admissionSQL: ` + UPDATE community_post_admissions SET status = 'rejected', accepted_cid = NULL, + decision_code = 'off-topic', decision_at = NOW(), updated_at = NOW() + WHERE community_did = $1 AND post_uri = $2`}, + {name: "unadmitted postv2 naming the community", admissionSQL: ` + DELETE FROM community_post_admissions WHERE community_did = $1 AND post_uri = $2`}, + } { + t.Run(scenario.name, func(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + otherViewerDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "otherviewer")) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "never public", "") + _, err := db.ExecContext(t.Context(), scenario.admissionSQL, communityDID, subject.URI) + require.NoError(t, err) + postService := moderationPostGetService(db, postgres.NewPostRepository(db)) + before, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + require.NoError(t, err) + require.Len(t, before, 1) + require.NotNil(t, before[0].NotFound, "fixture: the post must be #notFoundPost to the public before the removal") + + removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + + for _, viewer := range []struct{ name, did string }{{"anonymous", ""}, {"other viewer", otherViewerDID}} { + t.Run(viewer.name, func(t *testing.T) { + results, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}, ViewerDID: viewer.did}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Nil(t, results[0].Moderated, "an instance removal must not disclose a post this viewer could not see before it") + require.NotNil(t, results[0].NotFound) + assert.Equal(t, subject.URI, results[0].NotFound.URI) + }) + } + t.Run("author", func(t *testing.T) { + results, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}, ViewerDID: authorDID}) + require.NoError(t, err) + require.Len(t, results, 1) + require.NotNil(t, results[0].Moderated, "the author could see their own post before the removal, so they get the tombstone") + assert.Equal(t, subject.URI, results[0].Moderated.URI) + }) + }) + } +} diff --git a/internal/db/postgres/moderation_repo.go b/internal/db/postgres/moderation_repo.go index c92b521..699d2a7 100644 --- a/internal/db/postgres/moderation_repo.go +++ b/internal/db/postgres/moderation_repo.go @@ -151,6 +151,41 @@ func (t *moderationTransaction) LockSubject(ctx context.Context, subjectURI stri return version, err } +func (t *moderationTransaction) ReadIndexedPost(ctx context.Context, subjectURI string) (*moderation.IndexedPost, error) { + var post moderation.IndexedPost + var authorDID string + var embed sql.NullString + err := t.tx.QueryRowContext(ctx, ` + SELECT uri, cid, author_did, community_did, (deleted_at IS NOT NULL), embed + FROM posts WHERE uri = $1 FOR SHARE + `, subjectURI).Scan(&post.URI, &post.CID, &authorDID, &post.CommunityDID, &post.AuthorDeleted, &embed) + if errors.Is(err, sql.ErrNoRows) { + return nil, moderation.ErrSubjectNotIndexed + } + if err != nil { + return nil, err + } + uri, err := syntax.ParseATURI(post.URI) + if err != nil { + return nil, err + } + post.OwnerDID = authorDID + if uri.Collection().String() == moderation.LegacyPostCollection { + post.OwnerDID = post.CommunityDID + } + if embed.Valid { + // Post embeds are unvalidated indexed data; malformed shapes have no + // proxy-served blobs to block and must not prevent moderation. + var decoded any + if err := json.Unmarshal([]byte(embed.String), &decoded); err == nil { + if object, isObject := decoded.(map[string]any); isObject { + post.BlobCIDs = embeds.PostBlobCIDs(object) + } + } + } + return &post, nil +} + func (t *moderationTransaction) ReadIndexedComment(ctx context.Context, subjectURI string) (*moderation.IndexedComment, error) { var comment moderation.IndexedComment var communityDID, embed sql.NullString diff --git a/internal/db/postgres/post_repo.go b/internal/db/postgres/post_repo.go index acb79eb..96e20e5 100644 --- a/internal/db/postgres/post_repo.go +++ b/internal/db/postgres/post_repo.go @@ -13,6 +13,7 @@ import ( "Coves/internal/core/blobs" "Coves/internal/core/communities" + "Coves/internal/core/embeds" "Coves/internal/core/posts" "github.com/lib/pq" @@ -143,10 +144,10 @@ func (r *PostRepository) Create(ctx context.Context, post *posts.Post) error { // shared by GetRawIndexedRow and GetRawIndexedRowsByURIs so the two cannot drift // apart. It must stay byte-aligned with scanRawIndexedRow's positional Scan. const rawIndexedRowColumns = ` - id, uri, cid, rkey, author_did, community_did, - title, content, content_facets, embed, content_labels, - created_at, edited_at, indexed_at, deleted_at, - upvote_count + bridged_upvote_count AS upvote_count, downvote_count + bridged_downvote_count AS downvote_count, score, comment_count` + p.id, p.uri, p.cid, p.rkey, p.author_did, p.community_did, + p.title, p.content, p.content_facets, p.embed, p.content_labels, + p.created_at, p.edited_at, p.indexed_at, p.deleted_at, + p.upvote_count + p.bridged_upvote_count AS upvote_count, p.downvote_count + p.bridged_downvote_count AS downvote_count, p.score, p.comment_count` // ════════════════════════════════════════════════════════════════════════════ // DANGER — GetRawIndexedRow IS NOT A DISPLAY READ. @@ -173,10 +174,10 @@ const rawIndexedRowColumns = ` // ════════════════════════════════════════════════════════════════════════════ func (r *PostRepository) GetRawIndexedRow(ctx context.Context, uri string) (*posts.Post, error) { query := `SELECT` + rawIndexedRowColumns + ` - FROM posts - WHERE uri = $1` + FROM posts p + WHERE p.uri = $1` - post, err := scanRawIndexedRow(r.db.QueryRowContext(ctx, query, uri)) + post, err := scanRawIndexedRow(r.db.QueryRowContext(ctx, query, uri), false) if errors.Is(err, sql.ErrNoRows) { return nil, posts.ErrNotFound } @@ -186,13 +187,74 @@ func (r *PostRepository) GetRawIndexedRow(ctx context.Context, uri string) (*pos return post, nil } +// AdmittedURIsForViewer returns the subset of uris whose post passes the +// viewer-bound admission half of the read-path visibility predicate +// (admittedPostsPredicate), ignoring active moderation removals. viewerDID is "" +// for an anonymous read. Soft-deleted rows are not filtered here; the caller +// already holds that state from the raw row. +func (r *PostRepository) AdmittedURIsForViewer(ctx context.Context, uris []string, viewerDID string) (map[string]bool, error) { + admitted := make(map[string]bool) + if len(uris) == 0 { + return admitted, nil + } + joinSQL, whereSQL := admittedPostsPredicate("$2") + rows, err := r.db.QueryContext(ctx, `SELECT p.uri FROM posts p`+joinSQL+` + WHERE p.uri = ANY($1) AND `+whereSQL, pq.Array(uris), viewerDID) + if err != nil { + return nil, fmt.Errorf("fetch admitted post URIs: %w", err) + } + defer rows.Close() + for rows.Next() { + var uri string + if err := rows.Scan(&uri); err != nil { + return nil, fmt.Errorf("scan admitted post URI: %w", err) + } + admitted[uri] = true + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate admitted post URIs: %w", err) + } + return admitted, nil +} + +// ActiveRemovalsByURIs returns active removal sources keyed by subject URI. +func (r *PostRepository) ActiveRemovalsByURIs(ctx context.Context, uris []string) (map[string][]posts.RemovalSource, error) { + removals := make(map[string][]posts.RemovalSource) + if len(uris) == 0 { + return removals, nil + } + rows, err := r.db.QueryContext(ctx, ` + SELECT subject_uri, authority_did, scope_kind + FROM moderation_decisions + WHERE subject_uri = ANY($1) AND kind = 'removal' AND active + ORDER BY subject_uri, authority_did, scope_kind, scope_community_did + `, pq.Array(uris)) + if err != nil { + return nil, fmt.Errorf("fetch active post removals: %w", err) + } + defer rows.Close() + for rows.Next() { + var uri string + var source posts.RemovalSource + if err := rows.Scan(&uri, &source.AuthorityDID, &source.ScopeKind); err != nil { + return nil, fmt.Errorf("scan active post removal: %w", err) + } + removals[uri] = append(removals[uri], source) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate active post removals: %w", err) + } + return removals, nil +} + // GetRawIndexedRowsByURIs is the batched GetRawIndexedRow. THE SAME DANGER // APPLIES — read the banner above before calling it. // // URIs with no indexed row are absent from the returned map; that is not an // error, it is the answer ("this URI is not indexed here"), and it is what lets // the caller tell a genuine lookup FAILURE (a returned error) apart from a URI -// the AppView has never seen. +// the AppView has never seen. Rows also carry community handle/name when the +// community exists; a missing community must not hide an indexed post. func (r *PostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []string) (map[string]*posts.Post, error) { result := make(map[string]*posts.Post, len(uris)) if len(uris) == 0 { @@ -202,9 +264,10 @@ func (r *PostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []str // Bound through a single array parameter (= ANY($1)) rather than an // interpolated IN list, so the SQL stays fully parameterized and the plan is // cached regardless of batch size. - query := `SELECT` + rawIndexedRowColumns + ` - FROM posts - WHERE uri = ANY($1)` + query := `SELECT` + rawIndexedRowColumns + `, c.handle, c.name + FROM posts p + LEFT JOIN communities c ON c.did = p.community_did + WHERE p.uri = ANY($1)` rows, err := r.db.QueryContext(ctx, query, pq.Array(uris)) if err != nil { @@ -217,7 +280,7 @@ func (r *PostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []str }() for rows.Next() { - post, err := scanRawIndexedRow(rows) + post, err := scanRawIndexedRow(rows, true) if err != nil { return nil, fmt.Errorf("failed to scan raw post row: %w", err) } @@ -238,17 +301,22 @@ type rowScanner interface { // scanRawIndexedRow scans one rawIndexedRowColumns row into a posts.Post. The // Scan order below MUST stay byte-aligned with that column list. -func scanRawIndexedRow(row rowScanner) (*posts.Post, error) { +func scanRawIndexedRow(row rowScanner, withCommunity bool) (*posts.Post, error) { var post posts.Post var facetsJSON, embedJSON, labelsJSON sql.NullString + var communityHandle, communityName sql.NullString - err := row.Scan( + columns := []interface{}{ &post.ID, &post.URI, &post.CID, &post.RKey, &post.AuthorDID, &post.CommunityDID, &post.Title, &post.Content, &facetsJSON, &embedJSON, &labelsJSON, &post.CreatedAt, &post.EditedAt, &post.IndexedAt, &post.DeletedAt, &post.UpvoteCount, &post.DownvoteCount, &post.Score, &post.CommentCount, - ) + } + if withCommunity { + columns = append(columns, &communityHandle, &communityName) + } + err := row.Scan(columns...) if err != nil { return nil, err } @@ -264,6 +332,12 @@ func scanRawIndexedRow(row rowScanner) (*posts.Post, error) { // Labels are stored as JSONB containing full com.atproto.label.defs#selfLabels structure post.ContentLabels = &labelsJSON.String } + if communityHandle.Valid { + post.CommunityHandle = communityHandle.String + } + if communityName.Valid { + post.CommunityName = communityName.String + } return &post, nil } @@ -683,7 +757,7 @@ func scanPostView(rows *sql.Rows, extraDest ...interface{}) (*posts.PostView, er "error", err, ) } else { - postView.Embed = embedData + postView.Embed = embeds.ServableEmbed(embedData) } } diff --git a/internal/db/postgres/post_repo_cursor_test.go b/internal/db/postgres/post_repo_cursor_test.go index 50fcfae..7e3a4d4 100644 --- a/internal/db/postgres/post_repo_cursor_test.go +++ b/internal/db/postgres/post_repo_cursor_test.go @@ -231,6 +231,14 @@ func (m *mockPostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris [ return map[string]*posts.Post{}, nil } +func (m *mockPostRepository) ActiveRemovalsByURIs(context.Context, []string) (map[string][]posts.RemovalSource, error) { + return map[string][]posts.RemovalSource{}, nil +} + +func (m *mockPostRepository) AdmittedURIsForViewer(context.Context, []string, string) (map[string]bool, error) { + return map[string]bool{}, nil +} + func (m *mockPostRepository) GetByAuthor(ctx context.Context, req posts.GetAuthorPostsRequest) ([]*posts.PostView, *string, error) { return nil, nil, nil } diff --git a/internal/db/postgres/post_visibility.go b/internal/db/postgres/post_visibility.go index f135bf5..455fe4a 100644 --- a/internal/db/postgres/post_visibility.go +++ b/internal/db/postgres/post_visibility.go @@ -70,6 +70,10 @@ import ( // core: every non-accepted post that HAS a decision, and every postv2 with no // decision, is invisible to non-authors on every read path. // +// An active moderation removal hides the post from everyone, including its +// author, regardless of admission status. Restoring it reverts to the admission +// rule above; neither operation changes the post or its admission row. +// // The collection is the fourth '/'-segment of the AT-URI // (at:////), read with split_part; authorities and // rkeys carry no '/', so segment 4 is exactly CollectionOfPostURI's answer. @@ -101,11 +105,30 @@ const anonymousViewerSQL = `''` // anonymousViewerSQL for one that structurally does not. // // Everything above about the join key, the collection-aware status rule and the -// pinned CID describes THIS function; visiblePostsJoin is the parameterized -// spelling of it. Two viewer bindings, one predicate: a count and a display -// query cannot disagree about what "visible" means, because there is only one -// string. +// pinned CID, and active-removal exclusion describes THIS function; +// visiblePostsJoin is the parameterized spelling of it. Two viewer bindings, +// one predicate: a count and a display query cannot disagree about what +// "visible" means, because there is only one string. func visiblePostsPredicate(viewerExpr string) (joinSQL, whereSQL string) { + joinSQL, admittedSQL := admittedPostsPredicate(viewerExpr) + whereSQL = admittedSQL + ` AND NOT EXISTS ( + SELECT 1 FROM moderation_decisions d + WHERE d.subject_uri = p.uri AND d.kind = 'removal' AND d.active + )` + return joinSQL, whereSQL +} + +// admittedPostsPredicate is the viewer-bound admission half of +// visiblePostsPredicate: the join key and the collection-aware status rule, +// WITHOUT the active-removal exclusion. visiblePostsPredicate is this plus that +// exclusion, so the two cannot disagree about admission. +// +// Its only other caller is post.get's #moderatedPost gate +// (AdmittedURIsForViewer): an instance removal may tell a viewer that a post +// was removed only if that viewer could have seen the post had it not been +// removed. Anything else would disclose a pending, rejected or unadmitted post +// and tie it to a community that never accepted it. +func admittedPostsPredicate(viewerExpr string) (joinSQL, whereSQL string) { joinSQL = ` LEFT JOIN community_post_admissions a ON a.community_did = p.community_did AND a.post_uri = p.uri` @@ -121,7 +144,8 @@ func visiblePostsPredicate(viewerExpr string) (joinSQL, whereSQL string) { // visiblePostCountSubquery renders a scalar subquery counting the posts that are // PUBLICLY visible in the community named by communityExpr — the same predicate -// every display query runs, with the anonymous viewer. +// every display query runs, with the anonymous viewer. Active moderation +// removals are excluded from the count just as they are from display queries. // // It exists because `communities.post_count` was a STORED column, and the only // thing that ever incremented it (community_repo_memberships.go's diff --git a/internal/db/postgres/user_repo.go b/internal/db/postgres/user_repo.go index e1611a1..d40fb5f 100644 --- a/internal/db/postgres/user_repo.go +++ b/internal/db/postgres/user_repo.go @@ -232,24 +232,26 @@ func (r *postgresUserRepo) GetProfileStats(ctx context.Context, did string) (*us // count: the author self-view branches turn on `p.author_did = $2`, and no // DID is "". // - // comment_count is DELIBERATELY ROOT-BLIND — it counts the actor's comments - // whatever the admission state of the post they hang under, and that is not - // an oversight to be fixed by symmetry with post_count above. A comment is - // the actor's own public speech, and actor.getComments already LISTS it when - // its root is pending or removed, carrying the root as a bare uri/cid - // reference that leaks nothing and resolves through the gated post.get - // (TestActorCommentsVisibility_RootIsReferenceOnly pins that shape). Gating - // the count would put the profile's headline number in disagreement with the - // list the same profile renders, and would leak in the other direction: a - // comment count that visibly drops tells the reader a root they cannot see - // was moderated. The asymmetry with post_count is real and intended — a - // post's visibility IS its community's decision, a comment's is not. + // comment_count counts exactly what actor.getComments lists + // (ListByCommenterWithCursor): the actor's undeleted comments, minus any + // comment that is itself under an active instance removal or whose root is. + // The exclusion is the same `IN (c.uri, c.root_uri)` predicate that query + // runs, so the profile's headline number and the list it renders cannot + // disagree. It does NOT follow the root's community admission state: a + // comment under a pending or community-removed root is still the actor's own + // public speech and is listed and counted. Only instance removals, which hide + // the root from everyone, take its comments out of both. visJoin, visWhere := visiblePostsJoin(2) query := ` SELECT (SELECT COUNT(*) FROM posts p` + visJoin + ` WHERE p.author_did = $1 AND p.deleted_at IS NULL AND ` + visWhere + `) as post_count, - (SELECT COUNT(*) FROM comments WHERE commenter_did = $1 AND deleted_at IS NULL) as comment_count, + (SELECT COUNT(*) FROM comments c + WHERE c.commenter_did = $1 AND c.deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM moderation_decisions d + WHERE d.subject_uri IN (c.uri, c.root_uri) AND d.kind = 'removal' AND d.active + )) as comment_count, (SELECT COUNT(*) FROM community_subscriptions WHERE user_did = $1) as community_count, (SELECT COUNT(*) FROM community_memberships WHERE user_did = $1 AND is_banned = false) as membership_count, (SELECT COALESCE(SUM(reputation_score), 0) FROM community_memberships WHERE user_did = $1) as reputation diff --git a/tests/e2e/moderation_contract_test.go b/tests/e2e/moderation_contract_test.go index c1ade12..f6e7e0b 100644 --- a/tests/e2e/moderation_contract_test.go +++ b/tests/e2e/moderation_contract_test.go @@ -236,3 +236,290 @@ func TestModerationCommentRemovalContract(t *testing.T) { }) } } + +// TestModerationPostRemovalContract follows an author-owned image post across +// the PDS, the AppView's consumers, instance removal, an author edit, community +// re-acceptance and restoration. The edited CID is the indexing barrier before +// checking that the overlay still hides the post and blocks the new image. +func TestModerationPostRemovalContract(t *testing.T) { + p := newPipeline(t) + author := p.IndexedAccount(t, "mpa") + community := indexedCommunity(t, p, "mpa", author.DID) + admin := testkit.ModerationAdmin(t, 1) + rkey := testkit.TID() + uri := authorPostURI(author.DID, rkey) + needle := "modpost" + testkit.UniqueID(t) + title := needle + " acceptance title" + content := "original post content " + testkit.UniqueID(t) + image := author.UploadBlob(t, testkit.TestPNG(64, 64), "image/png") + writePost := func(postTitle, postContent string, blobs ...testkit.BlobRef) string { + t.Helper() + record := postV2Record(community.DID, postTitle, postContent) + images := make([]any, 0, len(blobs)) + for _, blob := range blobs { + images = append(images, map[string]any{"image": blobRefValue(blob), "alt": "post moderation image"}) + } + record["embed"] = map[string]any{"$type": "social.coves.embed.images", "images": images} + return author.PutRecord(t, postV2Collection, rkey, record).CID + } + createdCID := writePost(title, content, image) + awaitStatus(t, p, uri, community.DID, "pending", "the author's image post to reach the admission queue") + acceptRkey := subjectRkey(uri) + community.PutRecord(t, acceptanceCollection, acceptRkey, acceptanceRecord(uri, createdCID)) + accepted := awaitStatus(t, p, uri, community.DID, "accepted", "the community to accept the image post") + + readPost := func() (map[string]any, error) { + var response struct { + Posts []map[string]any `json:"posts"` + } + err := p.AppView.Query(context.Background(), "social.coves.community.post.get", + url.Values{"uris": {uri}}, &response) + if err != nil { + return nil, err + } + if len(response.Posts) != 1 { + return nil, fmt.Errorf("post.get returned %d union members for one URI", len(response.Posts)) + } + return response.Posts[0], nil + } + var imageURL string + p.Await(t, "the accepted image post to serve through post.get", func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + record, ok := post["record"].(map[string]any) + if !ok || post["uri"] != uri || record["title"] != title || record["content"] != content || post["$type"] != nil { + return false, nil + } + embed, ok := post["embed"].(map[string]any) + if !ok { + return false, nil + } + images, ok := embed["images"].([]any) + if !ok || len(images) != 1 { + return false, nil + } + served, ok := images[0].(map[string]any) + if !ok { + return false, nil + } + imageURL, ok = served["fullsize"].(string) + return ok && imageURL != "", nil + }) + require.Contains(t, imageURL, image.CID()) + requireServesImage(t, p, "accepted post image", imageURL) + parsedImage, err := url.Parse(imageURL) + require.NoError(t, err) + require.True(t, strings.HasPrefix(parsedImage.Path, "/img/"), "the image must be served through the AppView proxy") + require.Contains(t, communityFeedURIs(t, p, community.DID), uri, + "the accepted post must appear in the feed before its removal can prove exclusion") + p.Await(t, "search to find the accepted post before removal", func() (bool, error) { + search, err := queryPostSearch(p, needle, community.DID) + if err != nil { + return false, err + } + return len(search.Feed) == 1 && search.Feed[0].Post.URI == uri, nil + }, withReadCadence()) + + commentRkey := testkit.TID() + commentURI := commentURI(author.DID, commentRkey) + commentText := "comment under removed post " + testkit.UniqueID(t) + ref := strongRef{URI: uri, CID: createdCID} + author.PutRecord(t, commentCollection, commentRkey, commentRecord(ref, ref, commentText)) + p.Await(t, "the comment to appear in the accepted post's thread", func() (bool, error) { + thread, err := p.Thread(context.Background(), uri, nil) + if err != nil { + return false, err + } + node, found := thread.find(commentURI) + return found && node.Comment.Record["content"] == commentText, nil + }, withReadCadence()) + + stateToken := admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + readState := func() (moderationSubjectStateResponse, error) { + var state moderationSubjectStateResponse + err := p.AppView.As(stateToken).Query(context.Background(), subjectStateMethod, + url.Values{"subject": {uri}}, &state) + return state, err + } + state, err := readState() + require.NoError(t, err) + require.Equal(t, createdCID, state.State.CurrentSubject.CID) + require.Equal(t, "clear", state.State.Moderation.State) + var removal struct { + Outcome string `json:"outcome"` + Action struct { + Action struct { + Ref struct { + ActionID string `json:"actionId"` + } `json:"ref"` + } `json:"action"` + } `json:"action"` + } + err = p.AppView.As(admin.ServiceAuth(t, communityInstanceDID, removeContentMethod)).Procedure( + t.Context(), removeContentMethod, map[string]any{ + "subject": map[string]any{"uri": uri, "cid": state.State.CurrentSubject.CID}, + "expectedVersion": state.State.Version, + "idempotencyKey": testkit.UniqueID(t), + "reason": "social.coves.moderation.defs#reasonSpam", + }, &removal) + require.NoError(t, err) + require.Equal(t, "applied", removal.Outcome) + require.NotEmpty(t, removal.Action.Action.Ref.ActionID) + admissionAfterRemoval, err := p.PostStatus(context.Background(), uri, community.DID) + require.NoError(t, err) + require.Equal(t, accepted, admissionAfterRemoval, "instance removal must not change the community's acceptance") + + moderated := func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + if post["$type"] != "social.coves.community.post.defs#moderatedPost" || post["uri"] != uri { + return false, nil + } + for _, key := range []string{"record", "title", "embed"} { + if _, leaked := post[key]; leaked { + return false, fmt.Errorf("moderated post leaked %s: %#v", key, post) + } + } + return true, nil + } + notFoundAnonymously := func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + for _, key := range []string{"record", "title", "embed", "cid"} { + if _, leaked := post[key]; leaked { + return false, fmt.Errorf("notFound post leaked %s: %#v", key, post) + } + } + return post["notFound"] == true && post["uri"] == uri, nil + } + removed, err := moderated() + require.NoError(t, err) + require.True(t, removed, "post.get must serve a content-free moderatedPost immediately after removal") + missingRootURI := authorPostURI(author.DID, testkit.TID()) + _, missingRootErr := p.Thread(context.Background(), missingRootURI, nil) + missingRoot := requireXRPCRefusal(t, missingRootErr, http.StatusNotFound, "RootNotFound", "a never-indexed post thread") + _, removedRootErr := p.Thread(context.Background(), uri, nil) + removedRoot := requireXRPCRefusal(t, removedRootErr, http.StatusNotFound, "RootNotFound", "an instance-removed post thread") + require.Equal(t, missingRoot.XRPCError, removedRoot.XRPCError) + require.NotContains(t, communityFeedURIs(t, p, community.DID), uri) + search, err := queryPostSearch(p, needle, community.DID) + require.NoError(t, err) + require.Empty(t, search.Feed, "the removed post must not appear in search for its unique title") + pathBlocked := func(path string) (bool, error) { + _, err := p.AppView.GetBinary(context.Background(), path) + if testkit.IsStatus(err, http.StatusNotFound) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + } + p.Await(t, "the removed post's cached image to return 404", func() (bool, error) { + return pathBlocked(parsedImage.Path) + }) + + editImage := author.UploadBlob(t, testkit.TestPNG(96, 96), "image/png") + require.NotEqual(t, image.CID(), editImage.CID()) + editImagePath := strings.Replace(parsedImage.Path, image.CID(), editImage.CID(), 1) + require.NotEqual(t, parsedImage.Path, editImagePath) + editedTitle := title + " edited" + editedCID := writePost(editedTitle, "edited while removed", image, editImage) + require.NotEqual(t, createdCID, editedCID) + // Observe the edit in the indexed post before testing the overlay or media + // reconciliation. A still-hidden pre-edit view proves neither behavior. + stateToken = admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + p.Await(t, "getSubjectState to report the author's edited post CID", func() (bool, error) { + indexed, err := readState() + if err != nil { + return false, err + } + return indexed.State.CurrentSubject.CID == editedCID, nil + }) + // The edited CID is not admitted yet, so an anonymous viewer gets notFound: + // removal never widens access to an unadmitted CID. The author's #moderatedPost + // view in this state is proven at T1 by TestModerationPostConsumerEditReconcilesNewImages, + // because this tier cannot hold an AppView-sealed OAuth session. + p.Holds(t, "the unadmitted edited post to read as notFound anonymously with its newly added image blocked", func() (bool, error) { + hidden, err := notFoundAnonymously() + if err != nil || !hidden { + return hidden, err + } + return pathBlocked(editImagePath) + }) + awaitStatus(t, p, uri, community.DID, "pending_reacceptance", + "the author's edit to invalidate the old community acceptance") + + // The community account can update its acceptance at the same subject rkey, + // so restore can be checked through the public thread rather than an + // author-only read of a still-pending post (R3). + community.PutRecord(t, acceptanceCollection, acceptRkey, acceptanceRecord(uri, editedCID)) + reaccepted := awaitStatus(t, p, uri, community.DID, "accepted", + "the community to re-accept the edited CID while instance removal stands") + removed, err = moderated() + require.NoError(t, err) + require.True(t, removed, "community re-acceptance must not undo instance removal") + + stateToken = admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + state, err = readState() + require.NoError(t, err) + require.Equal(t, editedCID, state.State.CurrentSubject.CID) + require.Equal(t, "removed", state.State.Moderation.State) + var restoration struct { + Outcome string `json:"outcome"` + } + err = p.AppView.As(admin.ServiceAuth(t, communityInstanceDID, restoreContentMethod)).Procedure( + t.Context(), restoreContentMethod, map[string]any{ + "actionId": removal.Action.Action.Ref.ActionID, + "reviewedSubject": map[string]any{"uri": uri, "cid": editedCID}, + "expectedVersion": state.State.Version, + "idempotencyKey": testkit.UniqueID(t), + "reason": "social.coves.moderation.defs#reasonModeratorDiscretion", + }, &restoration) + require.NoError(t, err) + require.Equal(t, "applied", restoration.Outcome) + admissionAfterRestore, err := p.PostStatus(context.Background(), uri, community.DID) + require.NoError(t, err) + require.Equal(t, reaccepted, admissionAfterRestore, "instance restoration must not change the community's re-acceptance") + + p.Await(t, "the restored edited post to serve to anonymous readers", func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + record, ok := post["record"].(map[string]any) + return ok && post["uri"] == uri && post["$type"] == nil && record["title"] == editedTitle, nil + }) + p.FreshReadQuota(t, "restored-post-thread") + p.Await(t, "the restored post's comment thread to serve publicly", func() (bool, error) { + thread, err := p.Thread(context.Background(), uri, nil) + if err != nil { + return false, err + } + node, found := thread.find(commentURI) + return thread.Post.URI == uri && found && node.Comment.Record["content"] == commentText, nil + }, withReadCadence()) + require.Contains(t, communityFeedURIs(t, p, community.DID), uri, + "restoration must return the re-accepted post to the community feed") + // Positive control for the blocked-path checks above: both paths are real, + // servable blobs, so their 404s came from the media blocks restore lifts. + for _, path := range []string{parsedImage.Path, editImagePath} { + p.Await(t, "the restored post's images to serve again", func() (bool, error) { + response, err := p.AppView.GetBinary(context.Background(), path) + if testkit.IsStatus(err, http.StatusNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return response.Status == http.StatusOK && len(response.Body) > 0 && + strings.HasPrefix(response.ContentType, "image/"), nil + }) + } +} -- 2.51.2