diff --git a/cmd/server/consumers.go b/cmd/server/consumers.go index fbaa453..324054c 100644 --- a/cmd/server/consumers.go +++ b/cmd/server/consumers.go @@ -230,6 +230,7 @@ func (a *application) registerFeedConsumers() ([]feedConsumer, error) { jetstream.WithAdmissions(a.admissionRepo), jetstream.WithDeletedAccounts(postgresRepo.NewDeletedAccountRepository(a.db)), jetstream.WithPostRecordFetcher(postFetcher), + jetstream.WithPostMediaReconciler(a.mediaReconciler), // The host-side half of an author's own deletion (§5.3): when the // author tombstones a post this instance's community accepted, the // acceptance in that community's repo is withdrawn. It refuses @@ -264,7 +265,7 @@ func (a *application) registerFeedConsumers() ([]feedConsumer, error) { name: jetstream.ConsumerComments, handler: jetstream.NewCommentEventConsumer(a.commentRepo, a.db, jetstream.WithCommentBridgeTrust(a.bridgeTrust), - jetstream.WithCommentMediaReconciler(a.commentMediaReconciler)), + jetstream.WithCommentMediaReconciler(a.mediaReconciler)), }) return consumers, nil diff --git a/cmd/server/wiring.go b/cmd/server/wiring.go index c847f86..6be96c7 100644 --- a/cmd/server/wiring.go +++ b/cmd/server/wiring.go @@ -139,7 +139,7 @@ type application struct { userBlockService userblocks.Service adminReportService adminreports.Service moderationService moderation.Service - commentMediaReconciler jetstream.CommentMediaReconciler + mediaReconciler *moderation.MediaReconciler communitySuggestionService communitysuggestions.Service feedService communityFeeds.Service timelineService timeline.Service @@ -221,7 +221,7 @@ func buildApplication( Purger: purger, }, ) - app.commentMediaReconciler = moderation.NewMediaReconciler( + app.mediaReconciler = moderation.NewMediaReconciler( postgresRepo.NewModerationRepository(app.db), app.cfg.Instance.DID, purger) app.buildJetstreamInfrastructure() if err = app.buildBridgedVotePoller(); err != nil { diff --git a/internal/api/handlers/moderation/remove_content.go b/internal/api/handlers/moderation/remove_content.go index 79f548b..216a772 100644 --- a/internal/api/handlers/moderation/remove_content.go +++ b/internal/api/handlers/moderation/remove_content.go @@ -172,7 +172,7 @@ func writeMutationError(w http.ResponseWriter, operation string, err error) { } { if errors.Is(err, entry.cause) { // Rule errors carry only fixed text and configured limits, such as - // "post removal is unsupported" or the live-key limit, never request + // "unsupported subject collection" or the live-key limit, never request // payloads, so the detail is safe to show the caller. xrpc.WriteError(w, http.StatusBadRequest, entry.code, strings.TrimPrefix(err.Error(), "moderation: ")) return diff --git a/internal/api/handlers/moderation/remove_content_test.go b/internal/api/handlers/moderation/remove_content_test.go index 9224c02..3b8ac38 100644 --- a/internal/api/handlers/moderation/remove_content_test.go +++ b/internal/api/handlers/moderation/remove_content_test.go @@ -224,16 +224,16 @@ func mutationErrorMessage(t *testing.T, response *httptest.ResponseRecorder) str return message } -func TestRemoveContentHandlerNamesPostRemovalAsUnsupported(t *testing.T) { +func TestRemoveContentHandlerRejectsUnsupportedCollectionBeforeStoreAccess(t *testing.T) { service := moderation.NewService(nil, nil, moderation.Config{ InstanceDID: mutationInstanceDID, IdempotencyRetention: time.Hour, MaxLiveIdempotencyKeys: 1, }) - postBody := strings.Replace(removeBody, moderation.CommentCollection, moderation.PostV2Collection, 1) + unsupportedBody := strings.Replace(removeBody, moderation.CommentCollection, "app.bsky.feed.post", 1) response := httptest.NewRecorder() NewRemoveContentHandler(service).HandleRemoveContent(response, - mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", postBody, "application/json")) + mutationRequest(http.MethodPost, "/xrpc/social.coves.moderation.removeContent", unsupportedBody, "application/json")) assertMutationError(t, response, http.StatusBadRequest, "InvalidSubject") - assert.Equal(t, "invalid subject: post removal is unsupported", mutationErrorMessage(t, response)) + assert.Equal(t, "invalid subject: unsupported subject collection", mutationErrorMessage(t, response)) } func TestMutationHandlersWriteRuleDetailButKeepUnavailableGeneric(t *testing.T) { diff --git a/internal/api/handlers/post/get_moderated_test.go b/internal/api/handlers/post/get_moderated_test.go new file mode 100644 index 0000000..ca2067b --- /dev/null +++ b/internal/api/handlers/post/get_moderated_test.go @@ -0,0 +1,69 @@ +package post + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + + "Coves/internal/core/posts" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestHandleGetModeratedPostUnionEncoding(t *testing.T) { + uri := "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/social.coves.community.postv2/abc123" + handler := NewGetHandler(&mockGetPostService{getPostsFunc: func(_ context.Context, _ posts.GetPostsRequest) ([]*posts.PostResult, error) { + return []*posts.PostResult{{Moderated: &posts.ModeratedPost{ + URI: uri, AuthorDID: "did:plc:postauthor", + Community: &posts.CommunityRef{DID: "did:plc:community", Handle: "community.test", Name: "community"}, + Moderation: &posts.ModerationView{State: "removed", Sources: []posts.ModerationSourceView{{ + AuthorityDID: "did:web:instance.test", Scope: posts.ModerationScopeView{Kind: "instance"}, + }}}, + }}}, nil + }}, nil, nil) + recorder := httptest.NewRecorder() + handler.HandleGet(recorder, httptest.NewRequest(http.MethodGet, "/xrpc/social.coves.community.post.get?"+url.Values{"uris": {uri}}.Encode(), nil)) + require.Equal(t, http.StatusOK, recorder.Code, recorder.Body.String()) + var body struct { + Posts []map[string]any `json:"posts"` + } + require.NoError(t, json.Unmarshal(recorder.Body.Bytes(), &body)) + require.Len(t, body.Posts, 1) + item := body.Posts[0] + assert.Equal(t, "social.coves.community.post.defs#moderatedPost", item["$type"]) + assert.Equal(t, uri, item["uri"]) + assert.Equal(t, "did:plc:postauthor", item["authorDid"]) + assert.Equal(t, map[string]any{ + "state": "removed", + "sources": []any{map[string]any{"authorityDid": "did:web:instance.test", "scope": map[string]any{"kind": "instance"}}}, + }, item["moderation"]) + assert.Equal(t, map[string]any{"did": "did:plc:community", "handle": "community.test", "name": "community"}, item["community"]) + for _, key := range []string{"record", "title", "embed", "cid", "content", "notFound", "removed", "blocked"} { + assert.NotContains(t, item, key, "a moderated tombstone cannot leak post content or claim another union member") + } +} + +func TestHandleGetExistingTombstoneUnionEncoding(t *testing.T) { + uri := "at://did:plc:ewvi7nxzyoun6zhxrhs64oiz/social.coves.community.post/abc123" + for _, scenario := range []struct { + name, expected string + result *posts.PostResult + }{ + {"not found", `{"uri":"` + uri + `","notFound":true}`, &posts.PostResult{NotFound: &posts.NotFoundPost{URI: uri, NotFound: true}}}, + {"community removed", `{"uri":"` + uri + `","removed":true,"code":"rule-violation"}`, &posts.PostResult{Removed: &posts.RemovedPost{URI: uri, Removed: true, Code: "rule-violation"}}}, + } { + t.Run(scenario.name, func(t *testing.T) { + handler := NewGetHandler(&mockGetPostService{getPostsFunc: func(_ context.Context, _ posts.GetPostsRequest) ([]*posts.PostResult, error) { + return []*posts.PostResult{scenario.result}, nil + }}, nil, nil) + recorder := httptest.NewRecorder() + handler.HandleGet(recorder, httptest.NewRequest(http.MethodGet, "/xrpc/social.coves.community.post.get?"+url.Values{"uris": {uri}}.Encode(), nil)) + require.Equal(t, http.StatusOK, recorder.Code) + assert.JSONEq(t, `{"posts":[`+scenario.expected+`]}`, recorder.Body.String()) + }) + } +} diff --git a/internal/api/handlers/post/getstatus_moderation_integration_test.go b/internal/api/handlers/post/getstatus_moderation_integration_test.go new file mode 100644 index 0000000..0b27ee5 --- /dev/null +++ b/internal/api/handlers/post/getstatus_moderation_integration_test.go @@ -0,0 +1,83 @@ +//go:build integration + +package post_test + +import ( + "database/sql" + "testing" + "time" + + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func admissionSnapshot(t *testing.T, db *sql.DB, communityDID, postURI string) string { + t.Helper() + var row string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT row_to_json(a)::text FROM community_post_admissions a + WHERE community_did = $1 AND post_uri = $2 + `, communityDID, postURI).Scan(&row)) + return row +} + +func TestGetStatus_InstancePostRemovalPreservesCommunityAcceptance(t *testing.T) { + db := testkit.DB(t) + stack := newStatusStack(db) + subject := newStatusSubject(t, db) // fixtures.Community sets hosted_by_did to this instance. + const postCID = "bafyreistatusseed" + rkey := testkit.TID() + acceptanceURI := "at://" + subject.CommunityDID + "/" + posts.AcceptanceCollection + "/" + rkey + _, err := stack.admissions.UpsertPending(t.Context(), posts.UpsertPendingCommand{ + CommunityDID: subject.CommunityDID, PostURI: subject.PostURI, EvaluatedCID: postCID, + }) + require.NoError(t, err) + result, err := stack.admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: subject.CommunityDID, PostURI: subject.PostURI, + AcceptanceURI: acceptanceURI, AcceptanceRkey: rkey, PinnedCID: postCID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, result.Outcome) + var hostingDID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT hosted_by_did FROM communities WHERE did = $1`, subject.CommunityDID).Scan(&hostingDID)) + require.Equal(t, fixtures.InstanceDID(), hostingDID) + before := admissionSnapshot(t, db, subject.CommunityDID, subject.PostURI) + checkAcceptance := func(t *testing.T) { + t.Helper() + assert.Equal(t, before, admissionSnapshot(t, db, subject.CommunityDID, subject.PostURI), + "instance moderation must not alter any admission column, including record URI, CID, watermarks and timestamps") + body := decodeStatus(t, getStatus(t, stack.handler, subject.PostURI, subject.CommunityDID)) + assert.Equal(t, "accepted", body["status"]) + assert.Equal(t, acceptanceURI, body["acceptanceUri"]) + assert.NotContains(t, body, "decisionCode", "instance removal is not a community removal") + } + checkAcceptance(t) + postRepository := postgres.NewPostRepository(db) + commentRepository := postgres.NewCommentRepository(db) + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepository, commentRepository), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + ref := moderation.StrongRef{URI: subject.PostURI, CID: postCID} + removed, err := moderationService.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "statusadmina")), moderation.RemoveContentRequest{ + Subject: ref, ExpectedVersion: "v0", IdempotencyKey: "remove-status-post", Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + checkAcceptance(t) + restored, err := moderationService.RestoreContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "statusadminb")), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &ref, ExpectedVersion: removed.State.Version, + IdempotencyKey: "restore-status-post", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + checkAcceptance(t) +} diff --git a/internal/api/routes/moderation_media_integration_test.go b/internal/api/routes/moderation_media_integration_test.go index c821c71..022da38 100644 --- a/internal/api/routes/moderation_media_integration_test.go +++ b/internal/api/routes/moderation_media_integration_test.go @@ -141,6 +141,9 @@ type moderationMediaHarness struct { postCID string ownerA string ownerB string + + // proxyService is the purger a consumer's media reconciler shares with the proxy. + proxyService *imageproxy.ImageProxyService } func newModerationMediaHarness(t *testing.T, blockFetch bool) (*moderationMediaHarness, *waitingMediaFetcher) { @@ -190,7 +193,7 @@ func newModerationMediaHarness(t *testing.T, blockFetch bool) (*moderationMediaH routes.RegisterImageProxyRoutes(router, imagehandler.NewHandler(proxyService, mediaPDSResolver{url: pdsServer.URL})) proxy := httptest.NewServer(router) t.Cleanup(proxy.Close) - return &moderationMediaHarness{db: db, cache: cache, cacheDir: cacheDir, proxy: proxy, pds: pds, + return &moderationMediaHarness{db: db, cache: cache, cacheDir: cacheDir, proxy: proxy, proxyService: proxyService, pds: pds, moderation: service, postURI: postURI, postCID: post.CID, ownerA: ownerA, ownerB: ownerB}, waiting } diff --git a/internal/api/routes/moderation_post_media_integration_test.go b/internal/api/routes/moderation_post_media_integration_test.go new file mode 100644 index 0000000..77d0749 --- /dev/null +++ b/internal/api/routes/moderation_post_media_integration_test.go @@ -0,0 +1,298 @@ +//go:build integration + +package routes_test + +import ( + "encoding/json" + "net/http" + "os" + "testing" + "time" + + "Coves/internal/atproto/jetstream" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/core/users" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + postMediaSpamReason = "social.coves.moderation.defs#reasonSpam" + postMediaIllegalReason = "social.coves.moderation.defs#reasonIllegalContent" + postMediaPreset = "content_preview" +) + +func (h *moderationMediaHarness) indexedImagePost(t *testing.T, collection, imageCID string) (moderation.StrongRef, string) { + t.Helper() + var communityDID string + require.NoError(t, h.db.QueryRowContext(t.Context(), `SELECT community_did FROM posts WHERE uri = $1`, h.postURI).Scan(&communityDID)) + rkey := testkit.TID() + authorDID := h.ownerA + blobOwnerDID := authorDID + uriAuthority := authorDID + if collection == moderation.LegacyPostCollection { + blobOwnerDID = communityDID + uriAuthority = communityDID + } + subject := moderation.StrongRef{ + URI: "at://" + uriAuthority + "/" + collection + "/" + rkey, + CID: mediaImageCID("post record " + rkey), + } + embed, err := json.Marshal(map[string]any{ + "$type": "social.coves.embed.images", + "images": []any{map[string]any{ + "alt": "shared post image", + "image": map[string]any{ + "$type": "blob", "ref": map[string]any{"$link": imageCID}, + "mimeType": "image/png", "size": 10, + }, + }}, + }) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, 'image post', 'body', $6::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, authorDID, communityDID, string(embed)) + require.NoError(t, err) + if collection == moderation.PostV2Collection { + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, NOW(), NOW()) + `, communityDID, subject.URI, subject.CID) + require.NoError(t, err) + } + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{blobOwnerDID, imageCID}] = true + h.pds.mu.Unlock() + indexed, err := postgres.NewPostRepository(h.db).GetRawIndexedRow(t.Context(), subject.URI) + require.NoError(t, err) + require.Equal(t, subject.CID, indexed.CID) + require.NotNil(t, indexed.Embed) + assert.Contains(t, *indexed.Embed, imageCID) + return subject, blobOwnerDID +} + +func TestModerationPostV2MediaRemovalColdAndWarm(t *testing.T) { + for _, warm := range []bool{false, true} { + name := "cold" + if warm { + name = "warm" + } + t.Run(name, func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + imageCID := mediaImageCID("postv2 removal " + name) + subject, ownerDID := h.indexedImagePost(t, moderation.PostV2Collection, imageCID) + require.Equal(t, h.ownerA, ownerDID, "postv2 image blobs belong to the author") + if warm { + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, ownerDID, imageCID)) + _, err := os.Stat(h.cachePath(preset, ownerDID, imageCID)) + require.NoError(t, err, "the image must be on disk before removal") + } + } + before := h.pds.count(ownerDID, imageCID) + if warm { + require.Equal(t, 2, before) + } else { + require.Zero(t, before, "a cold image has not been fetched") + } + removed := h.remove(t, subject, postMediaSpamReason) + h.assertNoCachedBlob(t, imageCID, ownerDID) + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusNotFound, h.request(t, preset, ownerDID, imageCID)) + } + assert.Equal(t, before, h.pds.count(ownerDID, imageCID), "blocked post image must not be fetched") + h.restore(t, subject, removed) + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, ownerDID, imageCID)) + assert.Equal(t, before+1, h.pds.count(ownerDID, imageCID), "restored postv2 image must come from a new PDS fetch") + }) + } +} + +func TestModerationLegacyPostSharedImageRemovalAndRestore(t *testing.T) { + for _, warm := range []bool{false, true} { + name := "cold" + if warm { + name = "warm" + } + t.Run(name, func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + imageCID := mediaImageCID("legacy shared post " + name) + first, ownerDID := h.indexedImagePost(t, moderation.LegacyPostCollection, imageCID) + second, secondOwner := h.indexedImagePost(t, moderation.LegacyPostCollection, imageCID) + require.Equal(t, ownerDID, secondOwner) + require.NotEqual(t, h.ownerA, ownerDID, "legacy post blobs belong to the community") + secondView, err := postgres.NewPostRepository(h.db).GetViewsByURIs(t.Context(), []string{second.URI}, "") + require.NoError(t, err) + require.NotNil(t, secondView[second.URI], "the unremoved legacy post must remain served") + if warm { + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, ownerDID, imageCID)) + _, err := os.Stat(h.cachePath(preset, ownerDID, imageCID)) + require.NoError(t, err) + } + } + before := h.pds.count(ownerDID, imageCID) + removed := h.remove(t, first, postMediaSpamReason) + h.assertNoCachedBlob(t, imageCID, ownerDID) + // PRD §9 Media: one community-owned blob shared by two legacy posts is blocked for both when either is removed. + secondView, err = postgres.NewPostRepository(h.db).GetViewsByURIs(t.Context(), []string{second.URI}, "") + require.NoError(t, err) + require.NotNil(t, secondView[second.URI], "the second post remains visible even though its shared image is blocked") + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusNotFound, h.request(t, preset, secondOwner, imageCID), "the shared image on L2 must be refused") + } + assert.Equal(t, before, h.pds.count(ownerDID, imageCID)) + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{h.ownerB, imageCID}] = true + h.pds.mu.Unlock() + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, h.ownerB, imageCID), "spam is scoped to the community blob owner") + assert.Equal(t, 1, h.pds.count(h.ownerB, imageCID)) + h.restore(t, first, removed) + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, ownerDID, imageCID)) + assert.Equal(t, before+1, h.pds.count(ownerDID, imageCID), "restore must re-fetch the purged community-owned image") + }) + } +} + +func TestModerationLegacyPostIllegalContentBlocksEveryOwner(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + imageCID := mediaImageCID("legacy global removal") + subject, communityDID := h.indexedImagePost(t, moderation.LegacyPostCollection, imageCID) + owners := []string{communityDID, h.ownerA, h.ownerB} + h.pds.mu.Lock() + for _, owner := range owners { + h.pds.known[mediaBlobKey{owner, imageCID}] = true + } + h.pds.mu.Unlock() + for _, owner := range owners { + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, owner, imageCID)) + _, err := os.Stat(h.cachePath(preset, owner, imageCID)) + require.NoError(t, err, "each owner's blob must be cached before global removal") + } + } + removed := h.remove(t, subject, postMediaIllegalReason) + h.assertNoCachedBlob(t, imageCID, "") + for _, owner := range owners { + before := h.pds.count(owner, imageCID) + require.Equal(t, http.StatusNotFound, h.request(t, postMediaPreset, owner, imageCID)) + assert.Equal(t, before, h.pds.count(owner, imageCID), "ownerless block must refuse every owner's blob without fetching") + } + h.restore(t, subject, removed) + for _, owner := range owners { + before := h.pds.count(owner, imageCID) + require.Equal(t, http.StatusOK, h.request(t, postMediaPreset, owner, imageCID)) + assert.Equal(t, before+1, h.pds.count(owner, imageCID), "restore must trigger a real PDS fetch for each purged owner") + } +} + +// postV2ImageEvent is a postv2 commit in ownerDID's repo whose record embeds imageCIDs. +func postV2ImageEvent(ownerDID, communityDID, operation, rkey, rev, recordCID string, timeUS int64, imageCIDs ...string) *jetstream.JetstreamEvent { + var record map[string]interface{} + if operation != "delete" { + images := make([]interface{}, 0, len(imageCIDs)) + for _, imageCID := range imageCIDs { + images = append(images, map[string]interface{}{"alt": "recreated image", "image": map[string]interface{}{ + "$type": "blob", "ref": map[string]interface{}{"$link": imageCID}, "mimeType": "image/png", "size": 10, + }}) + } + record = map[string]interface{}{ + "$type": jetstream.PostV2Collection, "community": communityDID, "title": "recreated post", + "content": "body", "createdAt": "2026-03-01T00:00:00Z", + "embed": map[string]interface{}{"$type": "social.coves.embed.images", "images": images}, + } + } + return &jetstream.JetstreamEvent{Kind: "commit", Did: ownerDID, TimeUS: timeUS, Commit: &jetstream.CommitEvent{ + Rev: rev, Operation: operation, Collection: jetstream.PostV2Collection, RKey: rkey, CID: recordCID, Record: record, + }} +} + +// PRD Definition of done: deleting and recreating a removed postv2 keeps it +// removed and blocks the new blob, which the author's repo serves even though +// the AppView keeps the tombstone. +func TestModerationPostV2RecreatedImageBlockedColdAndWarm(t *testing.T) { + for _, warm := range []bool{false, true} { + name := "cold" + if warm { + name = "warm" + } + t.Run(name, func(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + var communityDID string + require.NoError(t, h.db.QueryRowContext(t.Context(), `SELECT community_did FROM posts WHERE uri = $1`, h.postURI).Scan(&communityDID)) + admissions := postgres.NewAdmissionRepository(h.db) + consumer := jetstream.NewPostEventConsumer( + postgres.NewPostRepository(h.db), postgres.NewCommunityRepository(h.db, credentialciphertest.Fixed()), + users.NewUserService(postgres.NewUserRepository(h.db), nil, testkit.Endpoints().PDS.BaseURL, nil, ""), h.db, + jetstream.WithAdmissions(admissions), + jetstream.WithPostMediaReconciler(moderation.NewMediaReconciler( + postgres.NewModerationRepository(h.db), fixtures.InstanceDID(), h.proxyService)), + ) + originalCID, recreatedCID := mediaImageCID("recreate original "+name), mediaImageCID("recreate new "+name) + rkey := testkit.TID() + revs := []string{testkit.TID(), testkit.TID(), testkit.TID()} + createdAt := time.Now().Add(-time.Minute).UnixMicro() + subject := moderation.StrongRef{URI: "at://" + h.ownerA + "/" + jetstream.PostV2Collection + "/" + rkey, CID: mediaImageCID("recreate record " + name)} + require.NoError(t, consumer.HandleEvent(t.Context(), + postV2ImageEvent(h.ownerA, communityDID, "create", rkey, revs[0], subject.CID, createdAt, originalCID))) + acceptanceRkey := testkit.TID() + accepted, err := admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: communityDID, PostURI: subject.URI, + AcceptanceURI: "at://" + communityDID + "/" + posts.AcceptanceCollection + "/" + acceptanceRkey, + AcceptanceRkey: acceptanceRkey, PinnedCID: subject.CID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, accepted.Outcome) + removed := h.remove(t, subject, postMediaSpamReason) + require.NoError(t, consumer.HandleEvent(t.Context(), + postV2ImageEvent(h.ownerA, communityDID, "delete", rkey, revs[1], "", createdAt+1_000_000))) + + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{h.ownerA, recreatedCID}] = true + h.pds.mu.Unlock() + if warm { + // The new blob is fetchable from the author's repo before the + // recreate reaches the AppView, so it can already be cached. + for _, preset := range []string{postMediaPreset, "content_full"} { + require.Equal(t, http.StatusOK, h.request(t, preset, h.ownerA, recreatedCID)) + _, err := os.Stat(h.cachePath(preset, h.ownerA, recreatedCID)) + require.NoError(t, err, "the new image must be on disk before the recreate") + } + } + before := h.pds.count(h.ownerA, recreatedCID) + if warm { + require.Equal(t, 2, before) + } else { + require.Zero(t, before, "a cold image has not been fetched") + } + + require.NoError(t, consumer.HandleEvent(t.Context(), + postV2ImageEvent(h.ownerA, communityDID, "create", rkey, revs[2], subject.CID, createdAt+2_000_000, originalCID, recreatedCID))) + indexed, err := postgres.NewPostRepository(h.db).GetRawIndexedRow(t.Context(), subject.URI) + require.NoError(t, err) + require.NotNil(t, indexed.DeletedAt, "the recreate must not resurrect the tombstone") + state, err := h.moderation.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + + h.assertNoCachedBlob(t, recreatedCID, h.ownerA) + for _, preset := range []string{postMediaPreset, "content_full"} { + for range 2 { + require.Equal(t, http.StatusNotFound, h.request(t, preset, h.ownerA, recreatedCID)) + } + } + assert.Equal(t, before, h.pds.count(h.ownerA, recreatedCID), "the blocked recreated image must not be fetched") + }) + } +} diff --git a/internal/api/routes/moderation_post_removal_integration_test.go b/internal/api/routes/moderation_post_removal_integration_test.go new file mode 100644 index 0000000..c57b3eb --- /dev/null +++ b/internal/api/routes/moderation_post_removal_integration_test.go @@ -0,0 +1,258 @@ +//go:build integration + +package routes_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + commentsAPI "Coves/internal/api/handlers/comments" + "Coves/internal/api/middleware" + "Coves/internal/api/routes" + "Coves/internal/core/comments" + "Coves/internal/core/communities" + "Coves/internal/core/communityFeeds" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/internal/validation" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/bluesky-social/indigo/atproto/atdata" + "github.com/bluesky-social/indigo/atproto/lexicon" + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/require" +) + +func moderationPostAcceptanceNoContentKeys(t *testing.T, value any) { + t.Helper() + switch value := value.(type) { + case map[string]any: + for key, child := range value { + require.NotContains(t, []string{"record", "title", "embed"}, key, "content key in moderated post: %s", key) + moderationPostAcceptanceNoContentKeys(t, child) + } + case []any: + for _, child := range value { + moderationPostAcceptanceNoContentKeys(t, child) + } + } +} + +func moderationPostAcceptanceFeedURIs(t *testing.T, body map[string]any) []string { + t.Helper() + var uris []string + for _, entry := range moderationAcceptanceArray(t, body["feed"]) { + post := moderationAcceptanceObject(t, moderationAcceptanceObject(t, entry)["post"]) + uri, ok := post["uri"].(string) + require.True(t, ok) + uris = append(uris, uri) + } + return uris +} + +func TestModerationPostRemovalAcceptance(t *testing.T) { + db := testkit.DB(t) + postRepo := postgres.NewPostRepository(db) + commentRepo := postgres.NewCommentRepository(db) + userRepo := postgres.NewUserRepository(db) + communityRepo := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()) + admissionRepo := postgres.NewAdmissionRepository(db) + instanceDID := fixtures.InstanceDID() + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepo, commentRepo), + postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: instanceDID, IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + pdsURL := testkit.Endpoints().PDS.BaseURL + communityService := communities.NewCommunityServiceWithPDSFactory( + communityRepo, pdsURL, instanceDID, "", nil, nil, nil, + communities.PrivateHostOptions(true)..., + ) + postService := posts.NewPostService(postRepo, communityService, nil, nil, nil, nil, pdsURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(admissionRepo, nil), + ) + commentService := comments.NewCommentService(commentRepo, userRepo, postRepo, communityRepo, nil, nil, nil) + feedService := communityFeeds.NewCommunityFeedService( + postgres.NewCommunityFeedRepository(db, "moderation-acceptance-cursor-secret"), communityService, + ) + + authorName := testkit.UniqueIDWithPrefix(t, "postauthor") + const authorDID = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "postcommunity") + const communityDID = "did:plc:cccccccccccccccccccccccc" + const ownerDID = "did:plc:dddddddddddddddddddddddd" + fixtures.User(t, db, "owner"+communityName+".test", ownerDID) + _, err := db.ExecContext(t.Context(), ` + INSERT INTO communities (did, name, owner_did, created_by_did, hosted_by_did, handle, pds_url, created_at) + VALUES ($1, $2, $3, $3, $4, $5, $6, NOW()) + `, communityDID, communityName, ownerDID, instanceDID, communityName+".coves.social", pdsURL) + require.NoError(t, err) + const postTitle = "original post removal acceptance title" + const postBody = "original post removal acceptance body text" + const postCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + insertAcceptedPost := func(title, content string) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + rkey + _, err := db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, NOW()) + `, uri, postCID, rkey, authorDID, communityDID, title, content) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, last_community_rev, last_community_op_rank, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, '3lqqqqqqqqqq2', 1, NOW(), NOW()) + `, communityDID, uri, postCID) + require.NoError(t, err) + return uri + } + postURI := insertAcceptedPost(postTitle, postBody) + controlURI := insertAcceptedPost("unremoved acceptance control", "control body") + commentURI := moderationAcceptanceInsertComment(t, db, authorDID, postURI, postCID, postURI, postCID, + "bafyreipostacceptancecomment", "comment under the accepted post") + + adminADID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "postadmina")) + adminBDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "postadminb")) + const adminAToken = "post-removal-admin-a-session" + const adminBToken = "post-removal-admin-b-session" + const authorToken = "post-removal-author-session" + unsealer := fixtures.NewSessionUnsealer() + oauthStore := fixtures.NewOAuthStore() + for _, session := range []struct{ token, did, id string }{ + {adminAToken, adminADID, "post-admin-a"}, + {adminBToken, adminBDID, "post-admin-b"}, + {authorToken, authorDID, "post-author"}, + } { + unsealer.AddSession(session.token, session.did, session.id) + oauthStore.AddSession(session.did, session.id, "test-access-token") + } + adminAuth := middleware.NewInstanceAdminMiddleware(unsealer, oauthStore, nil, + moderation.NewAllowlistAuthority([]string{adminADID, adminBDID})) + optionalAuth := middleware.NewOAuthAuthMiddleware(unsealer, oauthStore) + mux := chi.NewRouter() + routes.RegisterModerationRoutes(mux, moderationService, adminAuth) + routes.RegisterPostRoutes(mux, postService, nil, nil, optionalAuth, optionalAuth) + routes.RegisterCommunityFeedRoutes(mux, feedService, nil, nil, optionalAuth) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.community.comment.getComments", + commentsAPI.NewGetCommentsHandler(commentsAPI.NewServiceAdapter(commentService), nil).HandleGetComments) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + client := server.Client() + postURL := server.URL + "/xrpc/social.coves.community.post.get?" + url.Values{"uris": {postURI}}.Encode() + threadURL := server.URL + "/xrpc/social.coves.community.comment.getComments?" + url.Values{"post": {postURI}, "sort": {"new"}}.Encode() + feedURL := server.URL + "/xrpc/social.coves.communityFeed.getCommunity?" + url.Values{"community": {communityDID}, "sort": {"new"}}.Encode() + + initialPost := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, postURL, "", nil)) + initialPosts := moderationAcceptanceArray(t, initialPost["posts"]) + require.Len(t, initialPosts, 1) + require.Equal(t, postTitle, moderationAcceptanceObject(t, moderationAcceptanceObject(t, initialPosts[0])["record"])["title"]) + initialThread := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil)) + initialComments := moderationAcceptanceArray(t, initialThread["comments"]) + require.Len(t, initialComments, 1) + initialComment := moderationAcceptanceObject(t, moderationAcceptanceObject(t, initialComments[0])["comment"]) + require.Equal(t, commentURI, initialComment["uri"]) + initialFeed := moderationPostAcceptanceFeedURIs(t, moderationAcceptanceBody(t, + moderationAcceptanceRequest(t, client, http.MethodGet, feedURL, "", nil))) + require.ElementsMatch(t, []string{postURI, controlURI}, initialFeed) + + state := moderationAcceptanceObject(t, moderationAcceptanceBody(t, + requestSubjectState(t, client, server.URL, postURI, adminAToken))["state"]) + require.Equal(t, postCID, moderationAcceptanceObject(t, state["currentSubject"])["cid"]) + version, ok := state["version"].(string) + require.True(t, ok) + removeResponse := moderationAcceptanceRequest(t, client, http.MethodPost, + server.URL+"/xrpc/social.coves.moderation.removeContent", adminAToken, map[string]any{ + "subject": map[string]string{"uri": postURI, "cid": postCID}, "expectedVersion": version, + "idempotencyKey": "remove-post", "reason": "social.coves.moderation.defs#reasonSpam", + }) + removed := moderationAcceptanceBody(t, removeResponse) + catalog := lexicon.NewBaseCatalog() + require.NoError(t, catalog.LoadDirectory("../../atproto/lexicon")) + mutationData, err := atdata.UnmarshalJSON(removeResponse.body) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, mutationData, "social.coves.moderation.defs#mutationResult", 0)) + require.Equal(t, "applied", removed["outcome"]) + removalAction := moderationAcceptanceObject(t, moderationAcceptanceObject(t, removed["action"])["action"]) + removalID, ok := moderationAcceptanceObject(t, removalAction["ref"])["actionId"].(string) + require.True(t, ok) + require.NotEmpty(t, removalID) + removedVersion, ok := moderationAcceptanceObject(t, removed["state"])["version"].(string) + require.True(t, ok) + + for _, viewer := range []struct{ name, token string }{{"anonymous", ""}, {"author", authorToken}} { + t.Run(viewer.name, func(t *testing.T) { + response := moderationAcceptanceRequest(t, client, http.MethodGet, postURL, viewer.token, nil) + body := moderationAcceptanceBody(t, response) + items := moderationAcceptanceArray(t, body["posts"]) + require.Len(t, items, 1) + item := moderationAcceptanceObject(t, items[0]) + require.Equal(t, "social.coves.community.post.defs#moderatedPost", item["$type"]) + require.Equal(t, postURI, item["uri"]) + view := moderationAcceptanceObject(t, item["moderation"]) + require.Equal(t, "removed", view["state"]) + sources := moderationAcceptanceArray(t, view["sources"]) + require.Len(t, sources, 1) + source := moderationAcceptanceObject(t, sources[0]) + require.Equal(t, instanceDID, source["authorityDid"]) + require.Equal(t, "instance", moderationAcceptanceObject(t, source["scope"])["kind"]) + moderationPostAcceptanceNoContentKeys(t, item) + require.NotContains(t, string(response.body), postTitle) + require.NotContains(t, string(response.body), postBody) + itemJSON, err := json.Marshal(item) + require.NoError(t, err) + data, err := atdata.UnmarshalJSON(itemJSON) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, data, "social.coves.community.post.defs#moderatedPost", 0)) + }) + } + + missingRootURI := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + testkit.TID() + missingThreadURL := server.URL + "/xrpc/social.coves.community.comment.getComments?" + url.Values{"post": {missingRootURI}, "sort": {"new"}}.Encode() + missingRoot := moderationAcceptanceRequest(t, client, http.MethodGet, missingThreadURL, "", nil) + removedRoot := moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil) + requireXRPCError(t, missingRoot, http.StatusNotFound, "RootNotFound") + requireXRPCError(t, removedRoot, http.StatusNotFound, "RootNotFound") + require.JSONEq(t, string(missingRoot.body), string(removedRoot.body), "a removed root must have the same error code and message as a never-indexed root") + removedFeed := moderationPostAcceptanceFeedURIs(t, moderationAcceptanceBody(t, + moderationAcceptanceRequest(t, client, http.MethodGet, feedURL, "", nil))) + require.NotContains(t, removedFeed, postURI) + require.Contains(t, removedFeed, controlURI) + + restoreResponse := moderationAcceptanceRequest(t, client, http.MethodPost, + server.URL+"/xrpc/social.coves.moderation.restoreContent", adminBToken, map[string]any{ + "actionId": removalID, "reviewedSubject": map[string]string{"uri": postURI, "cid": postCID}, + "expectedVersion": removedVersion, "idempotencyKey": "restore-post", + "reason": "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + restored := moderationAcceptanceBody(t, restoreResponse) + restoreData, err := atdata.UnmarshalJSON(restoreResponse.body) + require.NoError(t, err) + require.NoError(t, validation.ValidateData(catalog, restoreData, "social.coves.moderation.defs#mutationResult", 0)) + require.Equal(t, "applied", restored["outcome"]) + restoredPost := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, postURL, "", nil)) + restoredItems := moderationAcceptanceArray(t, restoredPost["posts"]) + require.Len(t, restoredItems, 1) + restoredItem := moderationAcceptanceObject(t, restoredItems[0]) + require.Equal(t, postURI, restoredItem["uri"]) + require.Equal(t, postTitle, moderationAcceptanceObject(t, restoredItem["record"])["title"]) + require.NotContains(t, restoredItem, "moderation") + restoredThread := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, threadURL, "", nil)) + restoredComments := moderationAcceptanceArray(t, restoredThread["comments"]) + require.Len(t, restoredComments, 1) + restoredComment := moderationAcceptanceObject(t, moderationAcceptanceObject(t, restoredComments[0])["comment"]) + require.Equal(t, commentURI, restoredComment["uri"]) + restoredFeed := moderationPostAcceptanceFeedURIs(t, moderationAcceptanceBody(t, + moderationAcceptanceRequest(t, client, http.MethodGet, feedURL, "", nil))) + require.ElementsMatch(t, []string{postURI, controlURI}, restoredFeed) + +} diff --git a/internal/api/routes/moderation_post_served_media_integration_test.go b/internal/api/routes/moderation_post_served_media_integration_test.go new file mode 100644 index 0000000..f87484e --- /dev/null +++ b/internal/api/routes/moderation_post_served_media_integration_test.go @@ -0,0 +1,145 @@ +//go:build integration + +package routes_test + +import ( + "encoding/json" + "net/http" + "net/url" + "strings" + "testing" + + "Coves/internal/core/blobs" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/db/postgres" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// servedProxyBlobs returns the (owner DID, CID) pair of every image-proxy URL +// under proxyBaseURL anywhere in value. +func servedProxyBlobs(t *testing.T, proxyBaseURL string, value any) []mediaBlobKey { + t.Helper() + var found []mediaBlobKey + switch typed := value.(type) { + case string: + if !strings.HasPrefix(typed, proxyBaseURL+"/img/") { + return nil + } + parsed, err := url.Parse(typed) + require.NoError(t, err) + // /img/{preset}/plain/{did}/{cid} + segments := strings.Split(strings.TrimPrefix(parsed.Path, "/"), "/") + require.Len(t, segments, 5, "unexpected proxy URL shape: %s", typed) + found = append(found, mediaBlobKey{segments[3], segments[4]}) + case map[string]any: + for _, field := range typed { + found = append(found, servedProxyBlobs(t, proxyBaseURL, field)...) + } + case []any: + for _, entry := range typed { + found = append(found, servedProxyBlobs(t, proxyBaseURL, entry)...) + } + } + return found +} + +// PRD §9 Media: a removal blocks the blobs PostBlobCIDs derives, so the served +// view must carry no proxy URL outside them. An author who hand-writes a proxy +// URL for their own blob into the record gets no image in the view; the same +// image referenced as a blob is served and is blocked by an illegal-content +// removal. +func TestModerationPostServedMediaIsBlockedByRemoval(t *testing.T) { + h, _ := newModerationMediaHarness(t, false) + blobs.ResetImageURLConfigForTesting() + blobs.SetImageURLConfig(blobs.ImageURLConfig{ProxyEnabled: true, ProxyBaseURL: h.proxy.URL}) + t.Cleanup(blobs.ResetImageURLConfigForTesting) + var communityDID string + require.NoError(t, h.db.QueryRowContext(t.Context(), `SELECT community_did FROM posts WHERE uri = $1`, h.postURI).Scan(&communityDID)) + repository := postgres.NewPostRepository(h.db) + + blobRef := func(imageCID string) map[string]any { + return map[string]any{"$type": "blob", "ref": map[string]any{"$link": imageCID}, "mimeType": "image/png", "size": 10} + } + authorURL := func(preset, imageCID string) string { + return h.proxy.URL + "/img/" + preset + "/plain/" + h.ownerA + "/" + imageCID + } + for _, test := range []struct { + name string + embed func(imageCID string) map[string]any + wantServed bool + }{ + {name: "external thumb as a URL string", embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.external", "external": map[string]any{ + "uri": "https://example.com/article", "thumb": authorURL("embed_thumbnail", imageCID), + }} + }}, + {name: "stored images#view", embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.images#view", "images": []any{map[string]any{ + "thumb": authorURL(postMediaPreset, imageCID), "fullsize": authorURL("content_full", imageCID), "alt": "forged", + }}} + }}, + {name: "stored external#view", embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.external#view", "external": map[string]any{ + "uri": "https://example.com/article", "thumb": authorURL("embed_thumbnail", imageCID), + }} + }}, + {name: "external thumb as a blob", wantServed: true, embed: func(imageCID string) map[string]any { + return map[string]any{"$type": "social.coves.embed.external", "external": map[string]any{ + "uri": "https://example.com/article", "thumb": blobRef(imageCID), + }} + }}, + } { + t.Run(test.name, func(t *testing.T) { + imageCID := mediaImageCID("served media " + test.name) + h.pds.mu.Lock() + h.pds.known[mediaBlobKey{h.ownerA, imageCID}] = true + h.pds.mu.Unlock() + rkey := testkit.TID() + subject := moderation.StrongRef{ + URI: "at://" + h.ownerA + "/" + moderation.PostV2Collection + "/" + rkey, + CID: mediaImageCID("served media record " + rkey), + } + embed, err := json.Marshal(test.embed(imageCID)) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, 'served media post', 'body', $6::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, h.ownerA, communityDID, string(embed)) + require.NoError(t, err) + _, err = h.db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, NOW(), NOW()) + `, communityDID, subject.URI, subject.CID) + require.NoError(t, err) + + views, err := repository.GetViewsByURIs(t.Context(), []string{subject.URI}, "") + require.NoError(t, err) + view := views[subject.URI] + require.NotNil(t, view, "the post must be served before removal") + posts.TransformBlobRefsToURLs(view) + encoded, err := json.Marshal(view.Embed) + require.NoError(t, err) + var servedEmbed any + require.NoError(t, json.Unmarshal(encoded, &servedEmbed)) + served := servedProxyBlobs(t, h.proxy.URL, servedEmbed) + if test.wantServed { + require.Equal(t, []mediaBlobKey{{h.ownerA, imageCID}}, served) + } else { + require.Empty(t, served, "a proxy URL the author wrote as a string must not be served: %s", encoded) + } + + h.remove(t, subject, postMediaIllegalReason) + for _, blob := range served { + blocked, err := h.proxyService.IsBlobBlocked(t.Context(), blob.did, blob.cid) + require.NoError(t, err) + assert.True(t, blocked, "served blob %s/%s must be blocked after removal", blob.did, blob.cid) + assert.Equal(t, http.StatusNotFound, h.request(t, postMediaPreset, blob.did, blob.cid)) + } + }) + } +} diff --git a/internal/api/routes/moderation_quote_projection_integration_test.go b/internal/api/routes/moderation_quote_projection_integration_test.go new file mode 100644 index 0000000..0ad48a4 --- /dev/null +++ b/internal/api/routes/moderation_quote_projection_integration_test.go @@ -0,0 +1,245 @@ +//go:build integration + +package routes_test + +import ( + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + actorAPI "Coves/internal/api/handlers/actor" + commentsAPI "Coves/internal/api/handlers/comments" + "Coves/internal/api/middleware" + "Coves/internal/api/routes" + "Coves/internal/core/blueskypost" + "Coves/internal/core/comments" + "Coves/internal/core/communities" + "Coves/internal/core/communityFeeds" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/go-chi/chi/v5" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Coves quotes must never invoke Bluesky resolution, but the serving handlers +// must still execute TransformPostEmbeds with a configured resolver. +type rejectingCovesQuoteResolver struct{} + +func (rejectingCovesQuoteResolver) ResolvePost(context.Context, string) (*blueskypost.BlueskyPostResult, error) { + panic("Coves quotes must not trigger Bluesky resolution") +} + +func (rejectingCovesQuoteResolver) ParseBlueskyURL(context.Context, string) (string, error) { + panic("Coves quote reads must not parse Bluesky URLs") +} + +func (rejectingCovesQuoteResolver) IsBlueskyURL(string) bool { + panic("Coves quote reads must not classify Bluesky URLs") +} + +func TestModerationCovesQuoteProjectionAcrossReadPaths(t *testing.T) { + for _, storedType := range []string{"social.coves.embed.post", "social.coves.embed.post#view"} { + t.Run(storedType, func(t *testing.T) { + db := testkit.DB(t) + postRepo := postgres.NewPostRepository(db) + commentRepo := postgres.NewCommentRepository(db) + userRepo := postgres.NewUserRepository(db) + communityRepo := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()) + instanceDID := fixtures.InstanceDID() + moderationService := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepo, commentRepo), postgres.NewModerationRepository(db), + moderation.Config{InstanceDID: instanceDID, IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + pdsURL := testkit.Endpoints().PDS.BaseURL + communityService := communities.NewCommunityServiceWithPDSFactory( + communityRepo, pdsURL, instanceDID, "", nil, nil, nil, + communities.PrivateHostOptions(true)..., + ) + postService := posts.NewPostService(postRepo, communityService, nil, nil, nil, nil, pdsURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(postgres.NewAdmissionRepository(db), nil), + ) + commentService := comments.NewCommentService(commentRepo, userRepo, postRepo, communityRepo, nil, nil, nil) + feedService := communityFeeds.NewCommunityFeedService( + postgres.NewCommunityFeedRepository(db, "moderation-quote-cursor-secret"), communityService, + ) + + authorName := testkit.UniqueIDWithPrefix(t, "quoteauthor") + const authorDID = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "quotecommunity") + const communityDID = "did:plc:cccccccccccccccccccccccc" + const ownerDID = "did:plc:dddddddddddddddddddddddd" + fixtures.User(t, db, "owner"+communityName+".test", ownerDID) + _, err := db.ExecContext(t.Context(), ` + INSERT INTO communities (did, name, owner_did, created_by_did, hosted_by_did, handle, pds_url, created_at) + VALUES ($1, $2, $3, $3, $4, $5, $6, NOW()) + `, communityDID, communityName, ownerDID, instanceDID, communityName+".coves.social", pdsURL) + require.NoError(t, err) + const postCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + insertPost := func(title string, embed any) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + rkey + encoded, err := json.Marshal(embed) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, $6, 'quote fixture body', $7, NOW()) + `, uri, postCID, rkey, authorDID, communityDID, title, encoded) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, last_community_rev, last_community_op_rank, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, '3lqqqqqqqqqq2', 1, NOW(), NOW()) + `, communityDID, uri, postCID) + require.NoError(t, err) + return uri + } + quotedURI := insertPost("quoted post private title", nil) + cleanQuotedURI := insertPost("clean quoted post", nil) + leakImage := map[string]any{"$type": "social.coves.embed.images#view", "images": []any{map[string]any{"fullsize": "leaked-image", "alt": "quoted image"}}} + quoterEmbed := map[string]any{ + "$type": storedType, "post": map[string]any{"uri": quotedURI, "cid": postCID}, + "resolved": map[string]any{"title": "quoted post private title", "text": "quoted private body", "embed": leakImage}, + "title": "leak", "text": "leak", "images": []any{leakImage}, + } + quoterURI := insertPost("quoter searchable marker", quoterEmbed) + storedQuoterEmbed, err := json.Marshal(quoterEmbed) + require.NoError(t, err) + cleanQuoterURI := insertPost("clean quoter", map[string]any{ + "$type": "social.coves.embed.post", "post": map[string]any{"uri": cleanQuotedURI, "cid": postCID}, + }) + + adminDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "quoteadmin")) + const adminToken = "quote-projection-admin-session" + unsealer := fixtures.NewSessionUnsealer() + oauthStore := fixtures.NewOAuthStore() + unsealer.AddSession(adminToken, adminDID, "quote-admin") + oauthStore.AddSession(adminDID, "quote-admin", "test-access-token") + optionalAuth := middleware.NewOAuthAuthMiddleware(unsealer, oauthStore) + adminAuth := middleware.NewInstanceAdminMiddleware(unsealer, oauthStore, nil, moderation.NewAllowlistAuthority([]string{adminDID})) + quoteResolver := rejectingCovesQuoteResolver{} + mux := chi.NewRouter() + routes.RegisterModerationRoutes(mux, moderationService, adminAuth) + routes.RegisterPostRoutes(mux, postService, nil, quoteResolver, optionalAuth, optionalAuth) + routes.RegisterCommunityFeedRoutes(mux, feedService, nil, quoteResolver, optionalAuth) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.actor.getPosts", + actorAPI.NewGetPostsHandler(postService, nil, nil, quoteResolver).HandleGetPosts) + mux.With(optionalAuth.OptionalAuth).Get("/xrpc/social.coves.community.comment.getComments", + commentsAPI.NewGetCommentsHandler(commentsAPI.NewServiceAdapter(commentService), nil).HandleGetComments) + server := httptest.NewServer(mux) + t.Cleanup(server.Close) + client := server.Client() + request := func(path string) map[string]any { + t.Helper() + return moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodGet, server.URL+path, "", nil)) + } + postPath := func(uri string) string { + return "/xrpc/social.coves.community.post.get?" + url.Values{"uris": {uri}}.Encode() + } + postItem := func(uri string) map[string]any { + t.Helper() + items := moderationAcceptanceArray(t, request(postPath(uri))["posts"]) + require.Len(t, items, 1) + return moderationAcceptanceObject(t, items[0]) + } + // A clean Coves quote is currently served as social.coves.embed.post, + // without server-side resolution. R4 pins that same strongRef-only type + // even for stored #view embeds claiming pre-resolved data. + quoteType := "social.coves.embed.post" + assertQuote := func(t *testing.T, post map[string]any, quoted string) { + t.Helper() + require.Equal(t, quoterURI, post["uri"]) + require.NotContains(t, post, "$type", "a normal postView carries no union discriminator") + _, hasRecord := post["record"] + require.True(t, hasRecord, "the quoter itself must be served as a normal post view") + embed := moderationAcceptanceObject(t, post["embed"]) + assert.Equal(t, map[string]any{ + "$type": quoteType, "post": map[string]any{"uri": quoted, "cid": postCID}, + }, embed, "the served embed must contain only the quoted strongRef, with no stored preview or media") + // Decision (orchestrator, from the DoD "P's embed holds Q's + // strongRef only"): record is the quoter's own authored record, + // served verbatim per the lexicon, so record.embed keeps every + // byte the quoter wrote, including the preview fields it forged. + // None of it is AppView-held content of the quoted post: the + // server adds resolved only at serve time, and only for Bluesky + // URIs. The projection applies to the top-level embed alone. + record := moderationAcceptanceObject(t, post["record"]) + recordEmbed, err := json.Marshal(record["embed"]) + require.NoError(t, err) + assert.JSONEq(t, string(storedQuoterEmbed), string(recordEmbed), "record.embed must be the stored embed verbatim") + } + t.Run("clean-control", func(t *testing.T) { + clean := postItem(cleanQuoterURI) + require.Equal(t, cleanQuoterURI, clean["uri"]) + assert.Equal(t, map[string]any{ + "$type": quoteType, "post": map[string]any{"uri": cleanQuotedURI, "cid": postCID}, + }, moderationAcceptanceObject(t, clean["embed"]), "unremoved clean Coves quotes have the same strongRef-only projection") + }) + + state := moderationAcceptanceObject(t, moderationAcceptanceBody(t, + requestSubjectState(t, client, server.URL, quotedURI, adminToken))["state"]) + version, ok := state["version"].(string) + require.True(t, ok) + removed := moderationAcceptanceBody(t, moderationAcceptanceRequest(t, client, http.MethodPost, + server.URL+"/xrpc/social.coves.moderation.removeContent", adminToken, map[string]any{ + "subject": map[string]string{"uri": quotedURI, "cid": postCID}, "expectedVersion": version, + "idempotencyKey": "remove-quoted-post", "reason": "social.coves.moderation.defs#reasonSpam", + })) + require.Equal(t, "applied", removed["outcome"]) + require.Equal(t, "social.coves.community.post.defs#moderatedPost", postItem(quotedURI)["$type"]) + + readPaths := []struct { + name string + path string + selectPost func(*testing.T, map[string]any) map[string]any + }{ + {"post.get", postPath(quoterURI), func(t *testing.T, body map[string]any) map[string]any { + items := moderationAcceptanceArray(t, body["posts"]) + require.Len(t, items, 1) + return moderationAcceptanceObject(t, items[0]) + }}, + {"community feed", "/xrpc/social.coves.communityFeed.getCommunity?" + url.Values{"community": {communityDID}, "sort": {"new"}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationQuoteFeedPost(t, body, quoterURI) + }}, + {"actor.getPosts", "/xrpc/social.coves.actor.getPosts?" + url.Values{"actor": {authorDID}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationQuoteFeedPost(t, body, quoterURI) + }}, + {"searchPosts", "/xrpc/social.coves.feed.searchPosts?" + url.Values{"q": {"quoter searchable marker"}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationQuoteFeedPost(t, body, quoterURI) + }}, + {"getComments header", "/xrpc/social.coves.community.comment.getComments?" + url.Values{"post": {quoterURI}, "sort": {"new"}}.Encode(), func(t *testing.T, body map[string]any) map[string]any { + return moderationAcceptanceObject(t, body["post"]) + }}, + } + for _, readPath := range readPaths { + t.Run(readPath.name, func(t *testing.T) { + assertQuote(t, readPath.selectPost(t, request(readPath.path)), quotedURI) + }) + } + }) + } +} + +func moderationQuoteFeedPost(t *testing.T, body map[string]any, uri string) map[string]any { + t.Helper() + for _, item := range moderationAcceptanceArray(t, body["feed"]) { + post := moderationAcceptanceObject(t, moderationAcceptanceObject(t, item)["post"]) + if post["uri"] == uri { + return post + } + } + require.FailNow(t, "quoter missing from read path", "uri: %s; body: %#v", uri, body) + return nil +} diff --git a/internal/atproto/jetstream/authorpost.go b/internal/atproto/jetstream/authorpost.go index e5d20ad..ee85c2d 100644 --- a/internal/atproto/jetstream/authorpost.go +++ b/internal/atproto/jetstream/authorpost.go @@ -766,7 +766,7 @@ func (c *PostEventConsumer) upsertAuthorPost(ctx context.Context, authorDID stri facets: facetsJSON, embed: embedJSON, labels: labelsJSON, bridgedUpvotes: up, bridgedDownvotes: down, bridgedAsOf: asOf, storedAsOf: stored.bridgedAsOf, storedDeletedAt: stored.deletedAt, - storedIndexedAt: stored.indexedAt, timeUS: timeUS, + storedIndexedAt: stored.indexedAt, timeUS: timeUS, authorDID: authorDID, }) if err != nil { return err diff --git a/internal/atproto/jetstream/comment_consumer.go b/internal/atproto/jetstream/comment_consumer.go index 1174514..c4c035f 100644 --- a/internal/atproto/jetstream/comment_consumer.go +++ b/internal/atproto/jetstream/comment_consumer.go @@ -39,39 +39,45 @@ type CommentEventConsumer struct { // bridgeTrust gates whether a comment's user repo may assert bridgedStats. // nil means default-deny (bridgedStats are ignored for every comment). bridgeTrust *BridgeTrust - mediaReconciler CommentMediaReconciler + mediaReconciler MediaReconciler } // CommentEventConsumerOption configures optional CommentEventConsumer behaviour. type CommentEventConsumerOption func(*CommentEventConsumer) -// CommentMediaReconciler blocks images introduced on a removed comment. -type CommentMediaReconciler interface { +// MediaReconciler blocks images introduced on removed comments or posts. +type MediaReconciler interface { ReconcileTx(ctx context.Context, tx *sql.Tx, subjectURI string) ([]moderation.MediaBlock, error) Purge(blocks []moderation.MediaBlock) } // WithCommentMediaReconciler reconciles media blocks when a removed comment is rewritten. -func WithCommentMediaReconciler(reconciler CommentMediaReconciler) CommentEventConsumerOption { +func WithCommentMediaReconciler(reconciler MediaReconciler) CommentEventConsumerOption { return func(c *CommentEventConsumer) { c.mediaReconciler = reconciler } } // commitCommentWrite reconciles the indexed embed within the write transaction; // cached bytes are purged only after both the comment and its blocks commit. func (c *CommentEventConsumer) commitCommentWrite(ctx context.Context, tx *sql.Tx, uri string) error { + return commitMediaWrite(ctx, tx, uri, c.mediaReconciler, "comment") +} + +// commitMediaWrite reconciles newly indexed media before committing the content +// and its blocks together, then purges cached copies only after a successful commit. +func commitMediaWrite(ctx context.Context, tx *sql.Tx, uri string, reconciler MediaReconciler, kind string) error { var blocks []moderation.MediaBlock - if c.mediaReconciler != nil { + if reconciler != nil { var err error - blocks, err = c.mediaReconciler.ReconcileTx(ctx, tx, uri) + blocks, err = reconciler.ReconcileTx(ctx, tx, uri) if err != nil { - return fmt.Errorf("reconcile comment media: %w", err) + return fmt.Errorf("reconcile %s media: %w", kind, err) } } if err := tx.Commit(); err != nil { return err } - if c.mediaReconciler != nil { - c.mediaReconciler.Purge(blocks) + if reconciler != nil { + reconciler.Purge(blocks) } return nil } diff --git a/internal/atproto/jetstream/post_consumer.go b/internal/atproto/jetstream/post_consumer.go index 75e79e4..4c680e9 100644 --- a/internal/atproto/jetstream/post_consumer.go +++ b/internal/atproto/jetstream/post_consumer.go @@ -4,6 +4,8 @@ import ( "Coves/internal/atproto/identity" "Coves/internal/core/bridgedvotes" "Coves/internal/core/communities" + "Coves/internal/core/embeds" + "Coves/internal/core/moderation" "Coves/internal/core/posts" "Coves/internal/core/richtext" "Coves/internal/core/users" @@ -25,6 +27,9 @@ type PostEventConsumer struct { communityRepo communities.Repository userService users.UserService db *sql.DB // Direct DB access for atomic count reconciliation + + // nil keeps ingestion independent of moderation media reconciliation. + mediaReconciler PostMediaReconciler // bridgeTrust gates whether a post's author repo may assert bridgedStats. // nil means default-deny (bridgedStats are ignored for every post). bridgeTrust *BridgeTrust @@ -262,6 +267,10 @@ type postContentUpdate struct { storedDeletedAt *time.Time storedIndexedAt time.Time timeUS int64 + + // authorDID owns the incoming blobs, which are blocked when the post is + // removed even if the update is skipped. + authorDID string } // applyPostContentUpdate runs the rev gate and the atomic content UPDATE. @@ -273,8 +282,23 @@ type postContentUpdate struct { // as an error would dead-letter healthy events. func (c *PostEventConsumer) applyPostContentUpdate(ctx context.Context, in postContentUpdate) (bool, error) { // Skip soft-deleted rows: a deleted post should not be resurrected by an edit. + // The author's repo still serves the incoming blobs, so a removed post's + // recreated images are blocked even though the content is not indexed. The + // rev gate runs first (read-only: this path never advances the rev) so an + // out-of-order event that predates the delete blocks nothing. if in.storedDeletedAt != nil { + stale, err := recordRevIsStale(ctx, c.db, in.uri, in.rev) + if err != nil { + return false, fmt.Errorf("failed to check rev of soft-deleted post update: %w", err) + } + if stale { + logSkippedStaleRev(ConsumerPosts, "update", in.uri, in.rev) + return false, nil + } log.Printf("Update event for soft-deleted post: %s (skipping)", in.uri) + if err := c.blockIncomingMedia(ctx, in.uri, in.authorDID, in.embed); err != nil { + return false, fmt.Errorf("failed to block media of skipped post update: %w", err) + } return false, nil } @@ -402,7 +426,7 @@ func (c *PostEventConsumer) applyPostContentUpdate(ctx context.Context, in postC return false, nil } - if err := tx.Commit(); err != nil { + if err := commitMediaWrite(ctx, tx, in.uri, c.mediaReconciler, "post"); err != nil { return false, fmt.Errorf("failed to commit post update transaction: %w", err) } @@ -520,7 +544,9 @@ func (c *PostEventConsumer) indexPostIfRevWins(ctx context.Context, post *posts. // (comments implement the in-place re-create because their resurrection // machinery already exists; see comment_consumer.go). log.Printf("Post already indexed: %s (idempotent)", post.URI) - if commitErr := tx.Commit(); commitErr != nil { + // The dropped content's blobs are still served from the author's repo, + // so a removed post blocks them from the incoming embed. + if commitErr := c.commitIncomingMediaWrite(ctx, tx, post.URI, post.AuthorDID, incomingPostBlobCIDs(embedJSON)); commitErr != nil { return false, fmt.Errorf("failed to commit transaction: %w", commitErr) } // Reported as NOT applied: no content was written, so a caller that @@ -562,7 +588,7 @@ func (c *PostEventConsumer) indexPostIfRevWins(ctx context.Context, post *posts. } // Commit transaction - if err := tx.Commit(); err != nil { + if err := commitMediaWrite(ctx, tx, post.URI, c.mediaReconciler, "post"); err != nil { return false, fmt.Errorf("failed to commit transaction: %w", err) } @@ -656,3 +682,67 @@ func parseRecordCreatedAt(raw, uri string) time.Time { } return createdAt } + +// PostMediaReconciler also blocks the blobs of incoming post content that the +// consumer does not index, because the stored row cannot name them. +type PostMediaReconciler interface { + MediaReconciler + ReconcileIncomingTx(ctx context.Context, tx *sql.Tx, subjectURI, ownerDID string, blobCIDs []string) ([]moderation.MediaBlock, error) +} + +// WithPostMediaReconciler reconciles media blocks when a removed post is created or edited. +func WithPostMediaReconciler(reconciler PostMediaReconciler) PostEventConsumerOption { + return func(c *PostEventConsumer) { c.mediaReconciler = reconciler } +} + +// incomingPostBlobCIDs returns the proxy-served blob CIDs of a serialized +// incoming embed. A malformed embed has no served blobs to block. +func incomingPostBlobCIDs(embed sql.NullString) []string { + if !embed.Valid { + return nil + } + var decoded map[string]interface{} + if err := json.Unmarshal([]byte(embed.String), &decoded); err != nil { + return nil + } + return embeds.PostBlobCIDs(decoded) +} + +// commitIncomingMediaWrite blocks the incoming blobs of a removed post inside +// tx, commits, and purges cached copies only after the blocks commit. +func (c *PostEventConsumer) commitIncomingMediaWrite(ctx context.Context, tx *sql.Tx, uri, ownerDID string, blobCIDs []string) error { + var blocks []moderation.MediaBlock + if c.mediaReconciler != nil { + var err error + blocks, err = c.mediaReconciler.ReconcileIncomingTx(ctx, tx, uri, ownerDID, blobCIDs) + if err != nil { + return fmt.Errorf("reconcile incoming post media: %w", err) + } + } + if err := tx.Commit(); err != nil { + return err + } + if c.mediaReconciler != nil { + c.mediaReconciler.Purge(blocks) + } + return nil +} + +// blockIncomingMedia blocks the blobs of an incoming post event that writes no +// post row. It is a no-op unless the post has an active removal. +func (c *PostEventConsumer) blockIncomingMedia(ctx context.Context, uri, ownerDID string, embed sql.NullString) error { + blobCIDs := incomingPostBlobCIDs(embed) + if c.mediaReconciler == nil || len(blobCIDs) == 0 { + return nil + } + tx, err := c.db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("failed to begin media block transaction: %w", err) + } + defer func() { + if rollbackErr := tx.Rollback(); rollbackErr != nil && rollbackErr != sql.ErrTxDone { + log.Printf("Failed to rollback transaction: %v", rollbackErr) + } + }() + return c.commitIncomingMediaWrite(ctx, tx, uri, ownerDID, blobCIDs) +} diff --git a/internal/atproto/jetstream/post_moderation_consumer_test.go b/internal/atproto/jetstream/post_moderation_consumer_test.go new file mode 100644 index 0000000..7e05c02 --- /dev/null +++ b/internal/atproto/jetstream/post_moderation_consumer_test.go @@ -0,0 +1,366 @@ +//go:build integration + +package jetstream + +import ( + "context" + "database/sql" + "encoding/json" + "testing" + "time" + + "Coves/internal/core/communityFeeds" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + postModerationCIDOne = "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" + postModerationCIDTwo = "bafkreicy44vctf2bgqnn5wwzdern7bc2khwi7ku2r66bozl4x6bsrvuj2q" + postModerationRecordCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" +) + +type postModerationPurgeCall struct { + ownerDID, blobCID string + committed bool +} + +type postModerationPurger struct { + db *sql.DB + calls []postModerationPurgeCall +} + +func (p *postModerationPurger) purge(ownerDID, blobCID string) error { + call := postModerationPurgeCall{ownerDID: ownerDID, blobCID: blobCID} + err := p.db.QueryRowContext(context.Background(), ` + SELECT EXISTS (SELECT 1 FROM moderation_media_blocks + WHERE owner_did IS NOT DISTINCT FROM NULLIF($1, '') + AND blob_cid = $2 AND active) + `, ownerDID, blobCID).Scan(&call.committed) + p.calls = append(p.calls, call) + return err +} + +func (p *postModerationPurger) PurgeOwnerBlob(ownerDID, blobCID string) error { + return p.purge(ownerDID, blobCID) +} + +func (p *postModerationPurger) PurgeBlob(blobCID string) error { + return p.purge("", blobCID) +} + +func postModerationRecord(imageCIDs ...string) map[string]interface{} { + record := pv2Record(pv2Community, "shared image post", "same authored body") + images := make([]interface{}, 0, len(imageCIDs)) + for _, imageCID := range imageCIDs { + images = append(images, map[string]interface{}{ + "alt": "indexed image", + "image": map[string]interface{}{ + "$type": "blob", "ref": map[string]interface{}{"$link": imageCID}, + "mimeType": "image/png", "size": 10, + }, + }) + } + record["embed"] = map[string]interface{}{"$type": "social.coves.embed.images", "images": images} + return record +} + +type postModerationConsumerFixture struct { + pv2Fixture + postRepository posts.Repository + postService posts.Service + moderator moderation.Service + purger *postModerationPurger + uri, rkey string + createdAt int64 + revs []string + create *JetstreamEvent +} + +func newPostModerationConsumerFixture(t *testing.T) postModerationConsumerFixture { + t.Helper() + db := testkit.DB(t) + f := newPV2Fixture(t, db) + postRepository := postgres.NewPostRepository(db) + moderationRepository := postgres.NewModerationRepository(db) + purger := &postModerationPurger{db: db} + f.consumer = NewPostEventConsumer( + postRepository, postgres.NewCommunityRepository(db, credentialciphertest.Fixed()), f.users, db, + WithAdmissions(f.admissions), WithDeletedAccounts(postgres.NewDeletedAccountRepository(db)), + WithPostMediaReconciler(moderation.NewMediaReconciler(moderationRepository, fixtures.InstanceDID(), purger)), + ) + moderator := moderation.NewService( + moderation.NewRepositorySubjectReader(postRepository, postgres.NewCommentRepository(db)), + moderationRepository, + moderation.Config{InstanceDID: fixtures.InstanceDID(), IdempotencyRetention: 24 * time.Hour, MaxLiveIdempotencyKeys: 1000}, + ) + postService := posts.NewPostService(postRepository, nil, nil, nil, nil, nil, testkit.Endpoints().PDS.BaseURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(f.admissions, nil)) + rkey := testkit.TID() + uri := pv2URI(pv2Author, rkey) + revs := increasingTIDs(t, 4) + createdAt := time.Now().Add(-time.Minute).UnixMicro() + create := pv2Event(pv2Author, "create", rkey, revs[0], postModerationRecordCID, createdAt, + postModerationRecord(postModerationCIDOne)) + require.NoError(t, f.consumer.HandleEvent(t.Context(), create)) + indexed, err := postRepository.GetRawIndexedRow(t.Context(), uri) + require.NoError(t, err) + require.Equal(t, postModerationRecordCID, indexed.CID) + acceptanceRkey := testkit.TID() + accepted, err := f.admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: pv2Community, PostURI: uri, + AcceptanceURI: "at://" + pv2Community + "/" + posts.AcceptanceCollection + "/" + acceptanceRkey, + AcceptanceRkey: acceptanceRkey, PinnedCID: postModerationRecordCID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, accepted.Outcome) + return postModerationConsumerFixture{ + pv2Fixture: f, postRepository: postRepository, postService: postService, + moderator: moderator, purger: purger, uri: uri, rkey: rkey, + createdAt: createdAt, revs: revs, create: create, + } +} + +func (f postModerationConsumerFixture) remove(t *testing.T, reason string) *moderation.MutationResult { + t.Helper() + removed, err := f.moderator.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "postmediaadmin")), moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: f.uri, CID: postModerationRecordCID}, + ExpectedVersion: "v0", IdempotencyKey: "remove-post-media", Reason: reason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + require.NotNil(t, removed.Action) + var originalBlocks int + require.NoError(t, f.db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDOne).Scan(&originalBlocks)) + require.Equal(t, 1, originalBlocks, "the original image must be blocked before testing edit reconciliation") + assert.Empty(t, f.purger.calls, "removal service has no purger; calls must come from the consumer") + return removed +} + +func (f postModerationConsumerFixture) assertReadPaths(t *testing.T, viewerDID string, wantModerated, wantNotFound bool) { + t.Helper() + results, err := f.postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{f.uri}, ViewerDID: viewerDID}) + require.NoError(t, err) + require.Len(t, results, 1) + if wantModerated { + require.NotNil(t, results[0].Moderated, "a present post under an active removal must be a moderatedPost") + assert.Nil(t, results[0].Post) + } + if wantNotFound { + require.NotNil(t, results[0].NotFound, "a deleted post, or one this viewer could not see before the removal, must be notFound") + assert.Nil(t, results[0].Moderated) + } + feed, _, err := postgres.NewCommunityFeedRepository(f.db, "post-moderation-consumer-cursor").GetCommunityFeed(t.Context(), + communityFeeds.GetCommunityFeedRequest{Community: pv2Community, Sort: "new", Timeframe: "all", Limit: 10}) + require.NoError(t, err) + for _, item := range feed { + assert.NotEqual(t, f.uri, item.Post.URI, "removed or deleted post must not appear in the community feed") + } + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) +} + +func TestModerationPostConsumerReplayAndDuplicateKeepRemoval(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, "social.coves.moderation.defs#reasonSpam") + for _, delivery := range []string{"replay", "duplicate"} { + t.Run(delivery, func(t *testing.T) { + require.NoError(t, f.consumer.HandleEvent(t.Context(), f.create)) + f.assertReadPaths(t, "", true, false) + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + assert.Empty(t, f.purger.calls) + }) + } +} + +func TestModerationPostConsumerEditReconcilesNewImages(t *testing.T) { + for _, scenario := range []struct { + name, reason string + ownerless bool + }{ + {name: "spam owner block", reason: "social.coves.moderation.defs#reasonSpam"}, + {name: "illegal content ownerless block", reason: "social.coves.moderation.defs#reasonIllegalContent", ownerless: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, scenario.reason) + update := pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)) + require.NoError(t, f.consumer.HandleEvent(t.Context(), update)) + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, postModerationCIDOne, indexed.CID, "the edited record must be indexed before media reconciliation") + require.NotNil(t, indexed.Embed) + assert.Contains(t, *indexed.Embed, postModerationCIDTwo, "the new image must be present in the indexed embed") + // The edit moves the admission to pending_reacceptance, which only the + // author could see before the removal, so the tombstone is the + // author's; everyone else gets notFound (#moderatedPost never widens + // access). + f.assertReadPaths(t, pv2Author, true, false) + f.assertReadPaths(t, "", false, true) + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + require.NotNil(t, state.CurrentSubject) + assert.Equal(t, postModerationCIDOne, state.CurrentSubject.CID) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + var ownedBlocks int + require.NoError(t, f.db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDTwo).Scan(&ownedBlocks)) + assert.Equal(t, 1, ownedBlocks, "the edited image needs an active author-owned block on the original action") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{ownerDID: pv2Author, blobCID: postModerationCIDTwo, committed: true}, + "the consumer must purge the author-owned image after the block commits") + if scenario.ownerless { + var ownerlessBlocks int + require.NoError(t, f.db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did IS NULL AND blob_cid = $2 AND active + `, removed.Action.ID, postModerationCIDTwo).Scan(&ownerlessBlocks)) + assert.Equal(t, 1, ownerlessBlocks, "illegal content must also block the edited image for every owner") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{blobCID: postModerationCIDTwo, committed: true}, + "the ownerless image cache must be purged after commit") + } + }) + } +} + +func TestModerationPostConsumerDeleteThenCreateRemainsNotFound(t *testing.T) { + for _, scenario := range []struct { + name, reason string + ownerless bool + }{ + {name: "spam owner block", reason: "social.coves.moderation.defs#reasonSpam"}, + {name: "illegal content ownerless block", reason: "social.coves.moderation.defs#reasonIllegalContent", ownerless: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, scenario.reason) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "delete", f.rkey, f.revs[1], "", f.createdAt+1_000_000, nil))) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "create", f.rkey, f.revs[2], postModerationCIDOne, + f.createdAt+2_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)))) + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + require.NotNil(t, indexed.DeletedAt, "a newer create must not resurrect a soft-deleted postv2") + f.assertReadPaths(t, "", false, true) + state, err := f.moderator.GetSubjectState(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, moderation.RecordStateDeleted, state.RecordState) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + // The recreated record's new image is served from the author's repo + // even though the AppView keeps the tombstone, so it must be blocked. + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDTwo), "the recreated image needs an active author-owned block on the original action") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{ownerDID: pv2Author, blobCID: postModerationCIDTwo, committed: true}, + "the consumer must purge the recreated image after the block commits") + if scenario.ownerless { + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did IS NULL AND blob_cid = $2 AND active + `, removed.Action.ID, postModerationCIDTwo), "illegal content must also block the recreated image for every owner") + assert.Contains(t, f.purger.calls, postModerationPurgeCall{blobCID: postModerationCIDTwo, committed: true}, + "the ownerless image cache must be purged after commit") + } + }) + } +} + +// The create path's ON CONFLICT branch discards incoming content when the row +// already exists (a concurrent insert, or the documented active-row re-create). +// The incoming images are still served from the author's repo, so a removed +// post must block them there too. +func TestModerationPostConsumerAlreadyIndexedCreateBlocksIncomingImages(t *testing.T) { + f := newPostModerationConsumerFixture(t) + removed := f.remove(t, "social.coves.moderation.defs#reasonIllegalContent") + embed, err := json.Marshal(postModerationRecord(postModerationCIDTwo)["embed"]) + require.NoError(t, err) + embedJSON := string(embed) + applied, err := f.consumer.indexPostIfRevWins(t.Context(), &posts.Post{ + URI: f.uri, CID: postModerationCIDOne, RKey: f.rkey, AuthorDID: pv2Author, CommunityDID: pv2Community, + Embed: &embedJSON, CreatedAt: time.Now(), IndexedAt: time.Now(), + }, f.revs[1]) + require.NoError(t, err) + assert.False(t, applied, "an existing row must keep its content") + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, postModerationRecordCID, indexed.CID) + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, pv2Author, postModerationCIDTwo), "the discarded create's image needs an author-owned block") + assert.Equal(t, 1, countRows(t, f.db, ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did IS NULL AND blob_cid = $2 AND active + `, removed.Action.ID, postModerationCIDTwo), "illegal content must block the discarded create's image for every owner") + assert.ElementsMatch(t, []postModerationPurgeCall{ + {ownerDID: pv2Author, blobCID: postModerationCIDTwo, committed: true}, + {blobCID: postModerationCIDTwo, committed: true}, + }, f.purger.calls) +} + +func TestModerationPostConsumerRemovalIsPerURI(t *testing.T) { + f := newPostModerationConsumerFixture(t) + f.remove(t, "social.coves.moderation.defs#reasonSpam") + otherRkey := testkit.TID() + otherURI := pv2URI(pv2Author, otherRkey) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "create", otherRkey, f.revs[1], + postModerationRecordCID, f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne)))) + acceptanceRkey := testkit.TID() + accepted, err := f.admissions.ApplyAcceptance(t.Context(), posts.ApplyAcceptanceCommand{ + CommunityDID: pv2Community, PostURI: otherURI, + AcceptanceURI: "at://" + pv2Community + "/" + posts.AcceptanceCollection + "/" + acceptanceRkey, + AcceptanceRkey: acceptanceRkey, PinnedCID: postModerationRecordCID, + Watermark: posts.CommunityWatermark{Rev: testkit.TID()}, + }) + require.NoError(t, err) + require.Equal(t, posts.AdmissionApplied, accepted.Outcome) + results, err := f.postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{f.uri, otherURI}}) + require.NoError(t, err) + require.Len(t, results, 2) + assert.NotNil(t, results[0].Moderated) + require.NotNil(t, results[1].Post, "a distinct accepted URI must remain a postView") + assert.Equal(t, otherURI, results[1].Post.URI) +} + +func TestModerationPostConsumerUnremovedEditCreatesNoBlocks(t *testing.T) { + f := newPostModerationConsumerFixture(t) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)))) + indexed, err := f.postRepository.GetRawIndexedRow(t.Context(), f.uri) + require.NoError(t, err) + assert.Equal(t, postModerationCIDOne, indexed.CID) + assert.Equal(t, 0, countRows(t, f.db, `SELECT count(*) FROM moderation_media_blocks WHERE blob_cid = $1`, postModerationCIDTwo)) + assert.Empty(t, f.purger.calls) +} + +// An out-of-order update that predates the delete is stale by rev; the rev gate +// must reject it before any media is blocked, or an older record's images would +// be blocked for a post whose newest state is the tombstone. +func TestModerationPostConsumerStaleUpdateAfterDeleteAddsNoBlocks(t *testing.T) { + f := newPostModerationConsumerFixture(t) + f.remove(t, "social.coves.moderation.defs#reasonIllegalContent") + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "delete", f.rkey, f.revs[2], "", f.createdAt+2_000_000, nil))) + require.NoError(t, f.consumer.HandleEvent(t.Context(), pv2Event(pv2Author, "update", f.rkey, f.revs[1], postModerationCIDOne, + f.createdAt+1_000_000, postModerationRecord(postModerationCIDOne, postModerationCIDTwo)))) + assert.Equal(t, 0, countRows(t, f.db, `SELECT count(*) FROM moderation_media_blocks WHERE blob_cid = $1`, postModerationCIDTwo), + "a stale update must not block its images") + assert.Empty(t, f.purger.calls) +} diff --git a/internal/core/comments/comment_servable_embed_test.go b/internal/core/comments/comment_servable_embed_test.go new file mode 100644 index 0000000..45bbc64 --- /dev/null +++ b/internal/core/comments/comment_servable_embed_test.go @@ -0,0 +1,84 @@ +package comments + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "Coves/internal/core/blobs" + "Coves/internal/core/users" +) + +// A comment's served embed carries only the proxy URLs HydrateCommentView +// derives from blobs, the same CIDs CommentImageCIDs blocks. URL strings the +// author wrote into the record are not served in the view; the record keeps +// them verbatim. +func TestBuildCommentView_ServesOnlyBlobDerivedMedia(t *testing.T) { + blobs.ResetImageURLConfigForTesting() + blobs.SetImageURLConfig(blobs.ImageURLConfig{ProxyEnabled: true, ProxyBaseURL: "https://img.coves.social"}) + t.Cleanup(blobs.ResetImageURLConfigForTesting) + + const ( + commenterDID = "did:plc:commenter123" + authorCID = "bafyreigj3fwnwjuzr35k2kuzmb5dixxczrzjhqkr5srlqplsh6gq3bj3si" + quotedURI = "at://did:plc:quoted/social.coves.community.postv2/3kquoted" + ) + authorURL := "https://img.coves.social/img/content_preview/plain/" + commenterDID + "/" + authorCID + postURI := "at://did:plc:post123/social.coves.community.postv2/test" + + for _, test := range []struct { + name string + embed map[string]interface{} + want interface{} + }{ + { + name: "stored images#view is not served", + embed: map[string]interface{}{"$type": "social.coves.embed.images#view", "images": []interface{}{ + map[string]interface{}{"thumb": authorURL, "fullsize": authorURL, "alt": "forged"}, + }}, + }, + { + name: "images record entry with view URLs and no blob keeps only its alt", + embed: map[string]interface{}{"$type": "social.coves.embed.images", "images": []interface{}{ + map[string]interface{}{"thumb": authorURL, "fullsize": authorURL, "alt": "forged"}, + }}, + want: map[string]interface{}{"$type": "social.coves.embed.images", "images": []interface{}{ + map[string]interface{}{"alt": "forged"}, + }}, + }, + { + name: "quote view with a stored resolved keeps only its strongRef", + embed: map[string]interface{}{ + "$type": "social.coves.embed.post#view", "post": map[string]interface{}{"uri": quotedURI, "cid": "bafyquoted"}, + "resolved": map[string]interface{}{"images": []interface{}{map[string]interface{}{"thumb": authorURL}}}, + }, + want: map[string]interface{}{ + "$type": "social.coves.embed.post", "post": map[string]interface{}{"uri": quotedURI, "cid": "bafyquoted"}, + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + encoded, err := json.Marshal(test.embed) + require.NoError(t, err) + embedJSON := string(encoded) + comment := createTestComment("at://"+commenterDID+"/social.coves.community.comment/1", commenterDID, "commenter.test", postURI, postURI, 0) + comment.Embed = &embedJSON + service := NewCommentService(newMockCommentRepo(), newMockUserRepo(), newMockPostRepo(), newMockCommunityRepo(), nil, nil, nil).(*commentService) + + view := service.buildCommentView(comment, nil, nil, map[string]*users.User{ + commenterDID: {DID: commenterDID, Handle: "commenter.test", PDSURL: "https://pds.example.com"}, + }) + + if test.want == nil { + assert.Nil(t, view.Embed) + } else { + assert.Equal(t, test.want, view.Embed) + } + record, ok := view.Record.(*CommentRecord) + require.True(t, ok) + assert.Equal(t, test.embed, record.Embed, "the comment record keeps the stored embed verbatim") + }) + } +} diff --git a/internal/core/comments/comment_service.go b/internal/core/comments/comment_service.go index f2947d4..28cba69 100644 --- a/internal/core/comments/comment_service.go +++ b/internal/core/comments/comment_service.go @@ -1,12 +1,6 @@ package comments import ( - "Coves/internal/core/blobs" - "Coves/internal/core/communities" - "Coves/internal/core/embeds" - "Coves/internal/core/posts" - "Coves/internal/core/richtext" - "Coves/internal/core/users" "context" "encoding/json" "errors" @@ -15,6 +9,13 @@ import ( "strings" "time" + "Coves/internal/core/blobs" + "Coves/internal/core/communities" + "Coves/internal/core/embeds" + "Coves/internal/core/posts" + "Coves/internal/core/richtext" + "Coves/internal/core/users" + "github.com/bluesky-social/indigo/atproto/auth/oauth" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/rivo/uniseg" @@ -633,13 +634,17 @@ func (s *commentService) buildCommentView( // union than posts, and the firehose applies no embed validation, so a // federated comment carrying a post-only embed type must not be stamped // with a #view type the comment union does not declare. + // + // ServableEmbed runs first so the view serves only proxy URLs derived from + // blobs, the CIDs CommentImageCIDs blocks; the record keeps the stored + // embed verbatim. var embed interface{} if comment.Embed != nil && *comment.Embed != "" { - var embedMap map[string]interface{} - if err := json.Unmarshal([]byte(*comment.Embed), &embedMap); err != nil { + var storedEmbed map[string]interface{} + if err := json.Unmarshal([]byte(*comment.Embed), &storedEmbed); err != nil { // Log error but don't fail request - embed is optional slog.Warn("failed to unmarshal embed for comment", "comment_uri", comment.URI, "error", err) - } else { + } else if embedMap, servable := embeds.ServableEmbed(storedEmbed).(map[string]interface{}); servable { var authorPDSURL string if user, found := usersByDID[comment.CommenterDID]; found && user != nil { authorPDSURL = user.PDSURL diff --git a/internal/core/comments/comment_service_test.go b/internal/core/comments/comment_service_test.go index 210d6bf..c62838f 100644 --- a/internal/core/comments/comment_service_test.go +++ b/internal/core/comments/comment_service_test.go @@ -300,6 +300,14 @@ func newMockPostRepo() *mockPostRepo { } } +func (m *mockPostRepo) ActiveRemovalsByURIs(context.Context, []string) (map[string][]posts.RemovalSource, error) { + return map[string][]posts.RemovalSource{}, nil +} + +func (m *mockPostRepo) AdmittedURIsForViewer(context.Context, []string, string) (map[string]bool, error) { + return map[string]bool{}, nil +} + // hideFromHeader makes the visibility predicate refuse this post, as it does for // any non-accepted admission state when the viewer is not the author. func (m *mockPostRepo) hideFromHeader(uri string) { @@ -2014,7 +2022,7 @@ func TestBuildCommentView_ValidEmbedDeserialization(t *testing.T) { communityRepo := newMockCommunityRepo() postURI := "at://did:plc:post123/app.bsky.feed.post/test" - embedJSON := `{"$type":"app.bsky.embed.images","images":[{"alt":"test","image":{"$type":"blob","ref":"bafytest"}}]}` + embedJSON := `{"$type":"social.coves.embed.images","images":[{"alt":"test","image":{"$type":"blob","ref":"bafytest"}}]}` comment := createTestComment("at://did:plc:commenter123/comment/1", "did:plc:commenter123", "commenter.test", postURI, postURI, 0) comment.Embed = &embedJSON @@ -2026,7 +2034,7 @@ func TestBuildCommentView_ValidEmbedDeserialization(t *testing.T) { assert.NotNil(t, result.Embed) embedMap, ok := result.Embed.(map[string]interface{}) assert.True(t, ok) - assert.Equal(t, "app.bsky.embed.images", embedMap["$type"]) + assert.Equal(t, "social.coves.embed.images", embedMap["$type"]) } func TestBuildCommentRecord_ValidLabelsDeserialization(t *testing.T) { @@ -2109,7 +2117,7 @@ func TestBuildCommentView_EmptyStringVsNilHandling(t *testing.T) { { name: "Valid JSON strings", facetsValue: strPtr(`[]`), - embedValue: strPtr(`{}`), + embedValue: strPtr(`{"$type":"social.coves.embed.post","post":{"uri":"at://did:plc:post123/social.coves.community.postv2/test","cid":"bafypost"}}`), labelsValue: strPtr(`{"$type":"com.atproto.label.defs#selfLabels","values":[]}`), expectFacetsNil: false, expectEmbedNil: false, diff --git a/internal/core/comments/post_removal_read_paths_integration_test.go b/internal/core/comments/post_removal_read_paths_integration_test.go new file mode 100644 index 0000000..0ff173c --- /dev/null +++ b/internal/core/comments/post_removal_read_paths_integration_test.go @@ -0,0 +1,194 @@ +//go:build integration + +package comments_test + +import ( + "database/sql" + "testing" + "time" + + "Coves/internal/core/comments" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func postRemovalComment(t *testing.T, db *sql.DB, commenterDID, postURI, postCID, content string, createdAt time.Time) string { + t.Helper() + rkey := testkit.TID() + uri := "at://" + commenterDID + "/" + moderation.CommentCollection + "/" + rkey + _, err := db.ExecContext(t.Context(), ` + INSERT INTO comments (uri, cid, rkey, commenter_did, root_uri, root_cid, parent_uri, parent_cid, content, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $5, $6, $7, $8) + `, uri, moderatedCommentCID, rkey, commenterDID, postURI, postCID, content, createdAt) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), `UPDATE posts SET comment_count = comment_count + 1 WHERE uri = $1`, postURI) + require.NoError(t, err) + return uri +} + +func postRemovalActorComments(t *testing.T, service comments.Service, commenterDID, communityDID string, limit int, cursor *string) *comments.GetActorCommentsResponse { + t.Helper() + response, err := service.GetActorComments(t.Context(), &comments.GetActorCommentsRequest{ + ActorDID: commenterDID, Community: communityDID, Limit: limit, Cursor: cursor, + }) + require.NoError(t, err) + return response +} + +func postRemovalFixture(t *testing.T) (*sql.DB, comments.Service, moderation.Service, moderation.StrongRef, string, string, string, string, string) { + t.Helper() + db, commentService, moderationService, postURI, _, authorDID := moderationThreadFixture(t) + post, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), postURI) + require.NoError(t, err) + communityDID := post.CommunityDID + visibleURI := fixtures.Post(t, db, communityDID, authorDID, "unremoved companion thread", 0, time.Now()) + visiblePost, err := postgres.NewPostRepository(db).GetRawIndexedRow(t.Context(), visibleURI) + require.NoError(t, err) + commenterName := testkit.UniqueIDWithPrefix(t, "rootcommenter") + commenterDID := fixtures.DID(commenterName) + fixtures.User(t, db, commenterName+".test", commenterDID) + first := postRemovalComment(t, db, commenterDID, postURI, post.CID, "first original comment", time.Now().Add(-time.Minute)) + visible := postRemovalComment(t, db, commenterDID, visibleURI, visiblePost.CID, "visible companion comment", time.Now().Add(-2*time.Minute)) + second := postRemovalComment(t, db, commenterDID, postURI, post.CID, "second original comment", time.Now().Add(-3*time.Minute)) + return db, commentService, moderationService, moderation.StrongRef{URI: postURI, CID: post.CID}, communityDID, commenterDID, first, visible, second +} + +func removeFixturePost(t *testing.T, service moderation.Service, subject moderation.StrongRef) *moderation.MutationResult { + t.Helper() + removed, err := service.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "rootadmina")), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-root", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + return removed +} + +func TestPostRemovalGetCommentsHidesRootForEveryViewer(t *testing.T) { + db, commentService, moderationService, subject, _, _, _, _, _ := postRemovalFixture(t) + baseline, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{PostURI: subject.URI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + require.Len(t, baseline.Comments, 3) + var authorDID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT author_did FROM posts WHERE uri = $1`, subject.URI).Scan(&authorDID)) + removeFixturePost(t, moderationService, subject) + for _, viewer := range []struct { + name string + did *string + }{ + {name: "anonymous"}, + {name: "post author", did: &authorDID}, + } { + t.Run(viewer.name, func(t *testing.T) { + missingURI := "at://" + authorDID + "/" + moderation.PostV2Collection + "/" + testkit.TID() + _, missingErr := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{ + PostURI: missingURI, ViewerDID: viewer.did, Sort: "new", Depth: 1, Limit: 10, + }) + require.ErrorIs(t, missingErr, comments.ErrRootNotFound) + _, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{ + PostURI: subject.URI, ViewerDID: viewer.did, Sort: "new", Depth: 1, Limit: 10, + }) + require.ErrorIs(t, err, comments.ErrRootNotFound) + }) + } +} + +func TestPostRemovalActorCommentsFiltersBeforePagination(t *testing.T) { + _, commentService, moderationService, subject, communityDID, commenterDID, first, visible, second := postRemovalFixture(t) + baseline := postRemovalActorComments(t, commentService, commenterDID, "", 10, nil) + require.Len(t, baseline.Comments, 3) + require.Equal(t, []string{first, visible, second}, []string{baseline.Comments[0].URI, baseline.Comments[1].URI, baseline.Comments[2].URI}) + removeFixturePost(t, moderationService, subject) + for _, filter := range []struct{ name, community string }{{"all communities", ""}, {"community filtered", communityDID}} { + t.Run(filter.name, func(t *testing.T) { + whole := postRemovalActorComments(t, commentService, commenterDID, filter.community, 10, nil) + require.Len(t, whole.Comments, 1, "removed-root comments must be excluded before the page is selected") + assert.Equal(t, visible, whole.Comments[0].URI) + assert.Nil(t, whole.Cursor) + page := postRemovalActorComments(t, commentService, commenterDID, filter.community, 1, nil) + require.Len(t, page.Comments, 1, "page one must advance past the newer removed-root comment") + assert.Equal(t, visible, page.Comments[0].URI) + assert.Nil(t, page.Cursor, "the older removed-root comment must not leak a continuation cursor") + }) + } +} + +func TestPostRemovalRestoreServesIndexedThreadAndActorComments(t *testing.T) { + db, commentService, moderationService, subject, _, commenterDID, first, visible, second := postRemovalFixture(t) + before, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{PostURI: subject.URI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + baselinePost, ok := before.Post.(*posts.PostView) + require.True(t, ok) + require.NotNil(t, baselinePost.Stats) + baselineCommentCount := baselinePost.Stats.CommentCount + removed := removeFixturePost(t, moderationService, subject) + during := postRemovalComment(t, db, commenterDID, subject.URI, subject.CID, "indexed while root removed", time.Now()) + voterDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "rootvoter")) + voteRkey := testkit.TID() + _, err = db.ExecContext(t.Context(), ` + INSERT INTO votes (uri, cid, rkey, voter_did, subject_uri, subject_cid, direction, created_at) + VALUES ($1, $2, $3, $4, $5, $6, 'up', NOW()) + `, "at://"+voterDID+"/social.coves.interaction.vote/"+voteRkey, moderatedCommentCID, voteRkey, voterDID, first, moderatedCommentCID) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), `UPDATE comments SET upvote_count = upvote_count + 1, score = score + 1 WHERE uri = $1`, first) + require.NoError(t, err) + restored, err := moderationService.RestoreContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "rootadminb")), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: removed.State.Version, + IdempotencyKey: "restore-root", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + thread, err := commentService.GetComments(t.Context(), &comments.GetCommentsRequest{PostURI: subject.URI, Sort: "new", Depth: 1, Limit: 10}) + require.NoError(t, err) + restoredPost, ok := thread.Post.(*posts.PostView) + require.True(t, ok) + require.NotNil(t, restoredPost.Stats) + assert.Equal(t, baselineCommentCount+1, restoredPost.Stats.CommentCount, + "a comment indexed during removal must contribute to the restored post's count") + for _, uri := range []string{first, second, during} { + view := moderationThreadComment(t, thread, uri).Comment + require.NotNil(t, view.Record) + if uri == first { + assert.Equal(t, 1, view.Stats.Upvotes, "vote indexed during removal must be served") + } + } + assert.Equal(t, "indexed while root removed", moderationThreadComment(t, thread, during).Comment.Record.(*comments.CommentRecord).Content) + actor := postRemovalActorComments(t, commentService, commenterDID, "", 10, nil) + require.Len(t, actor.Comments, 4) + assert.Equal(t, []string{during, first, visible, second}, []string{actor.Comments[0].URI, actor.Comments[1].URI, actor.Comments[2].URI, actor.Comments[3].URI}) +} + +// The profile's commentCount must agree with the list actor.getComments +// serves: both exclude comments under an instance-removed root and comments +// that are themselves instance-removed. +func TestPostRemovalProfileCommentCountMatchesActorComments(t *testing.T) { + db, commentService, moderationService, subject, _, commenterDID, _, visible, _ := postRemovalFixture(t) + var visibleRootURI, visibleRootCID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT root_uri, root_cid FROM comments WHERE uri = $1`, visible).Scan(&visibleRootURI, &visibleRootCID)) + directlyRemoved := postRemovalComment(t, db, commenterDID, visibleRootURI, visibleRootCID, "comment removed directly", time.Now().Add(-4*time.Minute)) + userRepo := postgres.NewUserRepository(db) + baseline, err := userRepo.GetProfileStats(t.Context(), commenterDID) + require.NoError(t, err) + require.Equal(t, 4, baseline.CommentCount, "fixture: four comments before any removal") + + removeFixturePost(t, moderationService, subject) + removedComment, err := moderationService.RemoveContent(t.Context(), fixtures.DID(testkit.UniqueIDWithPrefix(t, "commentadmin")), moderation.RemoveContentRequest{ + Subject: moderation.StrongRef{URI: directlyRemoved, CID: moderatedCommentCID}, ExpectedVersion: "v0", + IdempotencyKey: "remove-comment", Reason: moderationTestReason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removedComment.Outcome) + + listed := postRemovalActorComments(t, commentService, commenterDID, "", 50, nil) + require.Len(t, listed.Comments, 1) + assert.Equal(t, visible, listed.Comments[0].URI) + stats, err := userRepo.GetProfileStats(t.Context(), commenterDID) + require.NoError(t, err) + assert.Equal(t, len(listed.Comments), stats.CommentCount, + "profile commentCount must equal what actor.getComments lists: removed-root and removed comments are excluded from both") +} diff --git a/internal/core/embeds/post_blob_cids_test.go b/internal/core/embeds/post_blob_cids_test.go new file mode 100644 index 0000000..ceda365 --- /dev/null +++ b/internal/core/embeds/post_blob_cids_test.go @@ -0,0 +1,63 @@ +package embeds + +import ( + "testing" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multibase" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPostBlobCIDs(t *testing.T) { + parsed, err := cid.Decode(testCID) + require.NoError(t, err) + base58Alias, err := parsed.StringOfBase(multibase.Base58BTC) + require.NoError(t, err) + legacyBlob := map[string]interface{}{"cid": secondTestCID, "mimeType": "image/png"} + + for _, test := range []struct { + name string + embed map[string]interface{} + want []string + }{ + {name: "nil embed", embed: nil}, + {name: "empty embed", embed: map[string]interface{}{}}, + {name: "images retain first-seen order", embed: imagesEmbed(imageEntry(blobRef(testCID)), imageEntry(blobRef(secondTestCID))), want: []string{testCID, secondTestCID}}, + {name: "video thumbnail is proxied but video is not", embed: map[string]interface{}{ + "$type": TypeVideo, "video": blobRef(testCID), "thumbnail": blobRef(secondTestCID), + }, want: []string{secondTestCID}}, + {name: "video without thumbnail has no proxy blob", embed: map[string]interface{}{"$type": TypeVideo, "video": blobRef(testCID)}}, + {name: "external thumbnail and gallery", embed: map[string]interface{}{ + "$type": TypeExternal, "external": map[string]interface{}{ + "thumb": blobRef(testCID), "images": []interface{}{imageEntry(legacyBlob)}, + }, + }, want: []string{testCID, secondTestCID}}, + {name: "legacy top-level cid", embed: imagesEmbed(imageEntry(legacyBlob)), want: []string{secondTestCID}}, + {name: "noncanonical CID becomes canonical", embed: imagesEmbed(imageEntry(blobRef(base58Alias))), want: []string{testCID}}, + {name: "duplicate encodings keep first occurrence", embed: imagesEmbed( + imageEntry(blobRef(secondTestCID)), imageEntry(blobRef(testCID)), + imageEntry(blobRef(base58Alias)), imageEntry(legacyBlob), + ), want: []string{secondTestCID, testCID}}, + {name: "malformed image siblings are skipped", embed: imagesEmbed( + "not an image", imageEntry("not a blob"), imageEntry(map[string]interface{}{"ref": map[string]interface{}{"$link": 5}}), + imageEntry(blobRef("bafynotacid")), imageEntry(blobRef(testCID)), + ), want: []string{testCID}}, + {name: "malformed external siblings are skipped", embed: map[string]interface{}{ + "$type": TypeExternal, "external": map[string]interface{}{ + "thumb": map[string]interface{}{"ref": map[string]interface{}{}}, + "images": []interface{}{imageEntry(blobRef(testCID)), nil, imageEntry(blobRef("bad cid")), imageEntry(legacyBlob)}, + }, + }, want: []string{testCID, secondTestCID}}, + {name: "quote embeds do not expose blobs", embed: map[string]interface{}{ + "$type": TypePost, "post": map[string]interface{}{"embed": imagesEmbed(imageEntry(blobRef(testCID)))}, + }}, + {name: "projected images are not indexed blobs", embed: map[string]interface{}{ + "$type": TypeImages + viewSuffix, "images": []interface{}{imageEntry(blobRef(testCID))}, + }}, + } { + t.Run(test.name, func(t *testing.T) { + assert.Equal(t, test.want, PostBlobCIDs(test.embed)) + }) + } +} diff --git a/internal/core/embeds/servable_embed_test.go b/internal/core/embeds/servable_embed_test.go new file mode 100644 index 0000000..950b6ba --- /dev/null +++ b/internal/core/embeds/servable_embed_test.go @@ -0,0 +1,254 @@ +package embeds + +import ( + "encoding/json" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// authorURL is an image-proxy URL an author typed into a record by hand. It +// names secondTestCID, so the served view must not carry it unless +// PostBlobCIDs also reports that CID for the same stored embed. +func authorURL(preset string) string { + return "https://img.coves.social/img/" + preset + "/plain/" + testDID + "/" + secondTestCID +} + +func copyEmbed(t *testing.T, stored interface{}) interface{} { + t.Helper() + encoded, err := json.Marshal(stored) + require.NoError(t, err) + var decoded interface{} + require.NoError(t, json.Unmarshal(encoded, &decoded)) + return decoded +} + +// servedProxyURLs collects every string in value that names the image proxy. +func servedProxyURLs(value interface{}) []string { + var urls []string + switch typed := value.(type) { + case string: + if strings.Contains(typed, "/img/") { + urls = append(urls, typed) + } + case map[string]interface{}: + for _, field := range typed { + urls = append(urls, servedProxyURLs(field)...) + } + case []interface{}: + for _, entry := range typed { + urls = append(urls, servedProxyURLs(entry)...) + } + } + return urls +} + +// serve runs the post read path's projection: scanPostView's ServableEmbed, +// then the handlers' HydrateView. +func serve(t *testing.T, stored interface{}) interface{} { + t.Helper() + served := ServableEmbed(copyEmbed(t, stored)) + if embed, ok := served.(map[string]interface{}); ok { + HydrateView(embed, testDID, testPDS) + } + return served +} + +func TestServableEmbed_ServesOnlyMediaURLsDerivedFromBlobs(t *testing.T) { + withProxy(t, "https://img.coves.social") + const blueskyURI = "at://did:plc:quoted/app.bsky.feed.post/3kquoted" + const covesURI = "at://did:plc:quoted/social.coves.community.postv2/3kquoted" + forgedResolved := map[string]interface{}{ + "text": "forged preview", "images": []interface{}{map[string]interface{}{ + "thumb": authorURL("content_preview"), "fullsize": authorURL("content_full"), + }}, + } + quoteOf := func(embedType, uri string) map[string]interface{} { + return map[string]interface{}{ + "$type": embedType, "post": map[string]interface{}{"uri": uri, "cid": testCID}, + "resolved": forgedResolved, "images": []interface{}{map[string]interface{}{"thumb": authorURL("content_preview")}}, + } + } + strongRef := func(uri string) map[string]interface{} { + return map[string]interface{}{"$type": TypePost, "post": map[string]interface{}{"uri": uri, "cid": testCID}} + } + + for _, test := range []struct { + name string + stored interface{} + want interface{} + }{ + { + name: "external thumb given as a URL string is dropped", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "title": "An article", "thumb": authorURL(presetEmbedThumbnail), + }}, + want: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", "title": "An article", + }}, + }, + { + name: "external thumb given as a non-blob value is dropped", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": []interface{}{authorURL(presetEmbedThumbnail)}, + }}, + want: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", + }}, + }, + { + name: "external gallery view URLs without a blob are dropped", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/gallery", "images": []interface{}{map[string]interface{}{ + "alt": "gallery", "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }}, + }}, + want: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/gallery", "images": []interface{}{map[string]interface{}{"alt": "gallery"}}, + }}, + }, + { + name: "images record entry with view URLs and no blob is dropped to its alt", + stored: imagesEmbed(map[string]interface{}{ + "alt": "no blob", "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }), + want: imagesEmbed(map[string]interface{}{"alt": "no blob"}), + }, + { + name: "images record entry with a blob serves only the blob's URLs", + stored: imagesEmbed(map[string]interface{}{ + "alt": "real", "image": blobRef(testCID), "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }), + want: map[string]interface{}{"$type": TypeImages + viewSuffix, "images": []interface{}{map[string]interface{}{ + "alt": "real", + "thumb": "https://img.coves.social/img/content_preview/plain/" + testDID + "/" + testCID, + "fullsize": "https://img.coves.social/img/content_full/plain/" + testDID + "/" + testCID, + }}}, + }, + { + name: "video given as URL strings is dropped", + stored: map[string]interface{}{ + "$type": TypeVideo, "alt": "clip", "video": "https://pds.example.com/video.mp4", "thumbnail": authorURL(presetContentPreview), + }, + want: map[string]interface{}{"$type": TypeVideo, "alt": "clip"}, + }, + { + name: "stored images#view is not served", + stored: map[string]interface{}{"$type": TypeImages + viewSuffix, "images": []interface{}{map[string]interface{}{ + "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }}}, + }, + { + name: "stored external#view is not served", + stored: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": authorURL(presetEmbedThumbnail), + }}, + }, + { + name: "stored video#view is not served", + stored: map[string]interface{}{"$type": TypeVideo + viewSuffix, "video": "https://pds.example.com/video.mp4", "thumbnail": authorURL(presetContentPreview)}, + }, + { + name: "a type outside the post embed union is not served", + stored: map[string]interface{}{"$type": "app.bsky.embed.images#view", "images": []interface{}{map[string]interface{}{ + "thumb": authorURL(presetContentPreview), "fullsize": authorURL(presetContentFull), + }}}, + }, + {name: "an untyped embed is not served", stored: map[string]interface{}{"thumb": authorURL(presetEmbedThumbnail)}}, + {name: "a non-object embed is not served", stored: []interface{}{authorURL(presetEmbedThumbnail)}}, + {name: "a string embed is not served", stored: authorURL(presetEmbedThumbnail)}, + {name: "Coves quote record keeps only its strongRef", stored: quoteOf(TypePost, covesURI), want: strongRef(covesURI)}, + {name: "Coves quote view keeps only its strongRef", stored: quoteOf(TypePost+viewSuffix, covesURI), want: strongRef(covesURI)}, + { + name: "Bluesky quote view drops its stored resolved for server resolution", + stored: quoteOf(TypePost+viewSuffix, blueskyURI), + want: strongRef(blueskyURI), + }, + {name: "Bluesky quote record keeps only its strongRef", stored: quoteOf(TypePost, blueskyURI), want: strongRef(blueskyURI)}, + { + name: "Bluesky collection forged after a Coves collection keeps only its strongRef", + stored: quoteOf(TypePost+viewSuffix, "at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted"), + want: strongRef("at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted"), + }, + { + name: "Bluesky collection forged in a fragment keeps only its strongRef", + stored: quoteOf(TypePost+viewSuffix, covesURI+"#/app.bsky.feed.post/3k"), + want: strongRef(covesURI + "#/app.bsky.feed.post/3k"), + }, + { + name: "Bluesky collection forged as the authority keeps only its strongRef", + stored: quoteOf(TypePost+viewSuffix, "at://app.bsky.feed.post/social.coves.community.postv2/3kquoted"), + want: strongRef("at://app.bsky.feed.post/social.coves.community.postv2/3kquoted"), + }, + { + name: "blob-backed external thumb is served through the proxy", + stored: map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": blobRef(testCID), + }}, + want: map[string]interface{}{"$type": TypeExternal + viewSuffix, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": "https://img.coves.social/img/embed_thumbnail/plain/" + testDID + "/" + testCID, + }}, + }, + } { + t.Run(test.name, func(t *testing.T) { + served := serve(t, test.stored) + if test.want == nil { + assert.Nil(t, served) + } else { + assert.Equal(t, test.want, served) + } + + // The invariant the moderation blocks rely on: every proxy URL the + // view serves names a CID PostBlobCIDs reports for the stored embed. + stored, _ := copyEmbed(t, test.stored).(map[string]interface{}) + blocked := PostBlobCIDs(stored) + for _, url := range servedProxyURLs(served) { + covered := false + for _, cid := range blocked { + if strings.HasSuffix(url, "/"+testDID+"/"+cid) { + covered = true + } + } + assert.True(t, covered, "served proxy URL %s has no block-derivable CID in %v", url, blocked) + } + }) + } +} + +func TestServableEmbed_LeavesTheStoredRecordEmbedIntact(t *testing.T) { + stored := map[string]interface{}{"$type": TypeExternal, "external": map[string]interface{}{ + "uri": "https://example.com/article", "thumb": authorURL(presetEmbedThumbnail), + }} + encoded, err := json.Marshal(stored) + require.NoError(t, err) + var decoded interface{} + require.NoError(t, json.Unmarshal(encoded, &decoded)) + + ServableEmbed(decoded) + + // ServableEmbed may edit the value it is handed; the record path decodes + // its own copy, so the bytes it serves are untouched. + var record interface{} + require.NoError(t, json.Unmarshal(encoded, &record)) + assert.Equal(t, stored, record) +} + +func TestIsBlueskyPostURI(t *testing.T) { + for uri, want := range map[string]bool{ + "at://did:plc:quoted/app.bsky.feed.post/3kquoted": true, + "at://quoted.example.com/app.bsky.feed.post/3kquoted": true, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted": false, + "at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted": false, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted#/app.bsky.feed.post/1": false, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted?/app.bsky.feed.post/1": false, + "at://app.bsky.feed.post/social.coves.community.postv2/3kquoted": false, + "at://did:plc:quoted/app.bsky.feed.post": false, + "at://did:plc:quoted/app.bsky.feed.post/": false, + "https://did:plc:quoted/app.bsky.feed.post/3kquoted": false, + "": false, + } { + assert.Equal(t, want, IsBlueskyPostURI(uri), uri) + } +} diff --git a/internal/core/embeds/view.go b/internal/core/embeds/view.go index 9f9412d..cc4c604 100644 --- a/internal/core/embeds/view.go +++ b/internal/core/embeds/view.go @@ -28,6 +28,7 @@ package embeds import ( "log/slog" + "github.com/bluesky-social/indigo/atproto/syntax" "github.com/ipfs/go-cid" "Coves/internal/core/blobs" @@ -84,8 +85,8 @@ type mutation func() // idempotent because a projected embed's $type is a #view type, which this // function does not act on. // -// social.coves.embed.post carries no blobs; it is projected to its own #view by -// posts.TransformPostEmbeds, which resolves the quoted record. +// social.coves.embed.post carries no blobs; Bluesky quotes are resolved by +// posts.TransformPostEmbeds, while Coves quotes retain only their strongRef. func HydrateView(embed map[string]interface{}, ownerDID, ownerPDSURL string) { if embed == nil { return @@ -186,8 +187,7 @@ func HydrateCommentView(embed map[string]interface{}, ownerDID, ownerPDSURL stri return } - // TypePost carries no blobs and is projected by posts.TransformPostEmbeds, - // which resolves the quoted record; anything other than images is outside + // TypePost carries no blobs; anything other than images is outside // the comment union entirely. if embedType, _ := embed["$type"].(string); embedType != TypeImages { return @@ -196,6 +196,132 @@ func HydrateCommentView(embed map[string]interface{}, ownerDID, ownerPDSURL stri HydrateView(embed, ownerDID, ownerPDSURL) } +// ServableEmbed reduces a stored embed to what a post or comment view may +// serve, and runs before HydrateView or HydrateCommentView projects it. It is +// the serving half of the rule moderation blocks depend on: the only media URLs +// a view carries are the ones HydrateView derives from blob references, which +// are exactly the CIDs PostBlobCIDs and CommentImageCIDs report for blocking. +// +// The stored embed is the author's record, and nothing on the firehose path +// validates it, so a URL string in it is author-written. Served verbatim it +// would point readers at an image no removal blocks, including one under the +// author's own DID that an illegal-content removal must refuse. The AppView +// itself never stores a media URL: post create rejects a string thumb, the +// unfurl path uploads thumbnails as blobs, and posts.TransformPostEmbeds +// resolves Bluesky quote previews at serve time, after this runs. +// +// So: +// - images, video and external keep their record shape, minus any field the +// record declares as a blob whose value is not an object (image, thumb, +// video, thumbnail) and minus the view-only thumb and fullsize on image +// entries, including an external gallery's. +// - A quote, record or #view, is rebuilt to its strongRef; see projectQuote. +// - Anything else is not served and yields nil: a stored #view of a media +// type, a type outside the record union, an untyped or non-object embed. +// +// It edits the embed it is given. Callers pass a freshly decoded copy; the +// verbatim record decodes its own and keeps every byte. +func ServableEmbed(stored interface{}) interface{} { + embed, isObject := stored.(map[string]interface{}) + if !isObject { + return nil + } + + switch embedType, _ := embed["$type"].(string); embedType { + case TypeImages: + dropViewImageURLs(embed["images"]) + case TypeVideo: + dropNonBlob(embed, "video") + dropNonBlob(embed, "thumbnail") + case TypeExternal: + dropNonBlob(embed, "external") + if external, isObject := embed["external"].(map[string]interface{}); isObject { + dropNonBlob(external, "thumb") + dropViewImageURLs(external["images"]) + } + case TypePost, TypePost + viewSuffix: + return projectQuote(embed) + default: + return nil + } + return embed +} + +// dropNonBlob deletes object[field] when it is present and not an object. A +// blob reference is always an object, so anything else in a blob position is +// an author-written value the view must not carry. +func dropNonBlob(object map[string]interface{}, field string) { + value, present := object[field] + if !present { + return + } + if _, isObject := value.(map[string]interface{}); !isObject { + delete(object, field) + } +} + +// dropViewImageURLs removes the view-only URL fields from every image entry in +// a record's image list. HydrateView writes thumb and fullsize from the entry's +// blob when it projects; left in place they would be served whenever it +// cannot. +func dropViewImageURLs(value interface{}) { + images, _ := value.([]interface{}) + for _, entry := range images { + image, isObject := entry.(map[string]interface{}) + if !isObject { + continue + } + dropNonBlob(image, "image") + delete(image, "thumb") + delete(image, "fullsize") + } +} + +// projectQuote rebuilds a quote embed, record or #view, to its strongRef. +// +// D-QUOTES: Coves-quoted posts are not hydrated server-side. The served quote +// is the strongRef only, so a removed quoted post leaves no copied content; +// post.get on its URI returns #moderatedPost. A Bluesky quote is rebuilt the +// same way: posts.TransformPostEmbeds resolves its preview at serve time from +// the record type, so a stored resolved object, which only an author can have +// written, is never served in its place. +// +// Decision: record.embed is not projected. The lexicon serves record as the +// quoter's own authored record verbatim, so its embed keeps every byte the +// quoter wrote, including preview fields it forged. None of that is +// AppView-held content of the quoted post; the AppView adds resolved only at +// serve time, only for Bluesky URIs, and only to the top-level embed. +func projectQuote(embed map[string]interface{}) map[string]interface{} { + projected := map[string]interface{}{"$type": TypePost} + post, _ := embed["post"].(map[string]interface{}) + strongRef := make(map[string]interface{}, 2) + if uri, ok := post["uri"].(string); ok && uri != "" { + strongRef["uri"] = uri + } + if cid, ok := post["cid"].(string); ok && cid != "" { + strongRef["cid"] = cid + } + if len(strongRef) > 0 { + projected["post"] = strongRef + } + return projected +} + +// blueskyPostCollection is the NSID of a Bluesky post record. +const blueskyPostCollection = "app.bsky.feed.post" + +// IsBlueskyPostURI reports whether uri is a syntactically valid AT-URI whose +// collection segment is app.bsky.feed.post and which names a record. The +// substring elsewhere in a URI (a Coves collection's record key path, a +// fragment, a handle authority) does not make it a Bluesky post. +func IsBlueskyPostURI(uri string) bool { + parsed, err := syntax.ParseATURI(uri) + if err != nil { + return false + } + return parsed.Collection().String() == blueskyPostCollection && parsed.RecordKey() != "" +} + // CommentImageCIDs returns the canonical CIDs of every image blob that // HydrateCommentView can serve through the image proxy for a comment embed. // Moderation blocks exactly these, so blocking and serving cannot drift apart. @@ -384,3 +510,44 @@ func blobCID(value interface{}) string { return "" } + +// PostBlobCIDs returns the canonical, first-seen CIDs of post blobs served +// through the image proxy. Malformed entries are skipped; video blobs are +// served directly by the PDS, so only their thumbnails are included. +func PostBlobCIDs(embed map[string]interface{}) []string { + var cids []string + seen := make(map[string]bool) + addBlob := func(value interface{}) { + parsed, err := cid.Decode(blobCID(value)) + if err != nil { + return + } + canonical := parsed.String() + if !seen[canonical] { + seen[canonical] = true + cids = append(cids, canonical) + } + } + addImages := func(value interface{}) { + images, _ := value.([]interface{}) + for _, entry := range images { + image, ok := entry.(map[string]interface{}) + if ok { + addBlob(image["image"]) + } + } + } + + switch embedType, _ := embed["$type"].(string); embedType { + case TypeImages: + addImages(embed["images"]) + case TypeVideo: + addBlob(embed["thumbnail"]) + case TypeExternal: + if external, ok := embed["external"].(map[string]interface{}); ok { + addBlob(external["thumb"]) + addImages(external["images"]) + } + } + return cids +} diff --git a/internal/core/moderation/fake_store_test.go b/internal/core/moderation/fake_store_test.go index 3b9af32..3f3ba7b 100644 --- a/internal/core/moderation/fake_store_test.go +++ b/internal/core/moderation/fake_store_test.go @@ -22,6 +22,7 @@ type inMemoryModerationIdempotencyKey struct { type inMemoryModerationState struct { indexedComments map[string]moderation.IndexedComment + indexedPosts map[string]moderation.IndexedPost versions map[string]int64 actions map[string]moderation.Action activeRemovals map[inMemoryModerationDecisionKey]string @@ -33,6 +34,7 @@ type inMemoryModerationState struct { func (state inMemoryModerationState) copy() inMemoryModerationState { working := inMemoryModerationState{ indexedComments: make(map[string]moderation.IndexedComment, len(state.indexedComments)), + indexedPosts: make(map[string]moderation.IndexedPost, len(state.indexedPosts)), versions: make(map[string]int64, len(state.versions)), actions: make(map[string]moderation.Action, len(state.actions)), activeRemovals: make(map[inMemoryModerationDecisionKey]string, len(state.activeRemovals)), @@ -44,6 +46,10 @@ func (state inMemoryModerationState) copy() inMemoryModerationState { comment.ImageCIDs = append([]string(nil), comment.ImageCIDs...) working.indexedComments[key] = comment } + for key, post := range state.indexedPosts { + post.BlobCIDs = append([]string(nil), post.BlobCIDs...) + working.indexedPosts[key] = post + } for key, version := range state.versions { working.versions[key] = version } @@ -78,6 +84,7 @@ func newInMemoryModerationStore(now time.Time) *inMemoryModerationStore { now: now, state: inMemoryModerationState{ indexedComments: make(map[string]moderation.IndexedComment), + indexedPosts: make(map[string]moderation.IndexedPost), versions: make(map[string]int64), actions: make(map[string]moderation.Action), activeRemovals: make(map[inMemoryModerationDecisionKey]string), @@ -161,6 +168,15 @@ func (transaction *inMemoryModerationTransaction) ReadIndexedComment(_ context.C return &comment, nil } +func (transaction *inMemoryModerationTransaction) ReadIndexedPost(_ context.Context, subjectURI string) (*moderation.IndexedPost, error) { + post, exists := transaction.state.indexedPosts[subjectURI] + if !exists { + return nil, moderation.ErrSubjectNotIndexed + } + post.BlobCIDs = append([]string(nil), post.BlobCIDs...) + return &post, nil +} + func (transaction *inMemoryModerationTransaction) GetAction(_ context.Context, actionID string) (*moderation.Action, error) { action, exists := transaction.state.actions[actionID] if !exists { diff --git a/internal/core/moderation/indexed_subject.go b/internal/core/moderation/indexed_subject.go new file mode 100644 index 0000000..cba65d5 --- /dev/null +++ b/internal/core/moderation/indexed_subject.go @@ -0,0 +1,55 @@ +package moderation + +import ( + "context" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type indexedSubject struct { + URI string + CID string + Collection string + AuthorDeleted bool + OwnerDID string + CommunityDID string + BlobCIDs []string +} + +type indexedSubjectReader interface { + ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + ReadIndexedPost(ctx context.Context, subjectURI string) (*IndexedPost, error) +} + +func readIndexedSubject(ctx context.Context, reader indexedSubjectReader, subjectURI string) (*indexedSubject, error) { + uri, err := syntax.ParseATURI(subjectURI) + if err != nil { + return nil, fmt.Errorf("%w: invalid subject URI: %w", ErrInvalidSubject, err) + } + collection := uri.Collection().String() + switch collection { + case CommentCollection: + comment, err := reader.ReadIndexedComment(ctx, subjectURI) + if err != nil || comment == nil { + return nil, err + } + return &indexedSubject{ + URI: comment.URI, CID: comment.CID, Collection: collection, + AuthorDeleted: comment.AuthorDeleted, OwnerDID: comment.OwnerDID, + CommunityDID: comment.CommunityDID, BlobCIDs: comment.ImageCIDs, + }, nil + case PostV2Collection, LegacyPostCollection: + post, err := reader.ReadIndexedPost(ctx, subjectURI) + if err != nil || post == nil { + return nil, err + } + return &indexedSubject{ + URI: post.URI, CID: post.CID, Collection: collection, + AuthorDeleted: post.AuthorDeleted, OwnerDID: post.OwnerDID, + CommunityDID: post.CommunityDID, BlobCIDs: post.BlobCIDs, + }, nil + default: + return nil, fmt.Errorf("%w: unsupported subject collection", ErrInvalidSubject) + } +} diff --git a/internal/core/moderation/media.go b/internal/core/moderation/media.go index d2cc529..30d0ccf 100644 --- a/internal/core/moderation/media.go +++ b/internal/core/moderation/media.go @@ -6,10 +6,11 @@ import ( "log/slog" ) -// MediaTransaction provides the operations needed to reconcile a comment's images. +// MediaTransaction provides the operations needed to reconcile a subject's images. type MediaTransaction interface { ActiveRemoval(ctx context.Context, authorityDID, subjectURI string) (*Action, error) ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + ReadIndexedPost(ctx context.Context, subjectURI string) (*IndexedPost, error) // InsertNewMediaBlocks inserts only blocks not already active for the action // and returns the blocks it inserted. InsertNewMediaBlocks(ctx context.Context, blocks []MediaBlock) ([]MediaBlock, error) @@ -41,11 +42,30 @@ func (r *MediaReconciler) ReconcileTx(ctx context.Context, tx *sql.Tx, subjectUR if err != nil || action == nil { return nil, err } - comment, err := bound.ReadIndexedComment(ctx, subjectURI) + subject, err := readIndexedSubject(ctx, bound, subjectURI) if err != nil { return nil, err } - return bound.InsertNewMediaBlocks(ctx, imageMediaBlocks(comment, action)) + if subject == nil { + return nil, ErrSubjectNotIndexed + } + return bound.InsertNewMediaBlocks(ctx, imageMediaBlocks(subject, action)) +} + +// ReconcileIncomingTx blocks blobs of incoming content the consumer did not +// index, such as a recreate of a deleted post whose tombstone is kept. The +// owner's repository still serves those blobs, and the stored row does not +// name them, so the caller passes the owner and CIDs from the incoming record. +func (r *MediaReconciler) ReconcileIncomingTx(ctx context.Context, tx *sql.Tx, subjectURI, ownerDID string, blobCIDs []string) ([]MediaBlock, error) { + if len(blobCIDs) == 0 { + return nil, nil + } + bound := r.binder.BindTransaction(tx) + action, err := bound.ActiveRemoval(ctx, r.instanceDID, subjectURI) + if err != nil || action == nil { + return nil, err + } + return bound.InsertNewMediaBlocks(ctx, imageMediaBlocks(&indexedSubject{OwnerDID: ownerDID, BlobCIDs: blobCIDs}, action)) } // Purge removes cached bytes of newly blocked blobs after commit. @@ -76,15 +96,15 @@ func purgeMediaBlocks(purger MediaPurger, blocks []MediaBlock) { } } -func imageMediaBlocks(comment *IndexedComment, action *Action) []MediaBlock { +func imageMediaBlocks(subject *indexedSubject, action *Action) []MediaBlock { var blocks []MediaBlock seen := make(map[string]bool) - for _, cid := range comment.ImageCIDs { + for _, cid := range subject.BlobCIDs { if seen[cid] { continue } seen[cid] = true - blocks = append(blocks, MediaBlock{OwnerDID: comment.OwnerDID, BlobCID: cid, ActionID: action.ID}) + blocks = append(blocks, MediaBlock{OwnerDID: subject.OwnerDID, BlobCID: cid, ActionID: action.ID}) if action.Reason == illegalContentReason { blocks = append(blocks, MediaBlock{BlobCID: cid, ActionID: action.ID}) } diff --git a/internal/core/moderation/mutation_validation.go b/internal/core/moderation/mutation_validation.go index 4202090..3630b01 100644 --- a/internal/core/moderation/mutation_validation.go +++ b/internal/core/moderation/mutation_validation.go @@ -31,9 +31,14 @@ func validateMutationFields(key, expectedVersion, reason, privateNote string) er return nil } -func validCommentStrongRef(ref StrongRef) bool { +func validContentStrongRef(ref StrongRef) bool { uri, err := syntax.ParseATURI(ref.URI) - if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" || uri.Collection().String() != CommentCollection { + if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" { + return false + } + switch uri.Collection().String() { + case CommentCollection, PostV2Collection, LegacyPostCollection: + default: return false } _, err = syntax.ParseCID(ref.CID) diff --git a/internal/core/moderation/post_rules_test.go b/internal/core/moderation/post_rules_test.go new file mode 100644 index 0000000..1cd4567 --- /dev/null +++ b/internal/core/moderation/post_rules_test.go @@ -0,0 +1,287 @@ +package moderation_test + +import ( + "context" + "database/sql" + "testing" + + "Coves/internal/core/moderation" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + postRulesAuthorDID = "did:plc:postauthor" + postRulesCommunityDID = "did:plc:postcommunity" + postRulesCID = "bafyreipostversion" + postRulesURI = "at://did:plc:postauthor/social.coves.community.postv2/3kabc" + legacyPostRulesURI = "at://did:plc:postcommunity/social.coves.community.post/3kabc" +) + +func newPostRulesScenario(uri, ownerDID string, blobCIDs []string) removeRulesScenario { + scenario := newRemoveRulesScenario() + scenario.store.state.indexedPosts[uri] = moderation.IndexedPost{ + URI: uri, CID: postRulesCID, OwnerDID: ownerDID, + CommunityDID: postRulesCommunityDID, BlobCIDs: blobCIDs, + } + scenario.reader.record = &moderation.IndexedRecord{URI: uri, CID: postRulesCID} + scenario.request.Subject = moderation.StrongRef{URI: uri, CID: postRulesCID} + return scenario +} + +func TestRemovePostContentRecordsCollectionAndBlocksMedia(t *testing.T) { + for _, test := range []struct { + name, uri, collection, ownerDID, reason string + blobCIDs []string + wantBlocks []moderation.MediaBlock + wantOwnerPurges []removeRulesOwnerPurge + wantBlobPurges []string + }{ + { + name: "author-owned postv2", uri: postRulesURI, collection: moderation.PostV2Collection, + ownerDID: postRulesAuthorDID, reason: removeRulesSpam, + blobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + wantBlocks: []moderation.MediaBlock{ + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage}, + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesSecondImage}, + }, + wantOwnerPurges: []removeRulesOwnerPurge{ + {postRulesAuthorDID, removeRulesFirstImage}, {postRulesAuthorDID, removeRulesSecondImage}, + }, + }, + { + name: "community-owned legacy post", uri: legacyPostRulesURI, collection: moderation.LegacyPostCollection, + ownerDID: postRulesCommunityDID, reason: removeRulesSpam, + blobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + wantBlocks: []moderation.MediaBlock{ + {OwnerDID: postRulesCommunityDID, BlobCID: removeRulesFirstImage}, + {OwnerDID: postRulesCommunityDID, BlobCID: removeRulesSecondImage}, + }, + wantOwnerPurges: []removeRulesOwnerPurge{ + {postRulesCommunityDID, removeRulesFirstImage}, {postRulesCommunityDID, removeRulesSecondImage}, + }, + }, + { + name: "illegal content also blocks every owner", uri: postRulesURI, collection: moderation.PostV2Collection, + ownerDID: postRulesAuthorDID, reason: removeRulesIllegal, + blobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + wantBlocks: []moderation.MediaBlock{ + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage}, + {BlobCID: removeRulesFirstImage}, + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesSecondImage}, + {BlobCID: removeRulesSecondImage}, + }, + wantOwnerPurges: []removeRulesOwnerPurge{ + {postRulesAuthorDID, removeRulesFirstImage}, {postRulesAuthorDID, removeRulesSecondImage}, + }, + wantBlobPurges: []string{removeRulesFirstImage, removeRulesSecondImage}, + }, + { + name: "post with no blobs", uri: postRulesURI, collection: moderation.PostV2Collection, + ownerDID: postRulesAuthorDID, reason: removeRulesSpam, + }, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newPostRulesScenario(test.uri, test.ownerDID, test.blobCIDs) + scenario.request.Reason = test.reason + scenario.purger.onPurge = func() { + require.NotEmpty(t, scenario.store.state.activeRemovals, "purge must follow the committed decision") + require.NotEmpty(t, scenario.store.state.mediaBlocks, "purge must follow committed block insertion") + } + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + require.NoError(t, err) + require.NotNil(t, result) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + assert.Equal(t, test.collection, result.Action.SubjectCollection) + assert.Equal(t, postRulesCommunityDID, result.Action.SubjectCommunityDID) + assert.Equal(t, *result.Action, scenario.store.state.actions[result.Action.ID]) + assert.Equal(t, result.Action.ID, scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, test.uri}]) + wantBlocks := make(map[moderation.MediaBlock]bool) + for _, block := range test.wantBlocks { + block.ActionID = result.Action.ID + wantBlocks[block] = true + } + assert.Equal(t, wantBlocks, scenario.store.state.mediaBlocks) + assert.Equal(t, test.wantOwnerPurges, scenario.purger.ownerPurges) + assert.Equal(t, test.wantBlobPurges, scenario.purger.blobPurges) + assert.Equal(t, len(test.blobCIDs) > 0, containsModerationWrite(scenario.store.writeCalls, "InsertMediaBlocks")) + }) + } +} + +func containsModerationWrite(writes []string, wanted string) bool { + for _, write := range writes { + if write == wanted { + return true + } + } + return false +} + +func TestRemovePostContentChecksIndexedRecord(t *testing.T) { + for _, test := range []struct { + name, requestedCID string + deleted, missing bool + wantError error + }{ + {name: "author-deleted post ignores stale strongRef", requestedCID: "bafyreistalepostversion", deleted: true}, + {name: "present post refuses stale CID", requestedCID: "bafyreistalepostversion", wantError: moderation.ErrContentChanged}, + {name: "never-indexed post", requestedCID: postRulesCID, missing: true, wantError: moderation.ErrSubjectNotFound}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newPostRulesScenario(postRulesURI, postRulesAuthorDID, nil) + scenario.request.Subject.CID = test.requestedCID + post := scenario.store.state.indexedPosts[postRulesURI] + post.AuthorDeleted = test.deleted + scenario.store.state.indexedPosts[postRulesURI] = post + scenario.reader.record.Deleted = test.deleted + if test.missing { + delete(scenario.store.state.indexedPosts, postRulesURI) + scenario.reader.record = nil + scenario.reader.err = moderation.ErrSubjectNotIndexed + } + result, err := scenario.service.RemoveContent(t.Context(), removeRulesAdminDID, scenario.request) + if test.wantError != nil { + require.ErrorIs(t, err, test.wantError) + assert.Nil(t, result) + assertRemoveRulesNoWrites(t, scenario) + return + } + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + assert.Equal(t, postRulesCID, result.Action.ObservedCID) + assert.Equal(t, moderation.RecordStateDeleted, result.State.RecordState) + assert.Nil(t, result.State.CurrentSubject) + }) + } +} + +func TestRestorePostContentReviewsCurrentPostAndDeactivatesBlocks(t *testing.T) { + for _, test := range []struct { + name string + noReview bool + deleted bool + staleCID bool + wantError error + }{ + {name: "present post requires review", noReview: true, wantError: moderation.ErrInvalidRequest}, + {name: "present post refuses stale review", staleCID: true, wantError: moderation.ErrContentChanged}, + {name: "present post accepts current review"}, + {name: "author-deleted post needs no review", deleted: true, noReview: true}, + } { + t.Run(test.name, func(t *testing.T) { + scenario := newPostRulesScenario(postRulesURI, postRulesAuthorDID, []string{removeRulesFirstImage}) + post := scenario.store.state.indexedPosts[postRulesURI] + post.AuthorDeleted = test.deleted + scenario.store.state.indexedPosts[postRulesURI] = post + scenario.reader.record.Deleted = test.deleted + removal := moderation.Action{ + ID: "post-removal", Action: moderation.ActionRemove, ActorDID: removeRulesAdminDID, + AuthorityDID: removeRulesInstanceDID, ScopeKind: moderation.ScopeInstance, + SubjectURI: postRulesURI, SubjectCollection: moderation.PostV2Collection, + SubjectCommunityDID: postRulesCommunityDID, ObservedCID: postRulesCID, + } + scenario.store.state.actions[removal.ID] = removal + scenario.store.state.activeRemovals[inMemoryModerationDecisionKey{removeRulesInstanceDID, postRulesURI}] = removal.ID + scenario.store.state.versions[postRulesURI] = 1 + block := moderation.MediaBlock{OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage, ActionID: removal.ID} + scenario.store.state.mediaBlocks[block] = true + request := moderation.RestoreContentRequest{ + ActionID: removal.ID, ExpectedVersion: "v1", IdempotencyKey: "restore-post", + Reason: removeRulesSpam, ReviewedSubject: &moderation.StrongRef{URI: postRulesURI, CID: postRulesCID}, + } + if test.noReview { + request.ReviewedSubject = nil + } + if test.staleCID { + request.ReviewedSubject.CID = "bafyreistalepostversion" + } + before := scenario.store.state.copy() + result, err := scenario.service.RestoreContent(t.Context(), restoreRulesAdminDID, request) + if test.wantError != nil { + require.ErrorIs(t, err, test.wantError) + assert.Nil(t, result) + assert.Equal(t, before, scenario.store.state) + return + } + require.NoError(t, err) + require.NotNil(t, result) + assert.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + assert.Equal(t, moderation.ActionRestore, result.Action.Action) + assert.Equal(t, removal.ID, result.Action.ReversesActionID) + assert.Equal(t, moderation.PostV2Collection, result.Action.SubjectCollection) + assert.Equal(t, postRulesCommunityDID, result.Action.SubjectCommunityDID) + assert.Equal(t, "v2", result.State.Version) + assert.Equal(t, moderation.ModerationStateClear, result.State.Moderation.State) + assert.Empty(t, scenario.store.state.activeRemovals) + assert.False(t, scenario.store.state.mediaBlocks[block], "restore deactivates its removal's blob block") + if test.deleted { + assert.Equal(t, moderation.RecordStateDeleted, result.State.RecordState) + assert.Nil(t, result.State.CurrentSubject) + } else { + assert.Equal(t, moderation.RecordStatePresent, result.State.RecordState) + assert.Equal(t, &moderation.StrongRef{URI: postRulesURI, CID: postRulesCID}, result.State.CurrentSubject) + } + }) + } +} + +type postRulesMediaTransaction struct { + post moderation.IndexedPost + action moderation.Action + calls []string + blocks []moderation.MediaBlock +} + +func (transaction *postRulesMediaTransaction) ActiveRemoval(context.Context, string, string) (*moderation.Action, error) { + transaction.calls = append(transaction.calls, "ActiveRemoval") + return &transaction.action, nil +} + +func (transaction *postRulesMediaTransaction) ReadIndexedComment(context.Context, string) (*moderation.IndexedComment, error) { + transaction.calls = append(transaction.calls, "ReadIndexedComment") + return &moderation.IndexedComment{}, nil +} + +func (transaction *postRulesMediaTransaction) ReadIndexedPost(context.Context, string) (*moderation.IndexedPost, error) { + transaction.calls = append(transaction.calls, "ReadIndexedPost") + return &transaction.post, nil +} + +func (transaction *postRulesMediaTransaction) InsertNewMediaBlocks(_ context.Context, blocks []moderation.MediaBlock) ([]moderation.MediaBlock, error) { + transaction.calls = append(transaction.calls, "InsertNewMediaBlocks") + transaction.blocks = append(transaction.blocks, blocks...) + return blocks, nil +} + +type postRulesMediaBinder struct{ bound *postRulesMediaTransaction } + +func (binder postRulesMediaBinder) BindTransaction(*sql.Tx) moderation.MediaTransaction { + return binder.bound +} + +func TestReconcileRemovedPostMediaReadsIndexedPost(t *testing.T) { + bound := &postRulesMediaTransaction{ + post: moderation.IndexedPost{ + URI: postRulesURI, CID: postRulesCID, OwnerDID: postRulesAuthorDID, + BlobCIDs: []string{removeRulesFirstImage, removeRulesSecondImage}, + }, + action: moderation.Action{ID: "post-removal", Reason: removeRulesSpam}, + } + reconciler := moderation.NewMediaReconciler(postRulesMediaBinder{bound}, removeRulesInstanceDID, nil) + blocks, err := reconciler.ReconcileTx(t.Context(), nil, postRulesURI) + require.Equal(t, []string{"ActiveRemoval", "ReadIndexedPost", "InsertNewMediaBlocks"}, bound.calls) + require.NoError(t, err) + want := []moderation.MediaBlock{ + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesFirstImage, ActionID: bound.action.ID}, + {OwnerDID: postRulesAuthorDID, BlobCID: removeRulesSecondImage, ActionID: bound.action.ID}, + } + assert.Equal(t, want, bound.blocks) + assert.Equal(t, want, blocks) +} + +var _ moderation.MediaTransaction = (*postRulesMediaTransaction)(nil) diff --git a/internal/core/moderation/remove.go b/internal/core/moderation/remove.go index 55c0d77..5ea4d39 100644 --- a/internal/core/moderation/remove.go +++ b/internal/core/moderation/remove.go @@ -23,12 +23,10 @@ var removeReasons = map[string]struct{}{ func validateRemoveRequest(request RemoveContentRequest) error { uri, err := syntax.ParseATURI(request.Subject.URI) if err != nil || !uri.Authority().IsDID() || uri.RecordKey().String() == "" { - return fmt.Errorf("%w: expected a comment record URI with a DID authority", ErrInvalidSubject) + return fmt.Errorf("%w: expected a content record URI with a DID authority", ErrInvalidSubject) } switch uri.Collection().String() { - case PostV2Collection, LegacyPostCollection: - return fmt.Errorf("%w: post removal is unsupported", ErrInvalidSubject) - case CommentCollection: + case CommentCollection, PostV2Collection, LegacyPostCollection: default: return fmt.Errorf("%w: unsupported subject collection", ErrInvalidSubject) } @@ -85,17 +83,17 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re if request.ExpectedVersion != versionToken(version) { return fail(ErrStateConflict) } - comment, err := tx.ReadIndexedComment(ctx, request.Subject.URI) + subject, err := readIndexedSubject(ctx, tx, request.Subject.URI) if errors.Is(err, ErrSubjectNotIndexed) { return fail(ErrSubjectNotFound) } if err != nil { return unavailable(err) } - if comment == nil { - return unavailable(errors.New("indexed comment missing")) + if subject == nil { + return unavailable(errors.New("indexed subject missing")) } - if !comment.AuthorDeleted && comment.CID != request.Subject.CID { + if !subject.AuthorDeleted && subject.CID != request.Subject.CID { return fail(ErrContentChanged) } active, err := tx.ActiveRemoval(ctx, s.config.InstanceDID, request.Subject.URI) @@ -104,10 +102,10 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re } recordState := RecordStatePresent var current *StrongRef - if comment.AuthorDeleted { + if subject.AuthorDeleted { recordState = RecordStateDeleted } else { - current = &StrongRef{URI: comment.URI, CID: comment.CID} + current = &StrongRef{URI: subject.URI, CID: subject.CID} } if active != nil { state, err := newSubjectState(request.Subject.URI, version, recordState, current, active, s.config.InstanceDID) @@ -119,8 +117,8 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re action, err := tx.InsertAction(ctx, Action{ ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, ScopeKind: ScopeInstance, SubjectURI: request.Subject.URI, - SubjectCollection: CommentCollection, SubjectCommunityDID: comment.CommunityDID, - ObservedCID: comment.CID, Action: ActionRemove, Reason: request.Reason, + SubjectCollection: subject.Collection, SubjectCommunityDID: subject.CommunityDID, + ObservedCID: subject.CID, Action: ActionRemove, Reason: request.Reason, PrivateNote: request.PrivateNote, Origin: OriginLocal, CreatedAt: now, }) if err != nil { @@ -135,7 +133,7 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re if err := tx.SetSubjectVersion(ctx, request.Subject.URI, version+1); err != nil { return unavailable(err) } - newlyBlocked = imageMediaBlocks(comment, action) + newlyBlocked = imageMediaBlocks(subject, action) if len(newlyBlocked) > 0 { if err := tx.InsertMediaBlocks(ctx, newlyBlocked); err != nil { return unavailable(err) diff --git a/internal/core/moderation/remove_rules_test.go b/internal/core/moderation/remove_rules_test.go index 93abeaa..ad9e03c 100644 --- a/internal/core/moderation/remove_rules_test.go +++ b/internal/core/moderation/remove_rules_test.go @@ -33,14 +33,21 @@ type removeRulesOwnerPurge struct { type removeRulesPurger struct { ownerPurges []removeRulesOwnerPurge blobPurges []string + onPurge func() } func (purger *removeRulesPurger) PurgeOwnerBlob(ownerDID, blobCID string) error { + if purger.onPurge != nil { + purger.onPurge() + } purger.ownerPurges = append(purger.ownerPurges, removeRulesOwnerPurge{ownerDID, blobCID}) return nil } func (purger *removeRulesPurger) PurgeBlob(blobCID string) error { + if purger.onPurge != nil { + purger.onPurge() + } purger.blobPurges = append(purger.blobPurges, blobCID) return nil } diff --git a/internal/core/moderation/remove_validation_test.go b/internal/core/moderation/remove_validation_test.go index ec83006..ed20882 100644 --- a/internal/core/moderation/remove_validation_test.go +++ b/internal/core/moderation/remove_validation_test.go @@ -34,8 +34,7 @@ func TestRemoveContentRejectsUnsupportedSubjectsReasonsAndOversizedInputs(t *tes change func(*moderation.RemoveContentRequest) want error }{ - {name: "author-owned post removal is unsupported", collection: moderation.PostV2Collection, want: moderation.ErrInvalidSubject}, - {name: "legacy post removal is unsupported", collection: moderation.LegacyPostCollection, want: moderation.ErrInvalidSubject}, + {name: "non-content collection is unsupported", collection: "app.bsky.feed.post", want: moderation.ErrInvalidSubject}, {name: "unrecognized reason token", change: func(request *moderation.RemoveContentRequest) { request.Reason = "social.coves.moderation.defs#reasonCsam" }, want: moderation.ErrUnsupportedReason}, @@ -81,9 +80,6 @@ func TestRemoveContentRejectsUnsupportedSubjectsReasonsAndOversizedInputs(t *tes result, err := service.RemoveContent(t.Context(), actorDID, request) require.ErrorIs(t, err, test.want) assert.Nil(t, result) - if test.want == moderation.ErrInvalidSubject { - assert.Contains(t, err.Error(), "post", "post removal must be explicitly unsupported") - } assert.Empty(t, store.writeCalls, "validation must not attempt a mutation") assert.Empty(t, store.state.actions) assert.Empty(t, store.state.activeRemovals) diff --git a/internal/core/moderation/restore.go b/internal/core/moderation/restore.go index a86e204..0b6adb2 100644 --- a/internal/core/moderation/restore.go +++ b/internal/core/moderation/restore.go @@ -14,8 +14,8 @@ func validateRestoreRequest(request RestoreContentRequest) error { if err := validateMutationFields(request.IdempotencyKey, request.ExpectedVersion, request.Reason, request.PrivateNote); err != nil { return err } - if request.ReviewedSubject != nil && !validCommentStrongRef(*request.ReviewedSubject) { - return fmt.Errorf("%w: reviewedSubject must be a comment strongRef", ErrInvalidRequest) + if request.ReviewedSubject != nil && !validContentStrongRef(*request.ReviewedSubject) { + return fmt.Errorf("%w: reviewedSubject must be a content strongRef", ErrInvalidRequest) } return nil } @@ -94,7 +94,7 @@ func (s *service) restoreContent(ctx context.Context, actorDID string, request R if request.ReviewedSubject != nil && reviewedURI != removal.SubjectURI { return fail(fmt.Errorf("%w: reviewedSubject URI differs from removal subject", ErrInvalidRequest)) } - comment, err := tx.ReadIndexedComment(ctx, removal.SubjectURI) + subject, err := readIndexedSubject(ctx, tx, removal.SubjectURI) if err != nil && !errors.Is(err, ErrSubjectNotIndexed) { return unavailable(err) } @@ -102,19 +102,19 @@ func (s *service) restoreContent(ctx context.Context, actorDID string, request R var current *StrongRef var observedCID string if err == nil { - if comment == nil { - return unavailable(errors.New("indexed comment lookup returned no comment")) + if subject == nil { + return unavailable(errors.New("indexed subject lookup returned no subject")) } - observedCID = comment.CID - if comment.AuthorDeleted { + observedCID = subject.CID + if subject.AuthorDeleted { recordState = RecordStateDeleted } else { recordState = RecordStatePresent - current = &StrongRef{URI: comment.URI, CID: comment.CID} + current = &StrongRef{URI: subject.URI, CID: subject.CID} if request.ReviewedSubject == nil { - return fail(fmt.Errorf("%w: reviewedSubject is required for a present comment", ErrInvalidRequest)) + return fail(fmt.Errorf("%w: reviewedSubject is required for present content", ErrInvalidRequest)) } - if reviewedCID != comment.CID { + if reviewedCID != subject.CID { return fail(ErrContentChanged) } } diff --git a/internal/core/moderation/store.go b/internal/core/moderation/store.go index 8a5281a..cd49ad0 100644 --- a/internal/core/moderation/store.go +++ b/internal/core/moderation/store.go @@ -57,6 +57,20 @@ type IndexedComment struct { ImageCIDs []string } +// IndexedPost is the indexed post row a mutation inspects, read under a share +// lock so consumer writes serialize against the CID check. +type IndexedPost struct { + URI string + CID string + AuthorDeleted bool + // OwnerDID is the repository holding the post's blobs: the author for + // postv2, the community for legacy posts. + OwnerDID string + CommunityDID string + // BlobCIDs are the canonical CIDs of the post's proxy-served blobs. + BlobCIDs []string +} + // MediaBlock suppresses Coves-served bytes of a blob. An empty OwnerDID // blocks the CID for every owner. type MediaBlock struct { @@ -102,6 +116,8 @@ type Transaction interface { LockSubject(ctx context.Context, subjectURI string) (int64, error) // ReadIndexedComment returns ErrSubjectNotIndexed for a never-indexed URI. ReadIndexedComment(ctx context.Context, subjectURI string) (*IndexedComment, error) + // ReadIndexedPost returns ErrSubjectNotIndexed for a never-indexed URI. + ReadIndexedPost(ctx context.Context, subjectURI string) (*IndexedPost, error) // GetAction returns ErrDecisionNotFound for an unknown id. GetAction(ctx context.Context, actionID string) (*Action, error) // ActiveRemoval returns the active removal action, or nil. diff --git a/internal/core/posts/blob_transform.go b/internal/core/posts/blob_transform.go index 3ed2a4c..893c6f1 100644 --- a/internal/core/posts/blob_transform.go +++ b/internal/core/posts/blob_transform.go @@ -115,16 +115,9 @@ func TransformPostEmbeds(ctx context.Context, postView *PostView, blueskyService return } - // Only process app.bsky.feed.post URIs (Bluesky posts) - // Format: at://did:plc:xxx/app.bsky.feed.post/abc123 - if len(atURI) < 20 || atURI[:5] != "at://" { - log.Printf("[DEBUG] [TRANSFORM-EMBED] Skipping: invalid AT-URI format: %s", atURI) - return - } - - // Simple check for app.bsky.feed.post collection - // We don't want to process other types of embeds (e.g., Coves posts) - if !strings.Contains(atURI, "/app.bsky.feed.post/") { + // Only a URI whose collection segment is app.bsky.feed.post is a Bluesky + // post; the substring elsewhere (e.g. in a Coves post's path) is not. + if !embeds.IsBlueskyPostURI(atURI) { log.Printf("[DEBUG] [TRANSFORM-EMBED] Skipping: not a Bluesky post (URI: %s)", atURI) return } diff --git a/internal/core/posts/blob_transform_quote_uri_test.go b/internal/core/posts/blob_transform_quote_uri_test.go new file mode 100644 index 0000000..99cedd9 --- /dev/null +++ b/internal/core/posts/blob_transform_quote_uri_test.go @@ -0,0 +1,54 @@ +package posts + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "Coves/internal/core/blueskypost" +) + +// recordingBlueskyResolver records every URI it is asked to resolve. +type recordingBlueskyResolver struct{ resolved []string } + +func (r *recordingBlueskyResolver) ResolvePost(_ context.Context, uri string) (*blueskypost.BlueskyPostResult, error) { + r.resolved = append(r.resolved, uri) + return &blueskypost.BlueskyPostResult{URI: uri, Text: "resolved"}, nil +} + +func (r *recordingBlueskyResolver) ParseBlueskyURL(context.Context, string) (string, error) { + return "", nil +} + +func (r *recordingBlueskyResolver) IsBlueskyURL(string) bool { return false } + +// Only a URI whose collection segment is app.bsky.feed.post is sent to the +// Bluesky resolver; the substring elsewhere in a URI is not a Bluesky post. +func TestTransformPostEmbeds_ResolvesOnlyBlueskyCollectionURIs(t *testing.T) { + for uri, wantResolved := range map[string]bool{ + "at://did:plc:quoted/app.bsky.feed.post/3kquoted": true, + "at://did:plc:quoted/social.coves.community.postv2/app.bsky.feed.post/3kquoted": false, + "at://did:plc:quoted/social.coves.community.postv2/3kquoted#/app.bsky.feed.post/1": false, + "at://app.bsky.feed.post/social.coves.community.postv2/3kquoted": false, + } { + t.Run(uri, func(t *testing.T) { + resolver := &recordingBlueskyResolver{} + embed := map[string]interface{}{ + "$type": "social.coves.embed.post", "post": map[string]interface{}{"uri": uri, "cid": "bafyquoted"}, + } + TransformPostEmbeds(t.Context(), &PostView{Embed: embed}, resolver) + + if wantResolved { + assert.Equal(t, []string{uri}, resolver.resolved) + assert.Equal(t, "social.coves.embed.post#view", embed["$type"]) + assert.Contains(t, embed, "resolved") + return + } + assert.Empty(t, resolver.resolved, "a non-Bluesky collection must not reach the Bluesky resolver") + assert.Equal(t, map[string]interface{}{ + "$type": "social.coves.embed.post", "post": map[string]interface{}{"uri": uri, "cid": "bafyquoted"}, + }, embed) + }) + } +} diff --git a/internal/core/posts/interfaces.go b/internal/core/posts/interfaces.go index a2c0dbc..05d3991 100644 --- a/internal/core/posts/interfaces.go +++ b/internal/core/posts/interfaces.go @@ -113,6 +113,8 @@ type Repository interface { // GetRawIndexedRowsByURIs is the batched GetRawIndexedRow: same ungated raw // rows, one round trip. THE SAME DANGER APPLIES — read the banner above // before calling it. URIs with no indexed row are absent from the map. + // Rows also carry community handle/name when available; a missing community + // row does not remove an indexed post from the result. // // It exists because the post.get removal path needs the community and // soft-delete state of every absent URI in a caller-supplied batch, and @@ -120,6 +122,15 @@ type Repository interface { // list the caller controls. GetRawIndexedRowsByURIs(ctx context.Context, uris []string) (map[string]*Post, error) + // ActiveRemovalsByURIs returns the active removal sources of each URI that has any. + ActiveRemovalsByURIs(ctx context.Context, uris []string) (map[string][]RemovalSource, error) + + // AdmittedURIsForViewer returns the URIs whose posts pass the viewer-bound + // admission rule of the read-path visibility predicate, ignoring active + // moderation removals. viewerDID is "" for an anonymous read. post.get uses + // it so a removal tombstone never discloses a post the viewer could not see. + AdmittedURIsForViewer(ctx context.Context, uris []string, viewerDID string) (map[string]bool, error) + // GetViewsByURIs retrieves full post views (with author + community joins) for a // set of canonical DID-based AT-URIs. Returns a map keyed by URI; missing or // soft-deleted posts are simply absent from the map. Backs social.coves.community.post.get. diff --git a/internal/core/posts/post.go b/internal/core/posts/post.go index bd54deb..f22eefe 100644 --- a/internal/core/posts/post.go +++ b/internal/core/posts/post.go @@ -1,6 +1,7 @@ package posts import ( + "encoding/json" "time" ) @@ -34,11 +35,14 @@ type Post struct { RKey string `json:"rkey" db:"rkey"` URI string `json:"uri" db:"uri"` AuthorDID string `json:"authorDid" db:"author_did"` - ID int64 `json:"id" db:"id"` - UpvoteCount int `json:"upvoteCount" db:"upvote_count"` - DownvoteCount int `json:"downvoteCount" db:"downvote_count"` - Score int `json:"score" db:"score"` - CommentCount int `json:"commentCount" db:"comment_count"` + // Set by the batched raw read for content-free moderation tombstones. + CommunityHandle string `json:"-"` + CommunityName string `json:"-"` + ID int64 `json:"id" db:"id"` + UpvoteCount int `json:"upvoteCount" db:"upvote_count"` + DownvoteCount int `json:"downvoteCount" db:"downvote_count"` + Score int `json:"score" db:"score"` + CommentCount int `json:"commentCount" db:"comment_count"` // Bridge-asserted origin-platform vote aggregates for federated/bridged content. // Populated from the record's bridgedStats field; kept separate from native votes. @@ -193,17 +197,61 @@ type RemovedPost struct { Code string `json:"code,omitempty"` } +// RemovalSource attributes one active removal decision on a post. +type RemovalSource struct { + AuthorityDID string + ScopeKind string +} + +// ModerationView is a post's public removal state +// (social.coves.moderation.defs#moderationView). +type ModerationView struct { + State string `json:"state"` + Sources []ModerationSourceView `json:"sources,omitempty"` +} + +// ModerationSourceView attributes a removal (social.coves.moderation.defs#sourceView). +type ModerationSourceView struct { + AuthorityDID string `json:"authorityDid"` + Scope ModerationScopeView `json:"scope"` +} + +// ModerationScopeView is a removal's scope (social.coves.moderation.defs#scopeView). +type ModerationScopeView struct { + Kind string `json:"kind"` +} + +// ModeratedPost is the content-free social.coves.community.post.defs#moderatedPost +// tombstone for a post under an instance-scoped removal. +type ModeratedPost struct { + Moderation *ModerationView `json:"moderation"` + Community *CommunityRef `json:"community,omitempty"` + URI string `json:"uri"` + AuthorDID string `json:"authorDid,omitempty"` +} + +// MarshalJSON identifies this tombstone as the moderatedPost union member. +func (p ModeratedPost) MarshalJSON() ([]byte, error) { + type wirePost ModeratedPost + return json.Marshal(struct { + Type string `json:"$type"` + wirePost + }{Type: "social.coves.community.post.defs#moderatedPost", wirePost: wirePost(p)}) +} + // PostResult is one ordered element of a GetPosts response. Exactly one of Post, -// Blocked, Removed, or NotFound is set: Post when the post was found and visible -// to the viewer, Blocked when the viewer has blocked the author, Removed when the -// post's own community removed it, NotFound when the URI could not be resolved. +// Blocked, Removed, Moderated, or NotFound is set: Post when visible, Blocked +// when the viewer has blocked the author, Removed when the post's own community +// removed it, Moderated when an active instance removal hides it, and NotFound +// when the URI could not be resolved or the author deleted it. // Construct results via the result helpers so the const discriminators // (notFound/blocked/removed == true) cannot be left unset. type PostResult struct { - Post *PostView - Blocked *BlockedPost - Removed *RemovedPost - NotFound *NotFoundPost + Post *PostView + Blocked *BlockedPost + Removed *RemovedPost + NotFound *NotFoundPost + Moderated *ModeratedPost } // foundResult builds a found (postView) union member. @@ -222,6 +270,25 @@ func removedResult(uri, code string) *PostResult { return &PostResult{Removed: &RemovedPost{URI: uri, Removed: true, Code: code}} } +// moderatedResult builds a content-free removal tombstone from indexed metadata. +func moderatedResult(post *Post, sources []RemovalSource) *PostResult { + viewSources := make([]ModerationSourceView, 0, len(sources)) + for _, source := range sources { + viewSources = append(viewSources, ModerationSourceView{ + AuthorityDID: source.AuthorityDID, + Scope: ModerationScopeView{Kind: source.ScopeKind}, + }) + } + result := &ModeratedPost{ + URI: post.URI, AuthorDID: post.AuthorDID, + Moderation: &ModerationView{State: "removed", Sources: viewSources}, + } + if post.CommunityDID != "" && post.CommunityName != "" { + result.Community = &CommunityRef{DID: post.CommunityDID, Handle: post.CommunityHandle, Name: post.CommunityName} + } + return &PostResult{Moderated: result} +} + // blockedByAuthorResult builds a blockedPost union member (blockedBy "author") with its // const discriminator set. func blockedByAuthorResult(uri, authorDID string) *PostResult { @@ -233,10 +300,9 @@ func blockedByAuthorResult(uri, authorDID string) *PostResult { }} } -// GetPost returns the underlying PostView (nil for blocked/not-found results), -// satisfying the viewer-state enrichment helper's FeedPostProvider interface. Blocked -// and not-found results carry no PostView, so they are skipped by viewer enrichment -// and embed transforms. +// GetPost returns the underlying PostView (nil for tombstones), satisfying the +// viewer-state enrichment helper's FeedPostProvider interface. Content-free +// results are skipped by viewer enrichment and embed transforms. func (r *PostResult) GetPost() *PostView { return r.Post } @@ -247,7 +313,7 @@ func (r *PostResult) GetPost() *PostView { // an assembly bug; reporting it (rather than silently picking one by priority) is the // whole point of this guard, so a mis-assembled result surfaces as an internal error // instead of emitting a null or ambiguous array entry that violates the lexicon's union -// (postView | blockedPost | notFoundPost). +// (postView | blockedPost | removedPost | moderatedPost | notFoundPost). func (r *PostResult) Member() (interface{}, bool) { var member interface{} count := 0 @@ -263,6 +329,10 @@ func (r *PostResult) Member() (interface{}, bool) { member = r.Removed count++ } + if r.Moderated != nil { + member = r.Moderated + count++ + } if r.NotFound != nil { member = r.NotFound count++ diff --git a/internal/core/posts/service.go b/internal/core/posts/service.go index e15652b..bfa5f9b 100644 --- a/internal/core/posts/service.go +++ b/internal/core/posts/service.go @@ -1307,7 +1307,7 @@ const MaxGetPostsURIs = 25 // 1. Validate the URI count (1..25) and that every URI is a well-formed DID-based URI. // A malformed or handle-based URI is a client error -> InvalidRequest, not a silent miss. // 2. Batch fetch views for the (deduped) URIs. -// 3. Assemble results in request order; valid-but-absent URIs become notFoundPost. +// 3. Resolve removal tombstones for absent URIs, then assemble in request order. // // Viewer state (vote) and embed/blob transforms are applied by the handler layer. func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*PostResult, error) { @@ -1347,13 +1347,36 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos return nil, fmt.Errorf("failed to fetch post views: %w", err) } - // 3. Assemble results in request order. A visible view is a postView; an - // absent URI is a notFoundPost — UNLESS its own community removed it, in - // which case it becomes a #removedPost tombstone carrying the removal code - // (PRD §3.4/§6.2). The visibility predicate hides a removed post from - // GetViewsByURIs exactly as it hides a pending one, so the removal is - // recovered here from the admission row rather than from the (absent) view. - removed, err := s.removedMarkers(ctx, req.URIs, views) + // 3. Resolve absent URIs in batches. Raw rows establish that a post was + // indexed and has not been withdrawn by its author; neither the raw content + // nor the admission row can override an active instance removal. + absent := make([]string, 0, len(unique)) + for _, uri := range unique { + if views[uri] == nil { + absent = append(absent, uri) + } + } + var postsByURI map[string]*Post + var removals map[string][]RemovalSource + var admitted map[string]bool + if len(absent) != 0 { + postsByURI, err = s.repo.GetRawIndexedRowsByURIs(ctx, absent) + if err != nil { + return nil, fmt.Errorf("failed to resolve removal state for post.get: %w", err) + } + removals, err = s.repo.ActiveRemovalsByURIs(ctx, absent) + if err != nil { + return nil, fmt.Errorf("failed to fetch active post removals: %w", err) + } + // An instance removal must not widen access: the #moderatedPost + // tombstone discloses the author and the community, so it is served + // only to a viewer the admission rule would have shown the post to. + admitted, err = s.repo.AdmittedURIsForViewer(ctx, absent, req.ViewerDID) + if err != nil { + return nil, fmt.Errorf("failed to resolve post admission for post.get: %w", err) + } + } + removed, err := s.removedMarkers(ctx, absent, postsByURI) if err != nil { return nil, err } @@ -1362,6 +1385,11 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos switch { case views[uri] != nil: results[i] = foundResult(views[uri]) + case postsByURI[uri] != nil && postsByURI[uri].DeletedAt == nil && len(removals[uri]) != 0 && (admitted[uri] || hasRemovedMarker(removed, uri)): + // A same-community removal marker already makes the post and its + // community public as #removedPost, so the instance tombstone may + // replace it without disclosing anything new. + results[i] = moderatedResult(postsByURI[uri], removals[uri]) default: if code, ok := removed[uri]; ok { results[i] = removedResult(uri, code) @@ -1383,8 +1411,15 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos return results, nil } -// removedMarkers returns, for the requested URIs absent from the visible view -// set, the removal code of any whose OWN community removed it — so post.get can +// hasRemovedMarker reports whether removedMarkers produced a same-community +// removal marker for uri. +func hasRemovedMarker(markers map[string]string, uri string) bool { + _, ok := markers[uri] + return ok +} + +// removedMarkers returns, for the absent URIs with indexed rows, the removal +// code of any whose OWN community removed it — so post.get can // serve a #removedPost tombstone (PRD §3.4) instead of collapsing a moderator // removal into an indistinguishable notFoundPost. The presence of a URI in the // returned map is the removed signal; the value is the code (possibly empty). @@ -1394,7 +1429,7 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos // That is a CONFIGURATION fact, known before any lookup runs, and it is the only // thing that silently degrades to notFound. // -// A LOOKUP FAILURE IS AN ERROR, NOT A NOTFOUND. Both lookups here used to be +// A LOOKUP FAILURE IS AN ERROR, NOT A NOTFOUND. The lookups used to be // best-effort: a database blip turned a standing removal into notFoundPost, so // the same request answered with a different union member depending on the // health of the database, and a client (or a moderator checking their own @@ -1402,27 +1437,9 @@ func (s *postService) GetPosts(ctx context.Context, req GetPostsRequest) ([]*Pos // out". post.get answering 5xx is the honest response to "we do not know"; // silently downgrading the tombstone is not, and it is unfalsifiable from the // wire. Callers propagate the error. -func (s *postService) removedMarkers(ctx context.Context, uris []string, views map[string]*PostView) (map[string]string, error) { +func (s *postService) removedMarkers(ctx context.Context, absent []string, postsByURI map[string]*Post) (map[string]string, error) { markers := make(map[string]string) - if s.admissions == nil { - return markers, nil - } - - // Collect the absent URIs once (deduped), then resolve their admissions in a - // single batched lookup rather than one round-trip per URI. - seen := make(map[string]struct{}, len(uris)) - absent := make([]string, 0, len(uris)) - for _, uri := range uris { - if views[uri] != nil { - continue // visible — not a candidate for a tombstone - } - if _, done := seen[uri]; done { - continue - } - seen[uri] = struct{}{} - absent = append(absent, uri) - } - if len(absent) == 0 { + if s.admissions == nil || len(absent) == 0 { return markers, nil } @@ -1431,21 +1448,6 @@ func (s *postService) removedMarkers(ctx context.Context, uris []string, views m return nil, fmt.Errorf("failed to resolve removal state for post.get: %w", err) } - // The post rows are fetched in ONE batched round trip. Looping a per-URI - // lookup here was an N+1 on a public endpoint whose URI list the caller - // controls: 25 URIs (MaxGetPostsURIs) meant up to 25 sequential queries per - // request, all of them for URIs the visibility predicate had already refused. - // - // These are RAW rows on purpose — the predicate has already hidden every URI - // in `absent`, so a gated read would return nothing and there would be no - // removal to report. The raw row is used for exactly two facts, both checked - // below and neither of them content: which community owns the post, and - // whether its author withdrew it. - postsByURI, err := s.repo.GetRawIndexedRowsByURIs(ctx, absent) - if err != nil { - return nil, fmt.Errorf("failed to resolve removal state for post.get: %w", err) - } - for _, uri := range absent { // A removal is an admission-state change, not a soft delete, so the post // row still stands and its own community — the key the admission is scoped diff --git a/internal/core/posts/service_author_posts_test.go b/internal/core/posts/service_author_posts_test.go index 1a63063..1a95ab2 100644 --- a/internal/core/posts/service_author_posts_test.go +++ b/internal/core/posts/service_author_posts_test.go @@ -51,6 +51,14 @@ func (m *mockRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []str return out, nil } +func (m *mockRepository) ActiveRemovalsByURIs(context.Context, []string) (map[string][]RemovalSource, error) { + return map[string][]RemovalSource{}, nil +} + +func (m *mockRepository) AdmittedURIsForViewer(context.Context, []string, string) (map[string]bool, error) { + return map[string]bool{}, nil +} + func (m *mockRepository) GetViewsByURIs(ctx context.Context, uris []string, viewerDID string) (map[string]*PostView, error) { m.getViewsByURIsCalls++ m.gotViewsViewerDID = viewerDID diff --git a/internal/db/postgres/comment_repo.go b/internal/db/postgres/comment_repo.go index ede49fe..401a8d2 100644 --- a/internal/db/postgres/comment_repo.go +++ b/internal/db/postgres/comment_repo.go @@ -459,6 +459,7 @@ func (r *postgresCommentRepo) ListByCommenter(ctx context.Context, commenterDID // Used for user profile comment history (social.coves.actor.getComments) // Supports optional community filtering and returns next page cursor // Uses chronological ordering (newest first) with composite key cursor for stable pagination +// Excludes comments removed directly or under a removed root before pagination. func (r *postgresCommentRepo) ListByCommenterWithCursor(ctx context.Context, req comments.ListByCommenterRequest) ([]*comments.Comment, *string, error) { // Parse cursor for pagination cursorFilter, cursorValues, err := r.parseCommenterCursor(req.Cursor) @@ -523,7 +524,7 @@ func (r *postgresCommentRepo) ListByCommenterWithCursor(ctx context.Context, req AND c.deleted_at IS NULL AND NOT EXISTS ( SELECT 1 FROM moderation_decisions d - WHERE d.subject_uri = c.uri AND d.kind = 'removal' AND d.active + WHERE d.subject_uri IN (c.uri, c.root_uri) AND d.kind = 'removal' AND d.active ) %s %s diff --git a/internal/db/postgres/discover_hot_moderation_race_test.go b/internal/db/postgres/discover_hot_moderation_race_test.go new file mode 100644 index 0000000..199919a --- /dev/null +++ b/internal/db/postgres/discover_hot_moderation_race_test.go @@ -0,0 +1,143 @@ +//go:build integration + +package postgres + +import ( + "context" + "database/sql" + "testing" + "time" + + "Coves/internal/core/discover" + "Coves/tests/testkit" + + "github.com/stretchr/testify/require" +) + +// insertDiscoverHotRemoval uses the same action/decision shape as +// moderationTransaction.SetRemovalDecision. It avoids reading the posts view +// while the hydration-boundary gate holds its advisory lock. +func insertDiscoverHotRemoval(t *testing.T, db *sql.DB, uri string) { + t.Helper() + actionID := "hot-removal-" + testkit.UniqueID(t) + _, err := db.ExecContext(t.Context(), ` + INSERT INTO moderation_actions + (id, actor_did, authority_did, scope_kind, subject_uri, subject_collection, + action, reason, origin, created_at) + VALUES ($1, 'did:plc:hotmoderator', 'did:plc:hotinstance', 'instance', $2, + 'social.coves.community.post', 'remove', + 'social.coves.moderation.defs#reasonSpam', 'local', NOW()) + `, actionID, uri) + require.NoError(t, err) + _, err = db.ExecContext(t.Context(), ` + INSERT INTO moderation_decisions + (authority_did, scope_kind, scope_community_did, subject_uri, kind, value, active_action_id, active) + VALUES ('did:plc:hotinstance', 'instance', NULL, $2, 'removal', NULL, $1, TRUE) + `, actionID, uri) + require.NoError(t, err) +} + +func TestDiscoverRepo_HotRefillsAfterInstanceRemovalAtHydrationBoundary(t *testing.T) { + for _, timing := range []string{"before-revalidation", "after-revalidation-before-hydration"} { + t.Run(timing, func(t *testing.T) { + db := testkit.DB(t) + ctx := t.Context() + const ( + authorDID = "did:plc:hotmoderationauthor" + lockClass = 1_126_644_054 + lockID = 1 + ) + createTestUser(t, db, "hot-moderation-author.test", authorDID) + communities := []string{"did:plc:hotmoderationa", "did:plc:hotmoderationb", "did:plc:hotmoderationc"} + for i, did := range communities { + createTestCommunity(t, db, did, "hot-moderation-"+string(rune('a'+i))+".test", authorDID) + } + createdAt := time.Now().Add(-4 * time.Hour).Truncate(time.Second) + seed := func(communityDID, rkey string, score int) string { + t.Helper() + uri := seedFilterablePost(t, db, communityDID, authorDID, rkey, createdAt) + _, err := db.ExecContext(ctx, `UPDATE posts SET score = $2, upvote_count = $2 WHERE uri = $1`, uri, score) + require.NoError(t, err) + return uri + } + a1 := seed(communities[0], "hotmoda1", 1_000_000_000) + removedA2 := seed(communities[0], "hotmoda2", 1_000_000) + a3 := seed(communities[0], "hotmoda3", 2_000) + b1 := seed(communities[1], "hotmodb1", 5_000) + c1 := seed(communities[2], "hotmodc1", 100) + + repo := NewDiscoverRepository(db, "hot-moderation-race-secret").(*postgresDiscoverRepo) + repo.discoverHotWorkDeadline = 10 * time.Second + first, cursor, err := repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 1}) + require.NoError(t, err) + require.Equal(t, []string{a1}, discoverURIs(first)) + require.NotNil(t, cursor) + + type pageResult struct { + feed []*discover.FeedViewPost + cursor *string + err error + } + var second pageResult + if timing == "before-revalidation" { + insertDiscoverHotRemoval(t, db, removedA2) + second.feed, second.cursor, second.err = repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 2, Cursor: cursor}) + } else { + _, err := db.ExecContext(ctx, ` + CREATE TABLE hot_moderation_gate (candidate_uri TEXT PRIMARY KEY, lock_class INTEGER, lock_id INTEGER); + CREATE FUNCTION hot_moderation_wait(candidate_uri TEXT) RETURNS BOOLEAN + LANGUAGE plpgsql VOLATILE AS $$ + DECLARE gate hot_moderation_gate%ROWTYPE; + BEGIN + SELECT * INTO gate FROM hot_moderation_gate WHERE hot_moderation_gate.candidate_uri = $1; + IF FOUND THEN PERFORM pg_advisory_xact_lock(gate.lock_class, gate.lock_id); END IF; + RETURN TRUE; + END; + $$; + ALTER TABLE posts RENAME TO hot_moderation_posts; + CREATE VIEW posts WITH (security_barrier = true) AS + SELECT stored.* FROM hot_moderation_posts stored WHERE hot_moderation_wait(stored.uri); + `) + require.NoError(t, err) + _, err = db.ExecContext(ctx, `INSERT INTO hot_moderation_gate VALUES ($1, $2, $3)`, removedA2, lockClass, lockID) + require.NoError(t, err) + lockConnection, err := db.Conn(ctx) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, lockConnection.Close()) }) + _, err = lockConnection.ExecContext(ctx, `SELECT pg_advisory_lock($1, $2)`, lockClass, lockID) + require.NoError(t, err) + locked := true + t.Cleanup(func() { + if locked { + _, unlockErr := lockConnection.ExecContext(context.Background(), `SELECT pg_advisory_unlock($1, $2)`, lockClass, lockID) + require.NoError(t, unlockErr) + } + }) + var holderPID int + require.NoError(t, lockConnection.QueryRowContext(ctx, `SELECT pg_backend_pid()`).Scan(&holderPID)) + results := make(chan pageResult, 1) + go func() { + feed, next, requestErr := repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 2, Cursor: cursor}) + results <- pageResult{feed: feed, cursor: next, err: requestErr} + }() + waitForDiscoverHotAdvisoryLock(t, db, holderPID) + insertDiscoverHotRemoval(t, db, removedA2) + _, err = lockConnection.ExecContext(ctx, `SELECT pg_advisory_unlock($1, $2)`, lockClass, lockID) + require.NoError(t, err) + locked = false + second = <-results + } + + require.NoError(t, second.err) + require.Equal(t, []string{b1, a3}, discoverURIs(second.feed), "removal must refill without penalizing A's diversity slot") + require.NotNil(t, second.cursor) + third, end, err := repo.GetDiscover(ctx, discover.GetDiscoverRequest{Sort: "hot", Limit: 2, Cursor: second.cursor}) + require.NoError(t, err) + require.Equal(t, []string{c1}, discoverURIs(third)) + require.Nil(t, end) + all := append(append(discoverURIs(first), discoverURIs(second.feed)...), discoverURIs(third)...) + require.Equal(t, []string{a1, b1, a3, c1}, all, "every eligible snapshot candidate exactly once in checkpoint order") + require.NotContains(t, all, removedA2) + }) + } +} diff --git a/internal/db/postgres/moderation_post_integration_test.go b/internal/db/postgres/moderation_post_integration_test.go new file mode 100644 index 0000000..0e74646 --- /dev/null +++ b/internal/db/postgres/moderation_post_integration_test.go @@ -0,0 +1,359 @@ +//go:build integration + +package postgres_test + +import ( + "database/sql" + "fmt" + "testing" + + "Coves/internal/core/communityFeeds" + "Coves/internal/core/discover" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/core/timeline" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/ipfs/go-cid" + "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const moderationPostCID = "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + +func indexedModerationPost(t *testing.T, db *sql.DB, collection, communityDID, authorDID, title, embed string) moderation.StrongRef { + t.Helper() + rkey := testkit.TID() + authority := authorDID + if collection == moderation.LegacyPostCollection { + authority = communityDID + } + subject := moderation.StrongRef{URI: "at://" + authority + "/" + collection + "/" + rkey, CID: moderationPostCID} + _, err := db.ExecContext(t.Context(), ` + INSERT INTO posts (uri, cid, rkey, author_did, community_did, title, content, embed, created_at) + VALUES ($1, $2, $3, $4, $5, $6, 'indexed post content', NULLIF($7, '')::jsonb, NOW()) + `, subject.URI, subject.CID, rkey, authorDID, communityDID, title, embed) + require.NoError(t, err) + if collection == moderation.PostV2Collection { + _, err = db.ExecContext(t.Context(), ` + INSERT INTO community_post_admissions + (community_did, post_uri, status, accepted_cid, evaluated_cid, last_community_rev, last_community_op_rank, created_at, updated_at) + VALUES ($1, $2, 'accepted', $3, $3, '3lqqqqqqqqqq2', 1, NOW(), NOW()) + `, communityDID, subject.URI, subject.CID) + require.NoError(t, err) + } + return subject +} + +func moderationPostActors(t *testing.T, db *sql.DB) (communityDID, authorDID string) { + t.Helper() + authorName := testkit.UniqueIDWithPrefix(t, "postowner") + authorDID = fixtures.DID(authorName) + fixtures.User(t, db, authorName+".test", authorDID) + communityName := testkit.UniqueIDWithPrefix(t, "postplace") + var err error + communityDID, err = fixtures.Community(t.Context(), db, communityName, "owner"+communityName) + require.NoError(t, err) + return communityDID, authorDID +} + +func removeIndexedModerationPost(t *testing.T, service moderation.Service, subject moderation.StrongRef) *moderation.MutationResult { + t.Helper() + result, err := service.RemoveContent(t.Context(), fixtures.DID("postremovaladmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "remove-post", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err, "an indexed post must be readable inside the moderation transaction") + require.NotNil(t, result) + require.Equal(t, moderation.OutcomeApplied, result.Outcome) + require.NotNil(t, result.Action) + return result +} + +func TestModerationPostRepositoryRemovalPersistence(t *testing.T) { + for _, scenario := range []struct { + name, collection, embed string + softDeleted bool + requestedCID string + wantCIDs []string + communityOwned bool + }{ + {name: "postv2 two images", collection: moderation.PostV2Collection, embed: moderationTwoImageEmbed(), wantCIDs: []string{moderationImageCIDOne, moderationImageCIDTwo}}, + {name: "legacy image belongs to community", collection: moderation.LegacyPostCollection, embed: fmt.Sprintf(`{"$type":"social.coves.embed.images","images":[{"image":{"ref":{"$link":"%s"}}}]}`, moderationImageCIDOne), wantCIDs: []string{moderationImageCIDOne}, communityOwned: true}, + {name: "postv2 without media", collection: moderation.PostV2Collection}, + {name: "author deleted postv2 with stale requested CID", collection: moderation.PostV2Collection, softDeleted: true, requestedCID: moderationImageCIDTwo}, + } { + t.Run(scenario.name, func(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + subject := indexedModerationPost(t, db, scenario.collection, communityDID, authorDID, "post to moderate", scenario.embed) + if scenario.softDeleted { + _, err := db.ExecContext(t.Context(), `UPDATE posts SET deleted_at = NOW() WHERE uri = $1`, subject.URI) + require.NoError(t, err) + } + requested := subject + if scenario.requestedCID != "" { + requested.CID = scenario.requestedCID + } + service := newPostgresModerationService(db) + removed := removeIndexedModerationPost(t, service, requested) + expectedState := moderation.RecordStatePresent + if scenario.softDeleted { + expectedState = moderation.RecordStateDeleted + } + assert.Equal(t, expectedState, removed.State.RecordState) + var collection, associatedCommunity, observedCID string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT subject_collection, subject_community_did, observed_cid + FROM moderation_actions WHERE id = $1 AND subject_uri = $2 + `, removed.Action.ID, subject.URI).Scan(&collection, &associatedCommunity, &observedCID)) + assert.Equal(t, scenario.collection, collection) + assert.Equal(t, communityDID, associatedCommunity) + assert.Equal(t, subject.CID, observedCID, "the indexed CID, even when the deleted subject was requested with a different CID") + var decisionCount int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_decisions WHERE subject_uri = $1 AND active`, subject.URI).Scan(&decisionCount)) + assert.Equal(t, 1, decisionCount) + var activeAction string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT active_action_id FROM moderation_decisions + WHERE subject_uri = $1 AND authority_did = $2 AND kind = 'removal' AND active + `, subject.URI, fixtures.InstanceDID()).Scan(&activeAction)) + assert.Equal(t, removed.Action.ID, activeAction) + + rows, err := db.QueryContext(t.Context(), `SELECT owner_did, blob_cid FROM moderation_media_blocks WHERE action_id = $1 AND active`, removed.Action.ID) + require.NoError(t, err) + blocks := map[string][]string{} + for rows.Next() { + var owner sql.NullString + var cid string + require.NoError(t, rows.Scan(&owner, &cid)) + require.True(t, owner.Valid, "spam must not create ownerless media blocks") + blocks[owner.String] = append(blocks[owner.String], cid) + } + require.NoError(t, rows.Err()) + require.NoError(t, rows.Close()) + owner := authorDID + if scenario.communityOwned { + owner = communityDID + } + if len(scenario.wantCIDs) == 0 { + assert.Empty(t, blocks) + } else { + require.Len(t, blocks, 1, "only the correct blob owner may be blocked") + assert.ElementsMatch(t, scenario.wantCIDs, blocks[owner]) + } + var originalContent, storedCID string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT content, cid FROM posts WHERE uri = $1`, subject.URI).Scan(&originalContent, &storedCID)) + assert.Equal(t, "indexed post content", originalContent, "instance removal is an overlay") + assert.Equal(t, subject.CID, storedCID) + + state, err := service.GetSubjectState(t.Context(), subject.URI) + require.NoError(t, err) + assert.Equal(t, expectedState, state.RecordState) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, removed.Action.ID, state.LocalRemoval.ActionID) + }) + } +} + +func TestModerationPostRepositoryNeverIndexed(t *testing.T) { + db := testkit.DB(t) + service := newPostgresModerationService(db) + missing := moderation.StrongRef{URI: "at://" + fixtures.DID("neverindexedpost") + "/" + moderation.PostV2Collection + "/" + testkit.TID(), CID: moderationPostCID} + result, err := service.RemoveContent(t.Context(), fixtures.DID("postremovaladmin"), moderation.RemoveContentRequest{ + Subject: missing, ExpectedVersion: "v0", IdempotencyKey: "missing-post", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + assert.ErrorIs(t, err, moderation.ErrSubjectNotFound, "a post URI never indexed must not be moderated") + assert.Nil(t, result) + for _, table := range []string{"moderation_actions", "moderation_decisions", "moderation_media_blocks", "moderation_idempotency_keys"} { + var count int + require.NoError(t, db.QueryRowContext(t.Context(), "SELECT count(*) FROM "+table).Scan(&count)) + assert.Zero(t, count, table) + } +} + +func TestModerationPostRepositoryReconcilesEditedImagesInTransaction(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "edited moderated post", moderationTwoImageEmbed()) + removed := removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + tx, err := db.BeginTx(t.Context(), nil) + require.NoError(t, err) + t.Cleanup(func() { _ = tx.Rollback() }) + digest, err := multihash.Sum([]byte("moderation edited post third image"), multihash.SHA2_256, -1) + require.NoError(t, err) + thirdCID := cid.NewCidV1(cid.Raw, digest).String() + thirdImage := fmt.Sprintf(`{"$type":"social.coves.embed.images","images":[{"image":{"ref":{"$link":"%s"}}},{"image":{"ref":{"$link":"%s"}}},{"image":{"ref":{"$link":"%s"}}}]}`, moderationImageCIDOne, moderationImageCIDTwo, thirdCID) + _, err = tx.ExecContext(t.Context(), `UPDATE posts SET embed = $1::jsonb WHERE uri = $2`, thirdImage, subject.URI) + require.NoError(t, err) + reconciler := moderation.NewMediaReconciler(postgres.NewModerationRepository(db), fixtures.InstanceDID(), nil) + blocks, err := reconciler.ReconcileTx(t.Context(), tx, subject.URI) + require.NoError(t, err, "the consumer's post media seam must read the edited post inside its transaction") + require.Len(t, blocks, 1) + assert.Equal(t, authorDID, blocks[0].OwnerDID) + assert.Equal(t, thirdCID, blocks[0].BlobCID) + require.NoError(t, tx.Commit()) + var count int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_media_blocks + WHERE action_id = $1 AND owner_did = $2 AND blob_cid = $3 AND active + `, removed.Action.ID, authorDID, thirdCID).Scan(&count)) + assert.Equal(t, 1, count) +} + +func TestModerationPostRepositoryRemovalVisibilityAndRestore(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + viewerName := testkit.UniqueIDWithPrefix(t, "postviewer") + viewerDID := fixtures.DID(viewerName) + fixtures.User(t, db, viewerName+".test", viewerDID) + _, err := db.ExecContext(t.Context(), `INSERT INTO community_subscriptions (user_did, community_did, subscribed_at) VALUES ($1, $2, NOW())`, viewerDID, communityDID) + require.NoError(t, err) + title := "moderation visibility signal" + removed := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, title, "") + control := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, title, "") + postRepo := postgres.NewPostRepository(db) + feedRepo := postgres.NewCommunityFeedRepository(db, "moderation-post-visibility-secret") + discoverRepo := postgres.NewDiscoverRepository(db, "moderation-post-visibility-secret") + timelineRepo := postgres.NewTimelineRepository(db, "moderation-post-visibility-secret") + service := newPostgresModerationService(db) + + check := func(stage string, want []string, count int) { + t.Helper() + checks := []struct { + name string + read func(*testing.T) []string + }{ + {"post.get anonymous", func(t *testing.T) []string { + views, err := postRepo.GetViewsByURIs(t.Context(), []string{removed.URI, control.URI}, "") + require.NoError(t, err) + return moderationPostViewURIs(views) + }}, + {"post.get author", func(t *testing.T) []string { + views, err := postRepo.GetViewsByURIs(t.Context(), []string{removed.URI, control.URI}, authorDID) + require.NoError(t, err) + return moderationPostViewURIs(views) + }}, + {"comment header anonymous", func(t *testing.T) []string { return moderationPostHeaderURI(t, postRepo, removed.URI, control.URI, "") }}, + {"comment header author", func(t *testing.T) []string { + return moderationPostHeaderURI(t, postRepo, removed.URI, control.URI, authorDID) + }}, + {"actor.getPosts anonymous", func(t *testing.T) []string { + views, _, err := postRepo.GetByAuthor(t.Context(), posts.GetAuthorPostsRequest{ActorDID: authorDID, Limit: 50}) + require.NoError(t, err) + var uris []string + for _, view := range views { + uris = append(uris, view.URI) + } + return uris + }}, + {"actor.getPosts author", func(t *testing.T) []string { + views, _, err := postRepo.GetByAuthor(t.Context(), posts.GetAuthorPostsRequest{ActorDID: authorDID, ViewerDID: authorDID, Limit: 50}) + require.NoError(t, err) + var uris []string + for _, view := range views { + uris = append(uris, view.URI) + } + return uris + }}, + {"timeline subscriber", func(t *testing.T) []string { + feed, _, err := timelineRepo.GetTimeline(t.Context(), timeline.GetTimelineRequest{UserDID: viewerDID, Sort: "new", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}, + {"search matching title", func(t *testing.T) []string { + feed, _, err := feedRepo.SearchPosts(t.Context(), communityFeeds.SearchPostsRequest{Query: "moderation visibility", Community: communityDID, Sort: "relevance", Timeframe: "all", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}, + } + for _, sort := range []string{"hot", "new", "top"} { + checks = append(checks, struct { + name string + read func(*testing.T) []string + }{"community feed " + sort, func(t *testing.T) []string { + feed, _, err := feedRepo.GetCommunityFeed(t.Context(), communityFeeds.GetCommunityFeedRequest{Community: communityDID, Sort: sort, Timeframe: "all", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}) + } + for _, sort := range []string{"new", "top"} { + checks = append(checks, struct { + name string + read func(*testing.T) []string + }{"discover " + sort, func(t *testing.T) []string { + feed, _, err := discoverRepo.GetDiscover(t.Context(), discover.GetDiscoverRequest{Sort: sort, Timeframe: "all", Limit: 50}) + require.NoError(t, err) + var uris []string + for _, item := range feed { + uris = append(uris, item.Post.URI) + } + return uris + }}) + } + for _, check := range checks { + t.Run(stage+"/"+check.name, func(t *testing.T) { + assert.ElementsMatch(t, want, check.read(t), "instance removal must hide the post from this read path, including its author") + }) + } + t.Run(stage+"/community postCount", func(t *testing.T) { + community, err := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()).GetByDID(t.Context(), communityDID) + require.NoError(t, err) + assert.Equal(t, count, community.PostCount) + }) + t.Run(stage+"/profile post count", func(t *testing.T) { + stats, err := postgres.NewUserRepository(db).GetProfileStats(t.Context(), authorDID) + require.NoError(t, err) + assert.Equal(t, count, stats.PostCount) + }) + } + check("before removal", []string{removed.URI, control.URI}, 2) + result := removeIndexedModerationPost(t, service, removed) + check("while removed", []string{control.URI}, 1) + restored, err := service.RestoreContent(t.Context(), fixtures.DID("postrestoreadmin"), moderation.RestoreContentRequest{ + ActionID: result.Action.ID, ReviewedSubject: &removed, ExpectedVersion: result.State.Version, + IdempotencyKey: "restore-post", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + check("after restore", []string{removed.URI, control.URI}, 2) +} + +func moderationPostViewURIs(views map[string]*posts.PostView) []string { + var uris []string + for uri := range views { + uris = append(uris, uri) + } + return uris +} + +func moderationPostHeaderURI(t *testing.T, repo *postgres.PostRepository, removed, control, viewer string) []string { + t.Helper() + var uris []string + for _, uri := range []string{removed, control} { + view, err := repo.VisibleHeaderView(t.Context(), uri, viewer) + require.NoError(t, err) + if view != nil { + uris = append(uris, view.URI) + } + } + return uris +} diff --git a/internal/db/postgres/moderation_post_tombstone_integration_test.go b/internal/db/postgres/moderation_post_tombstone_integration_test.go new file mode 100644 index 0000000..0387efd --- /dev/null +++ b/internal/db/postgres/moderation_post_tombstone_integration_test.go @@ -0,0 +1,269 @@ +//go:build integration + +package postgres_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "testing" + + "Coves/internal/core/communities" + "Coves/internal/core/moderation" + "Coves/internal/core/posts" + "Coves/internal/crypto/credentialcipher/credentialciphertest" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func moderationPostGetService(db *sql.DB, repo posts.Repository) posts.Service { + communityRepo := postgres.NewCommunityRepository(db, credentialciphertest.Fixed()) + communityService := communities.NewCommunityServiceWithPDSFactory( + communityRepo, testkit.Endpoints().PDS.BaseURL, fixtures.InstanceDID(), "", nil, nil, nil, + communities.PrivateHostOptions(true)..., + ) + return posts.NewPostService(repo, communityService, nil, nil, nil, nil, testkit.Endpoints().PDS.BaseURL, + posts.WithAdmissionPolicy(posts.NewAllowAllAdmissionPolicyForTests()), + posts.WithSyncAcceptance(postgres.NewAdmissionRepository(db), nil), + ) +} + +func moderationPostTombstoneActors(t *testing.T, db *sql.DB) (communityDID, authorDID string) { + t.Helper() + communityDID, _ = moderationPostActors(t, db) + authorDID = "did:plc:bbbbbbbbbbbbbbbbbbbbbbbb" + fixtures.User(t, db, "posttombstoneauthor.test", authorDID) + return communityDID, authorDID +} + +func restoreModerationPost(t *testing.T, service moderation.Service, subject moderation.StrongRef, removed *moderation.MutationResult) { + t.Helper() + restored, err := service.RestoreContent(t.Context(), fixtures.DID("postrestoreadmin"), moderation.RestoreContentRequest{ + ActionID: removed.Action.ID, ReviewedSubject: &subject, ExpectedVersion: removed.State.Version, + IdempotencyKey: "restore-post", Reason: "social.coves.moderation.defs#reasonModeratorDiscretion", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) +} + +func TestModerationPostActiveRemovalsByURIs(t *testing.T) { + db := testkit.DB(t) + repo := postgres.NewPostRepository(db) + communityDID, authorDID := moderationPostActors(t, db) + active := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "active decision", "") + restored := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "restored decision", "") + control := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "never removed", "") + service := newPostgresModerationService(db) + removeIndexedModerationPost(t, service, active) + previous, err := service.RemoveContent(t.Context(), fixtures.DID("postremovaladminb"), moderation.RemoveContentRequest{ + Subject: restored, ExpectedVersion: "v0", IdempotencyKey: "remove-restored-post", + Reason: "social.coves.moderation.defs#reasonSpam", + }) + require.NoError(t, err) + require.NotNil(t, previous) + require.NotNil(t, previous.Action) + restoreModerationPost(t, service, restored, previous) + + got, err := repo.ActiveRemovalsByURIs(t.Context(), []string{control.URI, active.URI, restored.URI, active.URI}) + require.NoError(t, err) + assert.Equal(t, map[string][]posts.RemovalSource{ + active.URI: {{AuthorityDID: fixtures.InstanceDID(), ScopeKind: "instance"}}, + }, got, "only active decisions may appear, even in a batch with restored and unremoved URIs") + empty, err := repo.ActiveRemovalsByURIs(t.Context(), nil) + require.NoError(t, err) + assert.NotNil(t, empty, "an empty lookup must return an empty map") + assert.Empty(t, empty) +} + +func TestModerationPostGetTombstone(t *testing.T) { + for _, scenario := range []struct { + name string + communityRemoved bool + deleted bool + }{ + {name: "accepted post"}, + {name: "instance removal overrides community removal", communityRemoved: true}, + {name: "author deletion overrides instance removal", deleted: true}, + } { + t.Run(scenario.name, func(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "private after removal", "") + if scenario.communityRemoved { + _, err := db.ExecContext(t.Context(), ` + UPDATE community_post_admissions SET status = 'removed', accepted_cid = NULL, + decision_code = 'rule-violation', decision_at = NOW(), updated_at = NOW() + WHERE community_did = $1 AND post_uri = $2 + `, communityDID, subject.URI) + require.NoError(t, err) + } + if scenario.deleted { + _, err := db.ExecContext(t.Context(), `UPDATE posts SET deleted_at = NOW() WHERE uri = $1`, subject.URI) + require.NoError(t, err) + } + moderationService := newPostgresModerationService(db) + postService := moderationPostGetService(db, postgres.NewPostRepository(db)) + if scenario.communityRemoved { + before, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + require.NoError(t, err) + require.Len(t, before, 1) + require.NotNil(t, before[0].Removed, "the community-removal fixture must serve #removedPost before instance removal") + } + removed := removeIndexedModerationPost(t, moderationService, subject) + for _, viewer := range []struct{ name, did string }{{"anonymous", ""}, {"author", authorDID}} { + t.Run(viewer.name, func(t *testing.T) { + results, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}, ViewerDID: viewer.did}) + require.NoError(t, err) + require.Len(t, results, 1) + if scenario.deleted { + require.NotNil(t, results[0].NotFound, "author deletion takes precedence over instance moderation") + assert.Nil(t, results[0].Moderated) + return + } + require.NotNil(t, results[0].Moderated, "instance removal must serve #moderatedPost instead of #notFoundPost or #removedPost") + assert.Nil(t, results[0].Post) + assert.Nil(t, results[0].Removed) + assert.Nil(t, results[0].NotFound) + assert.Nil(t, results[0].Blocked) + member, ok := results[0].Member() + assert.True(t, ok) + assert.Same(t, results[0].Moderated, member) + tombstone := results[0].Moderated + assert.Equal(t, subject.URI, tombstone.URI) + assert.Equal(t, authorDID, tombstone.AuthorDID) + require.NotNil(t, tombstone.Moderation) + assert.Equal(t, "removed", tombstone.Moderation.State) + require.Len(t, tombstone.Moderation.Sources, 1) + assert.Equal(t, fixtures.InstanceDID(), tombstone.Moderation.Sources[0].AuthorityDID) + assert.Equal(t, "instance", tombstone.Moderation.Sources[0].Scope.Kind) + require.NotNil(t, tombstone.Community) + assert.Equal(t, communityDID, tombstone.Community.DID) + var name, handle string + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT name, handle FROM communities WHERE did = $1`, communityDID).Scan(&name, &handle)) + assert.Equal(t, name, tombstone.Community.Name) + assert.Equal(t, handle, tombstone.Community.Handle) + assert.Nil(t, tombstone.Community.Avatar) + assert.Nil(t, tombstone.Community.Origin) + assert.Empty(t, tombstone.Community.PDSURL) + }) + } + if scenario.communityRemoved { + restoreModerationPost(t, moderationService, subject, removed) + after, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + require.NoError(t, err) + require.Len(t, after, 1) + require.NotNil(t, after[0].Removed, "restoring the instance removal must reveal the community's #removedPost again") + assert.Equal(t, "rule-violation", after[0].Removed.Code) + assert.Nil(t, after[0].Moderated) + } + }) + } +} + +func TestModerationPostGetBatchPreservesOrder(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + uris := make([]string, 25) + var subject moderation.StrongRef + for i := range uris { + post := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, fmt.Sprintf("post %d", i), "") + uris[i] = post.URI + if i == 12 { + subject = post + } + } + removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + results, err := moderationPostGetService(db, postgres.NewPostRepository(db)).GetPosts(t.Context(), posts.GetPostsRequest{URIs: uris}) + require.NoError(t, err) + require.Len(t, results, len(uris)) + for i, result := range results { + if i == 12 { + require.NotNil(t, result.Moderated, "the removed post must occupy its original slot in the 25-URI batch") + assert.Equal(t, uris[i], result.Moderated.URI) + } else { + require.NotNil(t, result.Post, "the remaining 24 posts must still be #postView at index %d", i) + assert.Equal(t, uris[i], result.Post.URI) + } + } +} + +type failingPostRemovalRepository struct { + *postgres.PostRepository + err error +} + +func (r failingPostRemovalRepository) ActiveRemovalsByURIs(context.Context, []string) (map[string][]posts.RemovalSource, error) { + return nil, r.err +} + +func TestModerationPostGetRemovalLookupFailure(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "removed subject", "") + removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + lookupFailure := errors.New("removal lookup failed") + repo := failingPostRemovalRepository{PostRepository: postgres.NewPostRepository(db), err: lookupFailure} + results, err := moderationPostGetService(db, repo).GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + assert.ErrorIs(t, err, lookupFailure, "a failed instance-removal lookup must fail post.get, never return #notFoundPost") + assert.Nil(t, results) +} + +// An instance removal must not widen access: a post the viewer could not see +// before the removal (pending, rejected, or a postv2 its community never +// admitted) stays #notFoundPost for that viewer afterwards, instead of becoming +// a #moderatedPost that discloses its author and ties it to the community it +// names. The author, who could see it before, gets the tombstone. +func TestModerationPostGetTombstoneDoesNotWidenAccess(t *testing.T) { + for _, scenario := range []struct { + name, admissionSQL string + }{ + {name: "pending postv2", admissionSQL: ` + UPDATE community_post_admissions SET status = 'pending', accepted_cid = NULL, updated_at = NOW() + WHERE community_did = $1 AND post_uri = $2`}, + {name: "rejected postv2", admissionSQL: ` + UPDATE community_post_admissions SET status = 'rejected', accepted_cid = NULL, + decision_code = 'off-topic', decision_at = NOW(), updated_at = NOW() + WHERE community_did = $1 AND post_uri = $2`}, + {name: "unadmitted postv2 naming the community", admissionSQL: ` + DELETE FROM community_post_admissions WHERE community_did = $1 AND post_uri = $2`}, + } { + t.Run(scenario.name, func(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostTombstoneActors(t, db) + otherViewerDID := fixtures.DID(testkit.UniqueIDWithPrefix(t, "otherviewer")) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "never public", "") + _, err := db.ExecContext(t.Context(), scenario.admissionSQL, communityDID, subject.URI) + require.NoError(t, err) + postService := moderationPostGetService(db, postgres.NewPostRepository(db)) + before, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}}) + require.NoError(t, err) + require.Len(t, before, 1) + require.NotNil(t, before[0].NotFound, "fixture: the post must be #notFoundPost to the public before the removal") + + removeIndexedModerationPost(t, newPostgresModerationService(db), subject) + + for _, viewer := range []struct{ name, did string }{{"anonymous", ""}, {"other viewer", otherViewerDID}} { + t.Run(viewer.name, func(t *testing.T) { + results, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}, ViewerDID: viewer.did}) + require.NoError(t, err) + require.Len(t, results, 1) + assert.Nil(t, results[0].Moderated, "an instance removal must not disclose a post this viewer could not see before it") + require.NotNil(t, results[0].NotFound) + assert.Equal(t, subject.URI, results[0].NotFound.URI) + }) + } + t.Run("author", func(t *testing.T) { + results, err := postService.GetPosts(t.Context(), posts.GetPostsRequest{URIs: []string{subject.URI}, ViewerDID: authorDID}) + require.NoError(t, err) + require.Len(t, results, 1) + require.NotNil(t, results[0].Moderated, "the author could see their own post before the removal, so they get the tombstone") + assert.Equal(t, subject.URI, results[0].Moderated.URI) + }) + }) + } +} diff --git a/internal/db/postgres/moderation_repo.go b/internal/db/postgres/moderation_repo.go index c92b521..699d2a7 100644 --- a/internal/db/postgres/moderation_repo.go +++ b/internal/db/postgres/moderation_repo.go @@ -151,6 +151,41 @@ func (t *moderationTransaction) LockSubject(ctx context.Context, subjectURI stri return version, err } +func (t *moderationTransaction) ReadIndexedPost(ctx context.Context, subjectURI string) (*moderation.IndexedPost, error) { + var post moderation.IndexedPost + var authorDID string + var embed sql.NullString + err := t.tx.QueryRowContext(ctx, ` + SELECT uri, cid, author_did, community_did, (deleted_at IS NOT NULL), embed + FROM posts WHERE uri = $1 FOR SHARE + `, subjectURI).Scan(&post.URI, &post.CID, &authorDID, &post.CommunityDID, &post.AuthorDeleted, &embed) + if errors.Is(err, sql.ErrNoRows) { + return nil, moderation.ErrSubjectNotIndexed + } + if err != nil { + return nil, err + } + uri, err := syntax.ParseATURI(post.URI) + if err != nil { + return nil, err + } + post.OwnerDID = authorDID + if uri.Collection().String() == moderation.LegacyPostCollection { + post.OwnerDID = post.CommunityDID + } + if embed.Valid { + // Post embeds are unvalidated indexed data; malformed shapes have no + // proxy-served blobs to block and must not prevent moderation. + var decoded any + if err := json.Unmarshal([]byte(embed.String), &decoded); err == nil { + if object, isObject := decoded.(map[string]any); isObject { + post.BlobCIDs = embeds.PostBlobCIDs(object) + } + } + } + return &post, nil +} + func (t *moderationTransaction) ReadIndexedComment(ctx context.Context, subjectURI string) (*moderation.IndexedComment, error) { var comment moderation.IndexedComment var communityDID, embed sql.NullString diff --git a/internal/db/postgres/post_repo.go b/internal/db/postgres/post_repo.go index acb79eb..96e20e5 100644 --- a/internal/db/postgres/post_repo.go +++ b/internal/db/postgres/post_repo.go @@ -13,6 +13,7 @@ import ( "Coves/internal/core/blobs" "Coves/internal/core/communities" + "Coves/internal/core/embeds" "Coves/internal/core/posts" "github.com/lib/pq" @@ -143,10 +144,10 @@ func (r *PostRepository) Create(ctx context.Context, post *posts.Post) error { // shared by GetRawIndexedRow and GetRawIndexedRowsByURIs so the two cannot drift // apart. It must stay byte-aligned with scanRawIndexedRow's positional Scan. const rawIndexedRowColumns = ` - id, uri, cid, rkey, author_did, community_did, - title, content, content_facets, embed, content_labels, - created_at, edited_at, indexed_at, deleted_at, - upvote_count + bridged_upvote_count AS upvote_count, downvote_count + bridged_downvote_count AS downvote_count, score, comment_count` + p.id, p.uri, p.cid, p.rkey, p.author_did, p.community_did, + p.title, p.content, p.content_facets, p.embed, p.content_labels, + p.created_at, p.edited_at, p.indexed_at, p.deleted_at, + p.upvote_count + p.bridged_upvote_count AS upvote_count, p.downvote_count + p.bridged_downvote_count AS downvote_count, p.score, p.comment_count` // ════════════════════════════════════════════════════════════════════════════ // DANGER — GetRawIndexedRow IS NOT A DISPLAY READ. @@ -173,10 +174,10 @@ const rawIndexedRowColumns = ` // ════════════════════════════════════════════════════════════════════════════ func (r *PostRepository) GetRawIndexedRow(ctx context.Context, uri string) (*posts.Post, error) { query := `SELECT` + rawIndexedRowColumns + ` - FROM posts - WHERE uri = $1` + FROM posts p + WHERE p.uri = $1` - post, err := scanRawIndexedRow(r.db.QueryRowContext(ctx, query, uri)) + post, err := scanRawIndexedRow(r.db.QueryRowContext(ctx, query, uri), false) if errors.Is(err, sql.ErrNoRows) { return nil, posts.ErrNotFound } @@ -186,13 +187,74 @@ func (r *PostRepository) GetRawIndexedRow(ctx context.Context, uri string) (*pos return post, nil } +// AdmittedURIsForViewer returns the subset of uris whose post passes the +// viewer-bound admission half of the read-path visibility predicate +// (admittedPostsPredicate), ignoring active moderation removals. viewerDID is "" +// for an anonymous read. Soft-deleted rows are not filtered here; the caller +// already holds that state from the raw row. +func (r *PostRepository) AdmittedURIsForViewer(ctx context.Context, uris []string, viewerDID string) (map[string]bool, error) { + admitted := make(map[string]bool) + if len(uris) == 0 { + return admitted, nil + } + joinSQL, whereSQL := admittedPostsPredicate("$2") + rows, err := r.db.QueryContext(ctx, `SELECT p.uri FROM posts p`+joinSQL+` + WHERE p.uri = ANY($1) AND `+whereSQL, pq.Array(uris), viewerDID) + if err != nil { + return nil, fmt.Errorf("fetch admitted post URIs: %w", err) + } + defer rows.Close() + for rows.Next() { + var uri string + if err := rows.Scan(&uri); err != nil { + return nil, fmt.Errorf("scan admitted post URI: %w", err) + } + admitted[uri] = true + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate admitted post URIs: %w", err) + } + return admitted, nil +} + +// ActiveRemovalsByURIs returns active removal sources keyed by subject URI. +func (r *PostRepository) ActiveRemovalsByURIs(ctx context.Context, uris []string) (map[string][]posts.RemovalSource, error) { + removals := make(map[string][]posts.RemovalSource) + if len(uris) == 0 { + return removals, nil + } + rows, err := r.db.QueryContext(ctx, ` + SELECT subject_uri, authority_did, scope_kind + FROM moderation_decisions + WHERE subject_uri = ANY($1) AND kind = 'removal' AND active + ORDER BY subject_uri, authority_did, scope_kind, scope_community_did + `, pq.Array(uris)) + if err != nil { + return nil, fmt.Errorf("fetch active post removals: %w", err) + } + defer rows.Close() + for rows.Next() { + var uri string + var source posts.RemovalSource + if err := rows.Scan(&uri, &source.AuthorityDID, &source.ScopeKind); err != nil { + return nil, fmt.Errorf("scan active post removal: %w", err) + } + removals[uri] = append(removals[uri], source) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate active post removals: %w", err) + } + return removals, nil +} + // GetRawIndexedRowsByURIs is the batched GetRawIndexedRow. THE SAME DANGER // APPLIES — read the banner above before calling it. // // URIs with no indexed row are absent from the returned map; that is not an // error, it is the answer ("this URI is not indexed here"), and it is what lets // the caller tell a genuine lookup FAILURE (a returned error) apart from a URI -// the AppView has never seen. +// the AppView has never seen. Rows also carry community handle/name when the +// community exists; a missing community must not hide an indexed post. func (r *PostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []string) (map[string]*posts.Post, error) { result := make(map[string]*posts.Post, len(uris)) if len(uris) == 0 { @@ -202,9 +264,10 @@ func (r *PostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []str // Bound through a single array parameter (= ANY($1)) rather than an // interpolated IN list, so the SQL stays fully parameterized and the plan is // cached regardless of batch size. - query := `SELECT` + rawIndexedRowColumns + ` - FROM posts - WHERE uri = ANY($1)` + query := `SELECT` + rawIndexedRowColumns + `, c.handle, c.name + FROM posts p + LEFT JOIN communities c ON c.did = p.community_did + WHERE p.uri = ANY($1)` rows, err := r.db.QueryContext(ctx, query, pq.Array(uris)) if err != nil { @@ -217,7 +280,7 @@ func (r *PostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris []str }() for rows.Next() { - post, err := scanRawIndexedRow(rows) + post, err := scanRawIndexedRow(rows, true) if err != nil { return nil, fmt.Errorf("failed to scan raw post row: %w", err) } @@ -238,17 +301,22 @@ type rowScanner interface { // scanRawIndexedRow scans one rawIndexedRowColumns row into a posts.Post. The // Scan order below MUST stay byte-aligned with that column list. -func scanRawIndexedRow(row rowScanner) (*posts.Post, error) { +func scanRawIndexedRow(row rowScanner, withCommunity bool) (*posts.Post, error) { var post posts.Post var facetsJSON, embedJSON, labelsJSON sql.NullString + var communityHandle, communityName sql.NullString - err := row.Scan( + columns := []interface{}{ &post.ID, &post.URI, &post.CID, &post.RKey, &post.AuthorDID, &post.CommunityDID, &post.Title, &post.Content, &facetsJSON, &embedJSON, &labelsJSON, &post.CreatedAt, &post.EditedAt, &post.IndexedAt, &post.DeletedAt, &post.UpvoteCount, &post.DownvoteCount, &post.Score, &post.CommentCount, - ) + } + if withCommunity { + columns = append(columns, &communityHandle, &communityName) + } + err := row.Scan(columns...) if err != nil { return nil, err } @@ -264,6 +332,12 @@ func scanRawIndexedRow(row rowScanner) (*posts.Post, error) { // Labels are stored as JSONB containing full com.atproto.label.defs#selfLabels structure post.ContentLabels = &labelsJSON.String } + if communityHandle.Valid { + post.CommunityHandle = communityHandle.String + } + if communityName.Valid { + post.CommunityName = communityName.String + } return &post, nil } @@ -683,7 +757,7 @@ func scanPostView(rows *sql.Rows, extraDest ...interface{}) (*posts.PostView, er "error", err, ) } else { - postView.Embed = embedData + postView.Embed = embeds.ServableEmbed(embedData) } } diff --git a/internal/db/postgres/post_repo_cursor_test.go b/internal/db/postgres/post_repo_cursor_test.go index 50fcfae..7e3a4d4 100644 --- a/internal/db/postgres/post_repo_cursor_test.go +++ b/internal/db/postgres/post_repo_cursor_test.go @@ -231,6 +231,14 @@ func (m *mockPostRepository) GetRawIndexedRowsByURIs(ctx context.Context, uris [ return map[string]*posts.Post{}, nil } +func (m *mockPostRepository) ActiveRemovalsByURIs(context.Context, []string) (map[string][]posts.RemovalSource, error) { + return map[string][]posts.RemovalSource{}, nil +} + +func (m *mockPostRepository) AdmittedURIsForViewer(context.Context, []string, string) (map[string]bool, error) { + return map[string]bool{}, nil +} + func (m *mockPostRepository) GetByAuthor(ctx context.Context, req posts.GetAuthorPostsRequest) ([]*posts.PostView, *string, error) { return nil, nil, nil } diff --git a/internal/db/postgres/post_visibility.go b/internal/db/postgres/post_visibility.go index f135bf5..455fe4a 100644 --- a/internal/db/postgres/post_visibility.go +++ b/internal/db/postgres/post_visibility.go @@ -70,6 +70,10 @@ import ( // core: every non-accepted post that HAS a decision, and every postv2 with no // decision, is invisible to non-authors on every read path. // +// An active moderation removal hides the post from everyone, including its +// author, regardless of admission status. Restoring it reverts to the admission +// rule above; neither operation changes the post or its admission row. +// // The collection is the fourth '/'-segment of the AT-URI // (at:////), read with split_part; authorities and // rkeys carry no '/', so segment 4 is exactly CollectionOfPostURI's answer. @@ -101,11 +105,30 @@ const anonymousViewerSQL = `''` // anonymousViewerSQL for one that structurally does not. // // Everything above about the join key, the collection-aware status rule and the -// pinned CID describes THIS function; visiblePostsJoin is the parameterized -// spelling of it. Two viewer bindings, one predicate: a count and a display -// query cannot disagree about what "visible" means, because there is only one -// string. +// pinned CID, and active-removal exclusion describes THIS function; +// visiblePostsJoin is the parameterized spelling of it. Two viewer bindings, +// one predicate: a count and a display query cannot disagree about what +// "visible" means, because there is only one string. func visiblePostsPredicate(viewerExpr string) (joinSQL, whereSQL string) { + joinSQL, admittedSQL := admittedPostsPredicate(viewerExpr) + whereSQL = admittedSQL + ` AND NOT EXISTS ( + SELECT 1 FROM moderation_decisions d + WHERE d.subject_uri = p.uri AND d.kind = 'removal' AND d.active + )` + return joinSQL, whereSQL +} + +// admittedPostsPredicate is the viewer-bound admission half of +// visiblePostsPredicate: the join key and the collection-aware status rule, +// WITHOUT the active-removal exclusion. visiblePostsPredicate is this plus that +// exclusion, so the two cannot disagree about admission. +// +// Its only other caller is post.get's #moderatedPost gate +// (AdmittedURIsForViewer): an instance removal may tell a viewer that a post +// was removed only if that viewer could have seen the post had it not been +// removed. Anything else would disclose a pending, rejected or unadmitted post +// and tie it to a community that never accepted it. +func admittedPostsPredicate(viewerExpr string) (joinSQL, whereSQL string) { joinSQL = ` LEFT JOIN community_post_admissions a ON a.community_did = p.community_did AND a.post_uri = p.uri` @@ -121,7 +144,8 @@ func visiblePostsPredicate(viewerExpr string) (joinSQL, whereSQL string) { // visiblePostCountSubquery renders a scalar subquery counting the posts that are // PUBLICLY visible in the community named by communityExpr — the same predicate -// every display query runs, with the anonymous viewer. +// every display query runs, with the anonymous viewer. Active moderation +// removals are excluded from the count just as they are from display queries. // // It exists because `communities.post_count` was a STORED column, and the only // thing that ever incremented it (community_repo_memberships.go's diff --git a/internal/db/postgres/user_repo.go b/internal/db/postgres/user_repo.go index e1611a1..d40fb5f 100644 --- a/internal/db/postgres/user_repo.go +++ b/internal/db/postgres/user_repo.go @@ -232,24 +232,26 @@ func (r *postgresUserRepo) GetProfileStats(ctx context.Context, did string) (*us // count: the author self-view branches turn on `p.author_did = $2`, and no // DID is "". // - // comment_count is DELIBERATELY ROOT-BLIND — it counts the actor's comments - // whatever the admission state of the post they hang under, and that is not - // an oversight to be fixed by symmetry with post_count above. A comment is - // the actor's own public speech, and actor.getComments already LISTS it when - // its root is pending or removed, carrying the root as a bare uri/cid - // reference that leaks nothing and resolves through the gated post.get - // (TestActorCommentsVisibility_RootIsReferenceOnly pins that shape). Gating - // the count would put the profile's headline number in disagreement with the - // list the same profile renders, and would leak in the other direction: a - // comment count that visibly drops tells the reader a root they cannot see - // was moderated. The asymmetry with post_count is real and intended — a - // post's visibility IS its community's decision, a comment's is not. + // comment_count counts exactly what actor.getComments lists + // (ListByCommenterWithCursor): the actor's undeleted comments, minus any + // comment that is itself under an active instance removal or whose root is. + // The exclusion is the same `IN (c.uri, c.root_uri)` predicate that query + // runs, so the profile's headline number and the list it renders cannot + // disagree. It does NOT follow the root's community admission state: a + // comment under a pending or community-removed root is still the actor's own + // public speech and is listed and counted. Only instance removals, which hide + // the root from everyone, take its comments out of both. visJoin, visWhere := visiblePostsJoin(2) query := ` SELECT (SELECT COUNT(*) FROM posts p` + visJoin + ` WHERE p.author_did = $1 AND p.deleted_at IS NULL AND ` + visWhere + `) as post_count, - (SELECT COUNT(*) FROM comments WHERE commenter_did = $1 AND deleted_at IS NULL) as comment_count, + (SELECT COUNT(*) FROM comments c + WHERE c.commenter_did = $1 AND c.deleted_at IS NULL + AND NOT EXISTS ( + SELECT 1 FROM moderation_decisions d + WHERE d.subject_uri IN (c.uri, c.root_uri) AND d.kind = 'removal' AND d.active + )) as comment_count, (SELECT COUNT(*) FROM community_subscriptions WHERE user_did = $1) as community_count, (SELECT COUNT(*) FROM community_memberships WHERE user_did = $1 AND is_banned = false) as membership_count, (SELECT COALESCE(SUM(reputation_score), 0) FROM community_memberships WHERE user_did = $1) as reputation diff --git a/tests/e2e/moderation_contract_test.go b/tests/e2e/moderation_contract_test.go index c1ade12..f6e7e0b 100644 --- a/tests/e2e/moderation_contract_test.go +++ b/tests/e2e/moderation_contract_test.go @@ -236,3 +236,290 @@ func TestModerationCommentRemovalContract(t *testing.T) { }) } } + +// TestModerationPostRemovalContract follows an author-owned image post across +// the PDS, the AppView's consumers, instance removal, an author edit, community +// re-acceptance and restoration. The edited CID is the indexing barrier before +// checking that the overlay still hides the post and blocks the new image. +func TestModerationPostRemovalContract(t *testing.T) { + p := newPipeline(t) + author := p.IndexedAccount(t, "mpa") + community := indexedCommunity(t, p, "mpa", author.DID) + admin := testkit.ModerationAdmin(t, 1) + rkey := testkit.TID() + uri := authorPostURI(author.DID, rkey) + needle := "modpost" + testkit.UniqueID(t) + title := needle + " acceptance title" + content := "original post content " + testkit.UniqueID(t) + image := author.UploadBlob(t, testkit.TestPNG(64, 64), "image/png") + writePost := func(postTitle, postContent string, blobs ...testkit.BlobRef) string { + t.Helper() + record := postV2Record(community.DID, postTitle, postContent) + images := make([]any, 0, len(blobs)) + for _, blob := range blobs { + images = append(images, map[string]any{"image": blobRefValue(blob), "alt": "post moderation image"}) + } + record["embed"] = map[string]any{"$type": "social.coves.embed.images", "images": images} + return author.PutRecord(t, postV2Collection, rkey, record).CID + } + createdCID := writePost(title, content, image) + awaitStatus(t, p, uri, community.DID, "pending", "the author's image post to reach the admission queue") + acceptRkey := subjectRkey(uri) + community.PutRecord(t, acceptanceCollection, acceptRkey, acceptanceRecord(uri, createdCID)) + accepted := awaitStatus(t, p, uri, community.DID, "accepted", "the community to accept the image post") + + readPost := func() (map[string]any, error) { + var response struct { + Posts []map[string]any `json:"posts"` + } + err := p.AppView.Query(context.Background(), "social.coves.community.post.get", + url.Values{"uris": {uri}}, &response) + if err != nil { + return nil, err + } + if len(response.Posts) != 1 { + return nil, fmt.Errorf("post.get returned %d union members for one URI", len(response.Posts)) + } + return response.Posts[0], nil + } + var imageURL string + p.Await(t, "the accepted image post to serve through post.get", func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + record, ok := post["record"].(map[string]any) + if !ok || post["uri"] != uri || record["title"] != title || record["content"] != content || post["$type"] != nil { + return false, nil + } + embed, ok := post["embed"].(map[string]any) + if !ok { + return false, nil + } + images, ok := embed["images"].([]any) + if !ok || len(images) != 1 { + return false, nil + } + served, ok := images[0].(map[string]any) + if !ok { + return false, nil + } + imageURL, ok = served["fullsize"].(string) + return ok && imageURL != "", nil + }) + require.Contains(t, imageURL, image.CID()) + requireServesImage(t, p, "accepted post image", imageURL) + parsedImage, err := url.Parse(imageURL) + require.NoError(t, err) + require.True(t, strings.HasPrefix(parsedImage.Path, "/img/"), "the image must be served through the AppView proxy") + require.Contains(t, communityFeedURIs(t, p, community.DID), uri, + "the accepted post must appear in the feed before its removal can prove exclusion") + p.Await(t, "search to find the accepted post before removal", func() (bool, error) { + search, err := queryPostSearch(p, needle, community.DID) + if err != nil { + return false, err + } + return len(search.Feed) == 1 && search.Feed[0].Post.URI == uri, nil + }, withReadCadence()) + + commentRkey := testkit.TID() + commentURI := commentURI(author.DID, commentRkey) + commentText := "comment under removed post " + testkit.UniqueID(t) + ref := strongRef{URI: uri, CID: createdCID} + author.PutRecord(t, commentCollection, commentRkey, commentRecord(ref, ref, commentText)) + p.Await(t, "the comment to appear in the accepted post's thread", func() (bool, error) { + thread, err := p.Thread(context.Background(), uri, nil) + if err != nil { + return false, err + } + node, found := thread.find(commentURI) + return found && node.Comment.Record["content"] == commentText, nil + }, withReadCadence()) + + stateToken := admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + readState := func() (moderationSubjectStateResponse, error) { + var state moderationSubjectStateResponse + err := p.AppView.As(stateToken).Query(context.Background(), subjectStateMethod, + url.Values{"subject": {uri}}, &state) + return state, err + } + state, err := readState() + require.NoError(t, err) + require.Equal(t, createdCID, state.State.CurrentSubject.CID) + require.Equal(t, "clear", state.State.Moderation.State) + var removal struct { + Outcome string `json:"outcome"` + Action struct { + Action struct { + Ref struct { + ActionID string `json:"actionId"` + } `json:"ref"` + } `json:"action"` + } `json:"action"` + } + err = p.AppView.As(admin.ServiceAuth(t, communityInstanceDID, removeContentMethod)).Procedure( + t.Context(), removeContentMethod, map[string]any{ + "subject": map[string]any{"uri": uri, "cid": state.State.CurrentSubject.CID}, + "expectedVersion": state.State.Version, + "idempotencyKey": testkit.UniqueID(t), + "reason": "social.coves.moderation.defs#reasonSpam", + }, &removal) + require.NoError(t, err) + require.Equal(t, "applied", removal.Outcome) + require.NotEmpty(t, removal.Action.Action.Ref.ActionID) + admissionAfterRemoval, err := p.PostStatus(context.Background(), uri, community.DID) + require.NoError(t, err) + require.Equal(t, accepted, admissionAfterRemoval, "instance removal must not change the community's acceptance") + + moderated := func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + if post["$type"] != "social.coves.community.post.defs#moderatedPost" || post["uri"] != uri { + return false, nil + } + for _, key := range []string{"record", "title", "embed"} { + if _, leaked := post[key]; leaked { + return false, fmt.Errorf("moderated post leaked %s: %#v", key, post) + } + } + return true, nil + } + notFoundAnonymously := func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + for _, key := range []string{"record", "title", "embed", "cid"} { + if _, leaked := post[key]; leaked { + return false, fmt.Errorf("notFound post leaked %s: %#v", key, post) + } + } + return post["notFound"] == true && post["uri"] == uri, nil + } + removed, err := moderated() + require.NoError(t, err) + require.True(t, removed, "post.get must serve a content-free moderatedPost immediately after removal") + missingRootURI := authorPostURI(author.DID, testkit.TID()) + _, missingRootErr := p.Thread(context.Background(), missingRootURI, nil) + missingRoot := requireXRPCRefusal(t, missingRootErr, http.StatusNotFound, "RootNotFound", "a never-indexed post thread") + _, removedRootErr := p.Thread(context.Background(), uri, nil) + removedRoot := requireXRPCRefusal(t, removedRootErr, http.StatusNotFound, "RootNotFound", "an instance-removed post thread") + require.Equal(t, missingRoot.XRPCError, removedRoot.XRPCError) + require.NotContains(t, communityFeedURIs(t, p, community.DID), uri) + search, err := queryPostSearch(p, needle, community.DID) + require.NoError(t, err) + require.Empty(t, search.Feed, "the removed post must not appear in search for its unique title") + pathBlocked := func(path string) (bool, error) { + _, err := p.AppView.GetBinary(context.Background(), path) + if testkit.IsStatus(err, http.StatusNotFound) { + return true, nil + } + if err != nil { + return false, err + } + return false, nil + } + p.Await(t, "the removed post's cached image to return 404", func() (bool, error) { + return pathBlocked(parsedImage.Path) + }) + + editImage := author.UploadBlob(t, testkit.TestPNG(96, 96), "image/png") + require.NotEqual(t, image.CID(), editImage.CID()) + editImagePath := strings.Replace(parsedImage.Path, image.CID(), editImage.CID(), 1) + require.NotEqual(t, parsedImage.Path, editImagePath) + editedTitle := title + " edited" + editedCID := writePost(editedTitle, "edited while removed", image, editImage) + require.NotEqual(t, createdCID, editedCID) + // Observe the edit in the indexed post before testing the overlay or media + // reconciliation. A still-hidden pre-edit view proves neither behavior. + stateToken = admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + p.Await(t, "getSubjectState to report the author's edited post CID", func() (bool, error) { + indexed, err := readState() + if err != nil { + return false, err + } + return indexed.State.CurrentSubject.CID == editedCID, nil + }) + // The edited CID is not admitted yet, so an anonymous viewer gets notFound: + // removal never widens access to an unadmitted CID. The author's #moderatedPost + // view in this state is proven at T1 by TestModerationPostConsumerEditReconcilesNewImages, + // because this tier cannot hold an AppView-sealed OAuth session. + p.Holds(t, "the unadmitted edited post to read as notFound anonymously with its newly added image blocked", func() (bool, error) { + hidden, err := notFoundAnonymously() + if err != nil || !hidden { + return hidden, err + } + return pathBlocked(editImagePath) + }) + awaitStatus(t, p, uri, community.DID, "pending_reacceptance", + "the author's edit to invalidate the old community acceptance") + + // The community account can update its acceptance at the same subject rkey, + // so restore can be checked through the public thread rather than an + // author-only read of a still-pending post (R3). + community.PutRecord(t, acceptanceCollection, acceptRkey, acceptanceRecord(uri, editedCID)) + reaccepted := awaitStatus(t, p, uri, community.DID, "accepted", + "the community to re-accept the edited CID while instance removal stands") + removed, err = moderated() + require.NoError(t, err) + require.True(t, removed, "community re-acceptance must not undo instance removal") + + stateToken = admin.ServiceAuth(t, communityInstanceDID, subjectStateMethod) + state, err = readState() + require.NoError(t, err) + require.Equal(t, editedCID, state.State.CurrentSubject.CID) + require.Equal(t, "removed", state.State.Moderation.State) + var restoration struct { + Outcome string `json:"outcome"` + } + err = p.AppView.As(admin.ServiceAuth(t, communityInstanceDID, restoreContentMethod)).Procedure( + t.Context(), restoreContentMethod, map[string]any{ + "actionId": removal.Action.Action.Ref.ActionID, + "reviewedSubject": map[string]any{"uri": uri, "cid": editedCID}, + "expectedVersion": state.State.Version, + "idempotencyKey": testkit.UniqueID(t), + "reason": "social.coves.moderation.defs#reasonModeratorDiscretion", + }, &restoration) + require.NoError(t, err) + require.Equal(t, "applied", restoration.Outcome) + admissionAfterRestore, err := p.PostStatus(context.Background(), uri, community.DID) + require.NoError(t, err) + require.Equal(t, reaccepted, admissionAfterRestore, "instance restoration must not change the community's re-acceptance") + + p.Await(t, "the restored edited post to serve to anonymous readers", func() (bool, error) { + post, err := readPost() + if err != nil { + return false, err + } + record, ok := post["record"].(map[string]any) + return ok && post["uri"] == uri && post["$type"] == nil && record["title"] == editedTitle, nil + }) + p.FreshReadQuota(t, "restored-post-thread") + p.Await(t, "the restored post's comment thread to serve publicly", func() (bool, error) { + thread, err := p.Thread(context.Background(), uri, nil) + if err != nil { + return false, err + } + node, found := thread.find(commentURI) + return thread.Post.URI == uri && found && node.Comment.Record["content"] == commentText, nil + }, withReadCadence()) + require.Contains(t, communityFeedURIs(t, p, community.DID), uri, + "restoration must return the re-accepted post to the community feed") + // Positive control for the blocked-path checks above: both paths are real, + // servable blobs, so their 404s came from the media blocks restore lifts. + for _, path := range []string{parsedImage.Path, editImagePath} { + p.Await(t, "the restored post's images to serve again", func() (bool, error) { + response, err := p.AppView.GetBinary(context.Background(), path) + if testkit.IsStatus(err, http.StatusNotFound) { + return false, nil + } + if err != nil { + return false, err + } + return response.Status == http.StatusOK && len(response.Body) > 0 && + strings.HasPrefix(response.ContentType, "image/"), nil + }) + } +}