From 7aad882f7a2db4ba9bc1f6a2145c41fb45ba50b6 Mon Sep 17 00:00:00 2001 From: Bretton Date: Thu, 15 Oct 2026 14:38:56 -0700 Subject: [PATCH] feat(moderation): show removed text and comment root posts in the modlog Admins review removed content from the modlog, but after a removal every read path served a tombstone and the log carried no content, so nobody could see what was removed. Comment entries also could not link to their thread, because a comment URI alone does not build a client route. The user decided on 2026-10-13 that the snapshot is public, except for doxing and illegal-content removals, so the modlog stays public. Changes: - Migration 056 adds moderation_content_snapshots (keyed by action ID, cascade on action delete; author_did, title, text, link_url, captured_cid, author_deleted_at) and moderation_actions.subject_root_uri. - removeContent stores, in the removal transaction, the post title, body and external link URL, or the comment body, read from the CID the admin observed. No media blobs or blob references are copied. - Illegal-content removals store no snapshot. A doxing or illegal-content removal also deletes every snapshot earlier removals of the subject stored, for admins too. - Removing a comment its author already deleted stores no snapshot. Labels, label retractions and restores never store one. - listActions returns contentSnapshot only while the subject is public and not deleted by its author. listAdminActions returns it whenever a row exists, including for doxing removals and author-deleted subjects. - An author delete of a comment marks its snapshots author_deleted_at, in the same transaction as the soft delete, so a re-created comment at the same URI does not bring the text back to the public log. SoftDeleteWithReasonTx now opens its own transaction when given none. - Account erasure deletes the author's snapshots (by posts.author_did or comments.commenter_did, so legacy posts in a community repository are covered) and keeps the action rows. - Comment actions record subject.rootUri from comments.root_uri at action time; a restore copies it from its removal. A root that is not a valid AT-URI is not recorded, and a link URL that is not a valid URI is dropped. - The public log omits rootUri when the root post has a doxing or illegal-content action or is no longer indexed; listAdminActions then carries it in privateSubject. - docs/MODERATION.md documents the snapshot and rootUri rules. - Tests: unit projection tests, lexicon fixtures, Postgres integration tests for capture, erasure and visibility, and HTTP modlog output. Client notes: - Lexicon social.coves.moderation.defs adds #contentSnapshot {title?, text?, linkUrl?, capturedCid}, an optional contentSnapshot on #actionView and #adminActionView, and an optional rootUri on #subjectRef. All are additive. - contentSnapshot is returned by listActions and listAdminActions only, never in mutation results. - A restore entry carries no snapshot. Clients show the snapshot of the removal it reverses (reverses.actionId). - Snapshot text is author-written. Render it as plain text, and sanitize linkUrl before making it a link: it is only checked to be a syntactic URI, so it may use any scheme, including javascript:. - Actions recorded before this migration have no snapshot and no rootUri. - Update the "Coves Moderation: Client Handoff" doc. Co-Authored-By: Claude Opus 5.5 (1M context) --- cmd/server/cdn_purge_wiring_test.go | 4 + docs/MODERATION.md | 6 +- .../moderation_modlog_integration_test.go | 36 +- .../lexicon/social/coves/moderation/defs.json | 44 +- internal/core/moderation/fake_store_test.go | 8 + internal/core/moderation/indexed_subject.go | 10 + internal/core/moderation/modlog.go | 38 +- internal/core/moderation/modlog_test.go | 139 ++++++ internal/core/moderation/remove.go | 20 +- internal/core/moderation/restore.go | 3 +- internal/core/moderation/store.go | 34 ++ .../056_moderation_content_snapshots.sql | 28 ++ .../db/postgres/admission_repo_schema_test.go | 6 +- .../bridged_votes_peak_migration_test.go | 2 + internal/db/postgres/comment_repo.go | 51 ++- ...unity_subscriber_recount_migration_test.go | 2 + .../credential_reencrypt_migration_test.go | 4 + .../postgres/discover_hot_migration_test.go | 3 + ...uture_comment_created_at_migration_test.go | 2 + ..._action_log_restricted_integration_test.go | 53 ++- ...ntent_snapshot_erasure_integration_test.go | 122 +++++ ...ation_content_snapshot_integration_test.go | 424 ++++++++++++++++++ ...nt_snapshot_visibility_integration_test.go | 126 ++++++ internal/db/postgres/moderation_repo.go | 123 ++++- .../postgres/notification_migration_test.go | 4 + .../read_path_indexes_migration_test.go | 3 + .../rematerialize_ledger_schema_test.go | 2 + internal/db/postgres/user_repo.go | 15 +- .../vote_drift_recount_migration_test.go | 2 + ...tes_upvote_history_index_migration_test.go | 2 + ...tion-view-invalid-root-uri-not-at-uri.json | 20 + ...invalid-snapshot-missing-captured-cid.json | 22 + .../action-view-valid-content-snapshot.json | 25 ++ ...in-action-view-valid-content-snapshot.json | 29 ++ tests/lexicon_fixtures_test.go | 3 + tests/lexicon_moderation_test.go | 44 +- 36 files changed, 1383 insertions(+), 76 deletions(-) create mode 100644 internal/db/migrations/056_moderation_content_snapshots.sql create mode 100644 internal/db/postgres/moderation_content_snapshot_erasure_integration_test.go create mode 100644 internal/db/postgres/moderation_content_snapshot_integration_test.go create mode 100644 internal/db/postgres/moderation_content_snapshot_visibility_integration_test.go create mode 100644 tests/lexicon-test-data/moderation/action-view-invalid-root-uri-not-at-uri.json create mode 100644 tests/lexicon-test-data/moderation/action-view-invalid-snapshot-missing-captured-cid.json create mode 100644 tests/lexicon-test-data/moderation/action-view-valid-content-snapshot.json create mode 100644 tests/lexicon-test-data/moderation/admin-action-view-valid-content-snapshot.json diff --git a/cmd/server/cdn_purge_wiring_test.go b/cmd/server/cdn_purge_wiring_test.go index 3fb4b76..d1fd6d7 100644 --- a/cmd/server/cdn_purge_wiring_test.go +++ b/cmd/server/cdn_purge_wiring_test.go @@ -75,6 +75,10 @@ type cdnWiringTransaction struct { } func (*cdnWiringTransaction) LockActor(context.Context, string) error { return nil } +func (*cdnWiringTransaction) InsertContentSnapshot(context.Context, string, string, moderation.ContentSnapshot) error { + return nil +} +func (*cdnWiringTransaction) DeleteSubjectContentSnapshots(context.Context, string) error { return nil } func (*cdnWiringTransaction) LiveIdempotencyRecord(context.Context, string, string, string, time.Time) (*moderation.IdempotencyRecord, error) { return nil, nil } diff --git a/docs/MODERATION.md b/docs/MODERATION.md index 11f165e..f5e2f36 100644 --- a/docs/MODERATION.md +++ b/docs/MODERATION.md @@ -67,6 +67,10 @@ Q-LABEL-LOG (2026-09-30): `listActions` never returns `label` or `retract-label` Restricted subject (decided 2026-10-03): removals are public by default. The public log names a subject (URI, CID and the community it was posted in) and matches it under subject, collection and community filters while its post or comment row is indexed, whether soft-deleted or not and whatever its admission status. Only a subject whose row is gone, as after account erasure, is restricted: listed without subject or community, and excluded from those filters. +Content snapshots (decided 2026-10-13): a removal stores the text of the version the admin removed, as `contentSnapshot`: a post's title, body and external link URL, a comment's body, and `capturedCid`, the CID it was read from. Restores and labels store none, and illegal-content removals store none at all. A doxing or illegal-content removal also deletes every snapshot earlier removals of the same subject stored, for admins too. Removing a comment its author already deleted stores none, because the delete blanked its text. The public log shows the snapshot only with the subject, so it is withheld for doxing removals; `listAdminActions` shows it there. A restricted subject has no snapshot left: its row is gone only after account erasure, which also deletes the author's snapshots. Once the author deletes the subject, the public entry still names it but withholds the snapshot; `listAdminActions` keeps it. For a comment this is permanent: a new record that re-creates the comment's URI does not bring the snapshot back. The snapshot is kept while its action row exists, and a restore or a later edit does not change it. Erasing the author's account deletes the author's snapshots and keeps the actions. The author is the account that wrote the post or comment, which for a legacy post is not the community repository that holds it. + +Comment subjects carry `rootUri`, the post at the root of the comment's thread, recorded when the action was taken. A restore copies it from the removal it reverses. Actions recorded before 2026-10-13, and comments whose indexed root is not a valid AT-URI, have none. It is withheld from the public log with the rest of the subject, and on its own when the root post has a doxing or illegal-content action or is no longer indexed; `listAdminActions` then carries it in `privateSubject` (decided 2026-10-13). + ## Media blocks Every removal blocks the (owner DID, blob CID) pair of each image on the subject, on cache hits and misses, and purges the image proxy's disk cache. An illegal-content removal also blocks each CID for every owner, so a re-upload from another account is not served. Restoring deactivates the decision's blocks. @@ -93,5 +97,5 @@ Every mutation carries an idempotency key and the `expectedVersion` from `getSub - Federated moderation through signed labels: see [plans/MODERATION_LABELS.md](plans/MODERATION_LABELS.md). The schema already accepts `origin = inherited` and the `apply-removal` and `retract-removal` actions. - Community-admin moderation. `scope_kind = community` exists in the tables, but nothing writes it. -- Admin inspection of removed content, retained evidence, and modlog editing. +- Evidence retention for illegal-content removals, which keep no snapshot, and modlog editing. - Comment removal across the Lemmy bridge. `community.removal` is post-scoped, so a Lemmy moderator removing a native Coves comment is recorded only in Tidepool and the comment stays visible here, and a moderator-removed Lemmy comment arrives as a repo delete and shows as deleted by its author. diff --git a/internal/api/routes/moderation_modlog_integration_test.go b/internal/api/routes/moderation_modlog_integration_test.go index bea15a3..9cf3f5a 100644 --- a/internal/api/routes/moderation_modlog_integration_test.go +++ b/internal/api/routes/moderation_modlog_integration_test.go @@ -248,7 +248,7 @@ func TestModerationModlogListsRemovalHistory(t *testing.T) { if want.hidden { assert.NotContains(t, action, "subject") } else { - assert.Equal(t, map[string]any{"uri": want.uri, "cid": want.cid, "communityDid": f.communityDID}, action["subject"]) + assert.Equal(t, map[string]any{"uri": want.uri, "cid": want.cid, "communityDid": f.communityDID, "rootUri": f.postURI}, action["subject"]) } if want.reverses != "" { assert.Equal(t, want.reverses, moderationAcceptanceObject(t, action["reverses"])["actionId"]) @@ -270,7 +270,7 @@ func TestModerationModlogListsRemovalHistory(t *testing.T) { assert.Equal(t, want.note, entry["privateNote"]) if want.hidden { assert.NotContains(t, action, "subject") - assert.Equal(t, map[string]any{"uri": want.uri, "cid": want.cid, "communityDid": f.communityDID}, entry["privateSubject"]) + assert.Equal(t, map[string]any{"uri": want.uri, "cid": want.cid, "communityDid": f.communityDID, "rootUri": f.postURI}, entry["privateSubject"]) } else { assert.NotContains(t, entry, "privateSubject") } @@ -686,3 +686,35 @@ func TestModerationModlogCommunityAssociationSurvivesReindex(t *testing.T) { require.NoError(t, err) check(t) } + +// Acceptance: a public removal of a comment names the post that roots its +// thread and shows the removed text; a doxing removal shows neither its +// subject nor its text publicly, while admins see both. +func TestModerationModlogContentSnapshotsAndCommentRoot(t *testing.T) { + f := newModlogFixture(t) + commentURI, commentCID := f.comment(t) + commentRemoval, _ := f.remove(t, commentURI, commentCID, f.tokenA, modlogSpam, "comment removal note") + doxedURI := fixtures.Post(t, f.db, f.communityDID, f.authorDID, "Doxed post title", 0, time.Now()) + doxedRemoval, _ := f.remove(t, doxedURI, "bafytest", f.tokenA, modlogDoxing, "doxing removal note") + + publicResponse := f.list(t, false, "", nil) + _, publicItems := modlogPage(t, publicResponse, false) + public := map[string]map[string]any{} + for _, item := range publicItems { + action := modlogAction(t, item, false) + public[modlogID(t, action)] = action + } + require.Contains(t, public, commentRemoval) + assert.Equal(t, map[string]any{"uri": commentURI, "cid": commentCID, "communityDid": f.communityDID, "rootUri": f.postURI}, public[commentRemoval]["subject"]) + assert.Equal(t, map[string]any{"text": "modlog comment", "capturedCid": commentCID}, public[commentRemoval]["contentSnapshot"]) + require.Contains(t, public, doxedRemoval) + assert.NotContains(t, public[doxedRemoval], "subject") + assert.NotContains(t, public[doxedRemoval], "contentSnapshot") + assert.NotContains(t, string(publicResponse.body), "Doxed post title") + + _, adminItems := modlogPage(t, f.list(t, true, f.tokenB, url.Values{"actionId": {doxedRemoval}}), true) + require.Len(t, adminItems, 1) + entry := moderationAcceptanceObject(t, adminItems[0]) + assert.Equal(t, map[string]any{"title": "Doxed post title", "capturedCid": "bafytest"}, entry["contentSnapshot"]) + assert.Equal(t, map[string]any{"uri": doxedURI, "cid": "bafytest", "communityDid": f.communityDID}, entry["privateSubject"]) +} diff --git a/internal/atproto/lexicon/social/coves/moderation/defs.json b/internal/atproto/lexicon/social/coves/moderation/defs.json index 6195742..c12cb13 100644 --- a/internal/atproto/lexicon/social/coves/moderation/defs.json +++ b/internal/atproto/lexicon/social/coves/moderation/defs.json @@ -98,6 +98,11 @@ "type": "string", "format": "did", "description": "DID of the community the record was posted in, recorded when the action was taken" + }, + "rootUri": { + "type": "string", + "format": "at-uri", + "description": "AT-URI of the post at the root of a comment's thread, recorded when the action was taken; absent on actions recorded before 2026-10-13. The public log omits it when the root post has a doxing or illegal-content action or is no longer indexed" } } }, @@ -202,6 +207,11 @@ "type": "string", "format": "datetime", "description": "Source label creation time, not a moderation-log sort key" + }, + "contentSnapshot": { + "type": "ref", + "ref": "#contentSnapshot", + "description": "Text of a removed subject, captured at removal; omitted when the subject is withheld or deleted by its author. Returned by listActions and listAdminActions only, never in a mutation result" } } }, @@ -225,13 +235,45 @@ "privateSubject": { "type": "ref", "ref": "#subjectRef", - "description": "Private subject reference retained when the public projection restricts it" + "description": "Private subject reference retained when the public projection restricts it or withholds its rootUri" }, "privateNote": { "type": "string", "maxGraphemes": 1000, "maxLength": 10000, "description": "Private administrative note that is never included in public projections" + }, + "contentSnapshot": { + "type": "ref", + "ref": "#contentSnapshot", + "description": "Text of a removed subject, captured at removal, including when the public projection withholds it. Returned by listAdminActions only, never in a mutation result" + } + } + }, + "contentSnapshot": { + "type": "object", + "description": "Text of a removed record as indexed when it was removed", + "required": [ + "capturedCid" + ], + "properties": { + "title": { + "type": "string", + "description": "Post title" + }, + "text": { + "type": "string", + "description": "Post body or comment text" + }, + "linkUrl": { + "type": "string", + "format": "uri", + "description": "URL of the post's external link embed" + }, + "capturedCid": { + "type": "string", + "format": "cid", + "description": "CID of the record version the snapshot was taken from" } } }, diff --git a/internal/core/moderation/fake_store_test.go b/internal/core/moderation/fake_store_test.go index d0d2bef..e78d204 100644 --- a/internal/core/moderation/fake_store_test.go +++ b/internal/core/moderation/fake_store_test.go @@ -306,6 +306,14 @@ func (transaction *inMemoryModerationTransaction) SetLabelDecision(_ context.Con return nil } +func (transaction *inMemoryModerationTransaction) InsertContentSnapshot(context.Context, string, string, moderation.ContentSnapshot) error { + return nil +} + +func (transaction *inMemoryModerationTransaction) DeleteSubjectContentSnapshots(context.Context, string) error { + return nil +} + func (transaction *inMemoryModerationTransaction) InsertAction(_ context.Context, action moderation.Action) (*moderation.Action, error) { transaction.store.writeCalls = append(transaction.store.writeCalls, "InsertAction") if transaction.store.failInsertAction != nil { diff --git a/internal/core/moderation/indexed_subject.go b/internal/core/moderation/indexed_subject.go index cba65d5..86da6d1 100644 --- a/internal/core/moderation/indexed_subject.go +++ b/internal/core/moderation/indexed_subject.go @@ -15,6 +15,13 @@ type indexedSubject struct { OwnerDID string CommunityDID string BlobCIDs []string + // AuthorDID is the account that wrote the subject; for a legacy post it differs from OwnerDID. + AuthorDID string + // RootURI is the post at the root of a comment's thread; empty for posts. + RootURI string + Title string + Text string + LinkURL string } type indexedSubjectReader interface { @@ -38,6 +45,8 @@ func readIndexedSubject(ctx context.Context, reader indexedSubjectReader, subjec URI: comment.URI, CID: comment.CID, Collection: collection, AuthorDeleted: comment.AuthorDeleted, OwnerDID: comment.OwnerDID, CommunityDID: comment.CommunityDID, BlobCIDs: comment.ImageCIDs, + // A comment lives in its author's repository. + AuthorDID: comment.OwnerDID, RootURI: comment.RootURI, Text: comment.Text, }, nil case PostV2Collection, LegacyPostCollection: post, err := reader.ReadIndexedPost(ctx, subjectURI) @@ -48,6 +57,7 @@ func readIndexedSubject(ctx context.Context, reader indexedSubjectReader, subjec URI: post.URI, CID: post.CID, Collection: collection, AuthorDeleted: post.AuthorDeleted, OwnerDID: post.OwnerDID, CommunityDID: post.CommunityDID, BlobCIDs: post.BlobCIDs, + AuthorDID: post.AuthorDID, Title: post.Title, Text: post.Text, LinkURL: post.LinkURL, }, nil default: return nil, fmt.Errorf("%w: unsupported subject collection", ErrInvalidSubject) diff --git a/internal/core/moderation/modlog.go b/internal/core/moderation/modlog.go index 90b4dbe..12f01c8 100644 --- a/internal/core/moderation/modlog.go +++ b/internal/core/moderation/modlog.go @@ -82,6 +82,8 @@ type ActionView struct { LabelValue string `json:"labelValue,omitempty"` Actor *ActorRefView `json:"actor,omitempty"` Reverses *ActionRefView `json:"reverses,omitempty"` + // ContentSnapshot is present only while the subject is public and not deleted by its author. + ContentSnapshot *ContentSnapshotView `json:"contentSnapshot,omitempty"` } // ActionRefView is defs#actionRef. @@ -101,10 +103,26 @@ type SubjectRefView struct { URI string `json:"uri"` CID string `json:"cid,omitempty"` CommunityDID string `json:"communityDid,omitempty"` + RootURI string `json:"rootUri,omitempty"` } func newSubjectRefView(action Action) *SubjectRefView { - return &SubjectRefView{URI: action.SubjectURI, CID: action.ObservedCID, CommunityDID: action.SubjectCommunityDID} + return &SubjectRefView{URI: action.SubjectURI, CID: action.ObservedCID, CommunityDID: action.SubjectCommunityDID, RootURI: action.SubjectRootURI} +} + +// ContentSnapshotView is defs#contentSnapshot. +type ContentSnapshotView struct { + Title string `json:"title,omitempty"` + Text string `json:"text,omitempty"` + LinkURL string `json:"linkUrl,omitempty"` + CapturedCID string `json:"capturedCid"` +} + +func newContentSnapshotView(snapshot *ContentSnapshot) *ContentSnapshotView { + if snapshot == nil { + return nil + } + return &ContentSnapshotView{Title: snapshot.Title, Text: snapshot.Text, LinkURL: snapshot.LinkURL, CapturedCID: snapshot.CapturedCID} } // ActorRefView is defs#actorRef. @@ -118,6 +136,9 @@ type AdminActionView struct { ActorDID string `json:"actorDid,omitempty"` PrivateSubject *SubjectRefView `json:"privateSubject,omitempty"` PrivateNote string `json:"privateNote,omitempty"` + // ContentSnapshot is present whenever the action has one, including for + // a subject the public log withholds. + ContentSnapshot *ContentSnapshotView `json:"contentSnapshot,omitempty"` } // NewActionView projects an action for the public log. @@ -134,6 +155,12 @@ func NewActionView(action Action) ActionView { } if !privateSubject(action) { view.Subject = newSubjectRefView(action) + if action.SubjectRootWithheld { + view.Subject.RootURI = "" + } + if !action.SubjectAuthorDeleted { + view.ContentSnapshot = newContentSnapshotView(action.ContentSnapshot) + } } if restrictedSubject(action) { // A community scope would tie the restricted subject to its community. @@ -151,11 +178,12 @@ func NewActionView(action Action) ActionView { // NewAdminActionView projects an action for the admin log. func NewAdminActionView(action Action) AdminActionView { view := AdminActionView{ - Action: NewActionView(action), - ActorDID: action.ActorDID, - PrivateNote: action.PrivateNote, + Action: NewActionView(action), + ActorDID: action.ActorDID, + PrivateNote: action.PrivateNote, + ContentSnapshot: newContentSnapshotView(action.ContentSnapshot), } - if privateSubject(action) { + if privateSubject(action) || (action.SubjectRootWithheld && action.SubjectRootURI != "") { view.PrivateSubject = newSubjectRefView(action) } return view diff --git a/internal/core/moderation/modlog_test.go b/internal/core/moderation/modlog_test.go index 5de2e94..d8fb005 100644 --- a/internal/core/moderation/modlog_test.go +++ b/internal/core/moderation/modlog_test.go @@ -259,3 +259,142 @@ func TestModlogPublicProjectionDoesNotLeakStoredFields(t *testing.T) { } } } + +// A comment action's rootUri goes wherever its subject goes: in the public +// subject while the subject is public, and only in the admin privateSubject +// when the subject is hidden or restricted. +func TestModlogSubjectRootURIProjection(t *testing.T) { + const rootURI = "at://did:plc:rootauthor/social.coves.community.post/3mwmyrootpost" + for _, test := range []struct { + name string + rootURI string + reason string + reversedReason string + restore bool + access moderation.SubjectAccess + rootWithheld bool + private bool + }{ + {"public spam removal", rootURI, modlogReasonSpam, "", false, moderation.SubjectAccessPublic, false, false}, + {"doxing removal", rootURI, modlogReasonDoxing, "", false, moderation.SubjectAccessPublic, false, true}, + {"restore of doxing", rootURI, modlogReasonDiscretion, modlogReasonDoxing, true, moderation.SubjectAccessPublic, false, true}, + {"restricted spam removal", rootURI, modlogReasonSpam, "", false, moderation.SubjectAccessRestricted, false, true}, + {"public action without root", "", modlogReasonSpam, "", false, moderation.SubjectAccessPublic, false, false}, + {"hidden action without root", "", modlogReasonDoxing, "", false, moderation.SubjectAccessPublic, false, true}, + {"public spam removal under a withheld root", rootURI, modlogReasonSpam, "", false, moderation.SubjectAccessPublic, true, false}, + } { + t.Run(test.name, func(t *testing.T) { + action := modlogTestAction() + action.SubjectRootURI = test.rootURI + action.Reason = test.reason + action.SubjectAccess = test.access + action.SubjectRootWithheld = test.rootWithheld + if test.restore { + action.Action = moderation.ActionRestore + action.ReversesActionID = "3mwmyoriginalremoval" + action.ReversedActionReason = test.reversedReason + } + + publicObject, publicJSON := modlogJSON(t, moderation.NewActionView(action)) + adminObject, _ := modlogJSON(t, moderation.NewAdminActionView(action)) + if test.private { + assert.NotContains(t, publicObject, "subject") + assert.NotContains(t, string(publicJSON), rootURI, "a private subject's thread must not be named publicly") + privateSubject := modlogObject(t, adminObject["privateSubject"]) + if test.rootURI == "" { + assert.NotContains(t, privateSubject, "rootUri") + } else { + assert.Equal(t, rootURI, privateSubject["rootUri"]) + } + return + } + subject := modlogObject(t, publicObject["subject"]) + if test.rootWithheld { + // The subject stays public; only its thread's root goes admin-only. + assert.Equal(t, "at://did:plc:author/social.coves.community.comment/3mwmycomment", subject["uri"]) + assert.NotContains(t, subject, "rootUri") + assert.NotContains(t, string(publicJSON), rootURI, "a withheld root must not be named publicly") + privateSubject := modlogObject(t, adminObject["privateSubject"]) + assert.Equal(t, rootURI, privateSubject["rootUri"]) + return + } + if test.rootURI == "" { + assert.NotContains(t, subject, "rootUri") + } else { + assert.Equal(t, rootURI, subject["rootUri"]) + } + assert.NotContains(t, adminObject, "privateSubject") + }) + } +} + +// The public log shows a removal's content snapshot only while it shows the +// subject; the admin log shows it whenever the action has one. +func TestModlogContentSnapshotProjection(t *testing.T) { + postSnapshot := &moderation.ContentSnapshot{ + Title: "Removed post title 🔥", + Text: "Removed post body\nsecond line", + LinkURL: "https://news.example/story?id=42", + CapturedCID: "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + } + postSnapshotJSON := map[string]any{ + "title": "Removed post title 🔥", + "text": "Removed post body\nsecond line", + "linkUrl": "https://news.example/story?id=42", + "capturedCid": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + } + commentSnapshot := &moderation.ContentSnapshot{ + Text: "Removed comment body", + CapturedCID: "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi", + } + commentSnapshotJSON := map[string]any{ + "text": "Removed comment body", + "capturedCid": "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi", + } + for _, test := range []struct { + name string + snapshot *moderation.ContentSnapshot + reason string + access moderation.SubjectAccess + // authorDeleted marks a subject its author has since deleted: the + // public entry still names it but withholds the snapshot. + authorDeleted bool + wantPublic map[string]any + wantAdmin map[string]any + }{ + {"public post removal", postSnapshot, modlogReasonSpam, moderation.SubjectAccessPublic, false, postSnapshotJSON, postSnapshotJSON}, + {"public comment removal omits empty fields", commentSnapshot, modlogReasonHarassment, moderation.SubjectAccessPublic, false, commentSnapshotJSON, commentSnapshotJSON}, + {"doxing removal", postSnapshot, modlogReasonDoxing, moderation.SubjectAccessPublic, false, nil, postSnapshotJSON}, + {"restricted subject", postSnapshot, modlogReasonSpam, moderation.SubjectAccessRestricted, false, nil, postSnapshotJSON}, + {"author-deleted subject", postSnapshot, modlogReasonSpam, moderation.SubjectAccessPublic, true, nil, postSnapshotJSON}, + {"no snapshot", nil, modlogReasonSpam, moderation.SubjectAccessPublic, false, nil, nil}, + {"hidden action without snapshot", nil, modlogReasonDoxing, moderation.SubjectAccessPublic, false, nil, nil}, + } { + t.Run(test.name, func(t *testing.T) { + action := modlogTestAction() + action.ContentSnapshot = test.snapshot + action.Reason = test.reason + action.SubjectAccess = test.access + action.SubjectAuthorDeleted = test.authorDeleted + + publicObject, publicJSON := modlogJSON(t, moderation.NewActionView(action)) + if test.authorDeleted { + assert.Contains(t, publicObject, "subject", "an author-deleted subject is still named publicly") + } + if test.wantPublic == nil { + assert.NotContains(t, publicObject, "contentSnapshot") + assert.NotContains(t, string(publicJSON), "Removed post body", "a withheld snapshot must not leak publicly") + } else { + assert.Equal(t, test.wantPublic, publicObject["contentSnapshot"]) + } + + adminObject, _ := modlogJSON(t, moderation.NewAdminActionView(action)) + assert.Equal(t, publicObject, modlogObject(t, adminObject["action"]), "the admin entry embeds the public projection") + if test.wantAdmin == nil { + assert.NotContains(t, adminObject, "contentSnapshot") + } else { + assert.Equal(t, test.wantAdmin, adminObject["contentSnapshot"]) + } + }) + } +} diff --git a/internal/core/moderation/remove.go b/internal/core/moderation/remove.go index 8073088..4c8bce1 100644 --- a/internal/core/moderation/remove.go +++ b/internal/core/moderation/remove.go @@ -132,7 +132,8 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, ScopeKind: ScopeInstance, SubjectURI: request.Subject.URI, SubjectCollection: subject.Collection, SubjectCommunityDID: subject.CommunityDID, - ObservedCID: subject.CID, Action: ActionRemove, Reason: request.Reason, + SubjectRootURI: subject.RootURI, + ObservedCID: subject.CID, Action: ActionRemove, Reason: request.Reason, PrivateNote: request.PrivateNote, Origin: OriginLocal, CreatedAt: now, }) if err != nil { @@ -141,6 +142,23 @@ func (s *service) removeContent(ctx context.Context, actorDID string, request Re if action == nil || action.ID == "" { return unavailable(errors.New("action insert returned no identifier")) } + // A doxing or illegal-content removal drops the text earlier + // removals of the subject kept, for admins too. + if request.Reason == doxingReason || request.Reason == illegalContentReason { + if err := tx.DeleteSubjectContentSnapshots(ctx, request.Subject.URI); err != nil { + return unavailable(err) + } + } + // Illegal content is never kept, not even for admins. A comment + // its author deleted has already lost its text. + authorDeletedComment := subject.AuthorDeleted && subject.Collection == CommentCollection + if request.Reason != illegalContentReason && !authorDeletedComment { + if err := tx.InsertContentSnapshot(ctx, action.ID, subject.AuthorDID, ContentSnapshot{ + Title: subject.Title, Text: subject.Text, LinkURL: subject.LinkURL, CapturedCID: action.ObservedCID, + }); err != nil { + return unavailable(err) + } + } if err := tx.SetRemovalDecision(ctx, s.config.InstanceDID, request.Subject.URI, action.ID, true); err != nil { return unavailable(err) } diff --git a/internal/core/moderation/restore.go b/internal/core/moderation/restore.go index d043391..a84a10e 100644 --- a/internal/core/moderation/restore.go +++ b/internal/core/moderation/restore.go @@ -128,7 +128,8 @@ func (s *service) restoreContent(ctx context.Context, actorDID string, request R ActorDID: actorDID, AuthorityDID: s.config.InstanceDID, ScopeKind: ScopeInstance, SubjectURI: removal.SubjectURI, SubjectCollection: removal.SubjectCollection, SubjectCommunityDID: removal.SubjectCommunityDID, - ObservedCID: observedCID, Action: ActionRestore, Reason: request.Reason, + SubjectRootURI: removal.SubjectRootURI, + ObservedCID: observedCID, Action: ActionRestore, Reason: request.Reason, PrivateNote: request.PrivateNote, ReversesActionID: removal.ID, ReversedActionReason: removal.Reason, Origin: OriginLocal, CreatedAt: now, diff --git a/internal/core/moderation/store.go b/internal/core/moderation/store.go index 0feb734..1c4c9aa 100644 --- a/internal/core/moderation/store.go +++ b/internal/core/moderation/store.go @@ -40,6 +40,7 @@ type Action struct { SubjectURI string SubjectCollection string SubjectCommunityDID string + SubjectRootURI string `json:",omitempty"` // omitempty keeps stored idempotency result JSON (pinned in idempotency_golden_test.go) unchanged for actions without a root. ObservedCID string Action string LabelValue string @@ -58,6 +59,24 @@ type Action struct { // service refuses a row that leaves it unset. It is read-time state, so it // stays out of stored idempotency results. SubjectAccess SubjectAccess `json:"-"` + // ContentSnapshot is read-time state too: only Store.ListActions sets it, + // so mutation results and stored idempotency results never carry it. + ContentSnapshot *ContentSnapshot `json:"-"` + // SubjectAuthorDeleted is read-time state set by Store.ListActions: the author has deleted the subject. + // Any reader that sets ContentSnapshot must set it too, or the public log shows text the author deleted. + SubjectAuthorDeleted bool `json:"-"` + // SubjectRootWithheld is read-time state set by Store.ListActions: the public log must not name the + // subject's root post, which has a hidden-reason action or is no longer indexed. + SubjectRootWithheld bool `json:"-"` +} + +// ContentSnapshot is the text of a removed subject, as indexed when it was +// removed. CapturedCID is the version it was read from. +type ContentSnapshot struct { + Title string + Text string + LinkURL string + CapturedCID string } // SubjectAccess says whether the public log may name an action's subject. @@ -85,6 +104,9 @@ type IndexedComment struct { CommunityDID string // ImageCIDs are the blob CIDs of the comment's indexed image embed. ImageCIDs []string + // RootURI is the post at the root of the comment's thread. + RootURI string + Text string } // IndexedPost is the indexed post row a mutation inspects, read under a share @@ -99,6 +121,13 @@ type IndexedPost struct { CommunityDID string // BlobCIDs are the canonical CIDs of the post's proxy-served blobs. BlobCIDs []string + // AuthorDID is the person who wrote the post, which differs from OwnerDID + // for legacy posts. + AuthorDID string + Title string + Text string + // LinkURL is the URI of an external link embed. + LinkURL string } // MediaBlock suppresses Coves-served bytes of a blob. An empty OwnerDID @@ -192,6 +221,11 @@ type Transaction interface { // ActiveLabels returns active instance-scope label actions, ordered by value. ActiveLabels(ctx context.Context, authorityDID, subjectURI string) ([]Action, error) InsertAction(ctx context.Context, action Action) (*Action, error) + // InsertContentSnapshot stores the snapshot of a removal's subject, + // written by authorDID, keyed to the removal's action. + InsertContentSnapshot(ctx context.Context, actionID, authorDID string, snapshot ContentSnapshot) error + // DeleteSubjectContentSnapshots deletes the snapshots of every action on subjectURI. + DeleteSubjectContentSnapshots(ctx context.Context, subjectURI string) error SetRemovalDecision(ctx context.Context, authorityDID, subjectURI, actionID string, active bool) error // SetLabelDecision changes the instance-scope label decision for // (authorityDID, subjectURI, value). With active true it activates the diff --git a/internal/db/migrations/056_moderation_content_snapshots.sql b/internal/db/migrations/056_moderation_content_snapshots.sql new file mode 100644 index 0000000..23cf8bd --- /dev/null +++ b/internal/db/migrations/056_moderation_content_snapshots.sql @@ -0,0 +1,28 @@ +-- +goose Up +-- The text of a removed post or comment as indexed when it was removed, kept +-- for the modlog while its action row exists. Illegal-content removals keep +-- none. author_did is the account that wrote the record (posts.author_did or +-- comments.commenter_did), not the repository owner, so account erasure can +-- delete an author's snapshots, including those of legacy posts held in a +-- community's repository. author_deleted_at is set when the author deletes a +-- removed comment, so the public log withholds its snapshot even after a new +-- record re-creates the comment's URI. +CREATE TABLE moderation_content_snapshots ( + action_id TEXT PRIMARY KEY REFERENCES moderation_actions(id) ON DELETE CASCADE, + author_did TEXT NOT NULL, + title TEXT, + text TEXT, + link_url TEXT, + captured_cid TEXT NOT NULL, + author_deleted_at TIMESTAMPTZ +); + +CREATE INDEX moderation_content_snapshots_author ON moderation_content_snapshots (author_did); + +-- The post at the root of a comment subject's thread, recorded when the action +-- was taken. NULL for post subjects. +ALTER TABLE moderation_actions ADD COLUMN subject_root_uri TEXT; + +-- +goose Down +ALTER TABLE moderation_actions DROP COLUMN subject_root_uri; +DROP TABLE moderation_content_snapshots; diff --git a/internal/db/postgres/admission_repo_schema_test.go b/internal/db/postgres/admission_repo_schema_test.go index 0b2d628..1d31abb 100644 --- a/internal/db/postgres/admission_repo_schema_test.go +++ b/internal/db/postgres/admission_repo_schema_test.go @@ -318,10 +318,12 @@ func TestMigration034_DownRestoresTheAuthorForeignKeyUnvalidated(t *testing.T) { require.NoError(t, err, "with fk_author dropped, a federated author's post must index even though no users row exists for them") - // The expected-version parameter is the tripwire. Migrations 035 through 055 - // all sit on top of 034, so all twenty-one have to come off first. Rolling back explicitly, + // The expected-version parameter is the tripwire. Migrations 035 through 056 + // all sit on top of 034, so all twenty-two have to come off first. Rolling back explicitly, // one asserted step at a time, is what keeps the assertions below pointed at // 034's Down rather than at whatever happens to be newest. + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/bridged_votes_peak_migration_test.go b/internal/db/postgres/bridged_votes_peak_migration_test.go index 0593fa7..aedbead 100644 --- a/internal/db/postgres/bridged_votes_peak_migration_test.go +++ b/internal/db/postgres/bridged_votes_peak_migration_test.go @@ -13,6 +13,8 @@ import ( func TestNotificationMigration053_BridgedUpvotePeakColumns(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55)) require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54)) require.EqualValues(t, 53, testkit.MigrateDownOne(t, db, 53)) diff --git a/internal/db/postgres/comment_repo.go b/internal/db/postgres/comment_repo.go index f9adaf0..441afb3 100644 --- a/internal/db/postgres/comment_repo.go +++ b/internal/db/postgres/comment_repo.go @@ -241,10 +241,31 @@ func (r *postgresCommentRepo) SoftDeleteWithReason(ctx context.Context, uri, rea } // SoftDeleteWithReasonTx performs a soft delete within an optional transaction -// If tx is nil, executes directly against the database +// If tx is nil, runs in a transaction of its own // Returns rows affected count for callers that need to check idempotency // This method is used by both the repository and the Jetstream consumer func (r *postgresCommentRepo) SoftDeleteWithReasonTx(ctx context.Context, tx *sql.Tx, uri, reason, deletedByDID string) (int64, error) { + if tx == nil { + // The delete and the snapshot withdrawal below commit together. + ownTx, err := r.db.BeginTx(ctx, nil) + if err != nil { + return 0, fmt.Errorf("failed to begin comment delete transaction: %w", err) + } + defer func() { + if rollbackErr := ownTx.Rollback(); rollbackErr != nil && !errors.Is(rollbackErr, sql.ErrTxDone) { + log.Printf("Failed to roll back comment delete transaction: %v", rollbackErr) + } + }() + rowsAffected, err := r.SoftDeleteWithReasonTx(ctx, ownTx, uri, reason, deletedByDID) + if err != nil { + return 0, err + } + if err := ownTx.Commit(); err != nil { + return 0, fmt.Errorf("failed to commit comment delete: %w", err) + } + return rowsAffected, nil + } + query := ` UPDATE comments SET @@ -258,15 +279,7 @@ func (r *postgresCommentRepo) SoftDeleteWithReasonTx(ctx context.Context, tx *sq WHERE uri = $1 AND deleted_at IS NULL ` - var result sql.Result - var err error - - if tx != nil { - result, err = tx.ExecContext(ctx, query, uri, reason, deletedByDID) - } else { - result, err = r.db.ExecContext(ctx, query, uri, reason, deletedByDID) - } - + result, err := tx.ExecContext(ctx, query, uri, reason, deletedByDID) if err != nil { return 0, fmt.Errorf("failed to soft delete comment: %w", err) } @@ -276,6 +289,24 @@ func (r *postgresCommentRepo) SoftDeleteWithReasonTx(ctx context.Context, tx *sq return 0, fmt.Errorf("failed to check delete result: %w", err) } + if rowsAffected > 0 && reason == comments.DeletionReasonAuthor { + // The author's delete withdraws the comment's removal snapshots from the + // public log for good, even if a new record later re-creates the URI. This + // runs after the UPDATE above, which waits on an in-flight removal's share + // lock on the comment row, so under READ COMMITTED this statement sees + // that removal's committed snapshot. + if _, err := tx.ExecContext(ctx, ` + UPDATE moderation_content_snapshots snapshot + SET author_deleted_at = NOW() + FROM moderation_actions action + WHERE action.id = snapshot.action_id + AND action.subject_uri = $1 + AND snapshot.author_deleted_at IS NULL + `, uri); err != nil { + return 0, fmt.Errorf("failed to withdraw removal snapshots of deleted comment: %w", err) + } + } + return rowsAffected, nil } diff --git a/internal/db/postgres/community_subscriber_recount_migration_test.go b/internal/db/postgres/community_subscriber_recount_migration_test.go index 9efe5c9..f8db788 100644 --- a/internal/db/postgres/community_subscriber_recount_migration_test.go +++ b/internal/db/postgres/community_subscriber_recount_migration_test.go @@ -22,6 +22,8 @@ func TestMigration045RecountsAndMaintainsCommunitySubscribers(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/credential_reencrypt_migration_test.go b/internal/db/postgres/credential_reencrypt_migration_test.go index eede460..4cd50f8 100644 --- a/internal/db/postgres/credential_reencrypt_migration_test.go +++ b/internal/db/postgres/credential_reencrypt_migration_test.go @@ -27,6 +27,8 @@ func TestMigration046DownRestoresUsableEncryptionKey(t *testing.T) { db := testkit.DB(t) assert.False(t, credentialReencryptKeyTable(t, db).Valid, "migration 046 Up must drop encryption_keys before its Down behavior can be tested") + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), @@ -407,6 +409,8 @@ func TestCredentialReencryptRejectsLegacyRowsAfterEncryptionKeysDropped(t *testi func credentialReencryptVersion45Database(t *testing.T) *sql.DB { t.Helper() db := testkit.DB(t) + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/discover_hot_migration_test.go b/internal/db/postgres/discover_hot_migration_test.go index 2e5d610..e0b575b 100644 --- a/internal/db/postgres/discover_hot_migration_test.go +++ b/internal/db/postgres/discover_hot_migration_test.go @@ -35,6 +35,9 @@ func TestMigration048DiscoverHotSnapshots(t *testing.T) { _, err := db.ExecContext(ctx, `CREATE TABLE discover_hot_migration_sentinel (id BIGINT PRIMARY KEY)`) require.NoError(t, err) + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") + require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/future_comment_created_at_migration_test.go b/internal/db/postgres/future_comment_created_at_migration_test.go index c2adfa9..ad9559b 100644 --- a/internal/db/postgres/future_comment_created_at_migration_test.go +++ b/internal/db/postgres/future_comment_created_at_migration_test.go @@ -18,6 +18,8 @@ func TestMigration041_ClampsFutureCommentCreatedAt(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/moderation_action_log_restricted_integration_test.go b/internal/db/postgres/moderation_action_log_restricted_integration_test.go index 85366bb..ff38fa6 100644 --- a/internal/db/postgres/moderation_action_log_restricted_integration_test.go +++ b/internal/db/postgres/moderation_action_log_restricted_integration_test.go @@ -41,11 +41,11 @@ func restrictedLogComment(t *testing.T, db *sql.DB, authorDID string, root moder return subject } -func restrictedLogRemove(t *testing.T, service moderation.Service, subject moderation.StrongRef) string { +func restrictedLogRemove(t *testing.T, service moderation.Service, subject moderation.StrongRef, reason string) string { t.Helper() result, err := service.RemoveContent(t.Context(), fixtures.DID("restrictedlogadmin"), moderation.RemoveContentRequest{ Subject: subject, ExpectedVersion: "v0", IdempotencyKey: testkit.UniqueIDWithPrefix(t, "remove"), - Reason: "social.coves.moderation.defs#reasonSpam", + Reason: reason, }) require.NoError(t, err) require.Equal(t, moderation.OutcomeApplied, result.Outcome) @@ -102,9 +102,17 @@ func TestModerationActionLogRestrictsOnlyUnindexedSubjects(t *testing.T) { pending := post("pending", "pending") orphanRoot := post("orphan root", "") erasedPost := post("erased post", "") - erasedComment := restrictedLogComment(t, db, authorDID, accepted) + // roots records each comment's root post URI as seeded. + roots := map[string]string{} + comment := func(root moderation.StrongRef) moderation.StrongRef { + subject := restrictedLogComment(t, db, authorDID, root) + roots[subject.URI] = root.URI + return subject + } + erasedComment := comment(accepted) + orphanComment := comment(orphanRoot) softDeletedPost := post("soft deleted post", "") - softDeletedComment := restrictedLogComment(t, db, authorDID, accepted) + softDeletedComment := comment(accepted) type entry struct { subject moderation.StrongRef @@ -119,8 +127,8 @@ func TestModerationActionLogRestrictsOnlyUnindexedSubjects(t *testing.T) { {post("community removed", "removed"), moderation.PostV2Collection, false}, {post("unadmitted", "unadmitted"), moderation.PostV2Collection, false}, {indexedModerationPost(t, db, moderation.LegacyPostCollection, communityDID, authorDID, "legacy", ""), moderation.LegacyPostCollection, false}, - {restrictedLogComment(t, db, authorDID, pending), moderation.CommentCollection, false}, - {restrictedLogComment(t, db, authorDID, orphanRoot), moderation.CommentCollection, false}, + {comment(pending), moderation.CommentCollection, false}, + {orphanComment, moderation.CommentCollection, false}, {softDeletedPost, moderation.PostV2Collection, false}, {softDeletedComment, moderation.CommentCollection, false}, {erasedPost, moderation.PostV2Collection, true}, @@ -129,7 +137,7 @@ func TestModerationActionLogRestrictsOnlyUnindexedSubjects(t *testing.T) { entries := map[string]entry{} var publicIDs []string for _, subject := range subjects { - id := restrictedLogRemove(t, service, subject.subject) + id := restrictedLogRemove(t, service, subject.subject, "social.coves.moderation.defs#reasonSpam") entries[id] = subject if !subject.restricted { publicIDs = append([]string{id}, publicIDs...) @@ -137,7 +145,8 @@ func TestModerationActionLogRestrictsOnlyUnindexedSubjects(t *testing.T) { } // Account erasure hard-deletes rows; an author's delete only sets // deleted_at, so the row stays and the entry stays public. A comment whose - // root post is gone keeps its own row, so it stays public too. + // root post is gone keeps its own row, so it stays public too, without its + // rootUri, which only the admin log carries (decided 2026-10-13). for _, statement := range []struct{ sql, uri string }{ {`UPDATE posts SET deleted_at = NOW() WHERE uri = $1`, softDeletedPost.URI}, {`UPDATE comments SET deleted_at = NOW() WHERE uri = $1`, softDeletedComment.URI}, @@ -148,6 +157,23 @@ func TestModerationActionLogRestrictsOnlyUnindexedSubjects(t *testing.T) { _, err := db.ExecContext(t.Context(), statement.sql, statement.uri) require.NoError(t, err) } + // subjectRef is the subject reference JSON a log entry carries; a comment's + // names the post that roots its thread. + subjectRef := func(want entry) map[string]any { + ref := map[string]any{"uri": want.subject.URI, "cid": want.subject.CID, "communityDid": communityDID} + if root, isComment := roots[want.subject.URI]; isComment { + ref["rootUri"] = root + } + return ref + } + // publicRef is the subject reference the public view gives. + publicRef := func(want entry) map[string]any { + ref := subjectRef(want) + if want.subject.URI == orphanComment.URI { + delete(ref, "rootUri") + } + return ref + } publicIn := func(collection string) []string { var ids []string for _, id := range publicIDs { @@ -168,7 +194,7 @@ func TestModerationActionLogRestrictsOnlyUnindexedSubjects(t *testing.T) { assert.Nil(t, action.Subject, "an erased subject must not appear in the public log") assert.NotContains(t, encoded, want.subject.URI) } else { - assert.Equal(t, &moderation.SubjectRefView{URI: want.subject.URI, CID: want.subject.CID, CommunityDID: communityDID}, action.Subject) + assert.Equal(t, publicRef(want), snapshotObject(t, action.Subject)) } } }) @@ -179,12 +205,15 @@ func TestModerationActionLogRestrictsOnlyUnindexedSubjects(t *testing.T) { require.Len(t, page.Actions, len(entries)) for _, action := range page.Actions { want := entries[action.Action.Ref.ActionID] - ref := &moderation.SubjectRefView{URI: want.subject.URI, CID: want.subject.CID, CommunityDID: communityDID} + ref := subjectRef(want) if want.restricted { assert.Nil(t, action.Action.Subject) - assert.Equal(t, ref, action.PrivateSubject) + assert.Equal(t, ref, snapshotObject(t, action.PrivateSubject)) + } else if want.subject.URI == orphanComment.URI { + assert.Equal(t, publicRef(want), snapshotObject(t, action.Action.Subject)) + assert.Equal(t, ref, snapshotObject(t, action.PrivateSubject)) } else { - assert.Equal(t, ref, action.Action.Subject) + assert.Equal(t, ref, snapshotObject(t, action.Action.Subject)) assert.Nil(t, action.PrivateSubject) } } diff --git a/internal/db/postgres/moderation_content_snapshot_erasure_integration_test.go b/internal/db/postgres/moderation_content_snapshot_erasure_integration_test.go new file mode 100644 index 0000000..ccdd6fc --- /dev/null +++ b/internal/db/postgres/moderation_content_snapshot_erasure_integration_test.go @@ -0,0 +1,122 @@ +//go:build integration + +package postgres_test + +import ( + "context" + "database/sql" + "testing" + "time" + + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func snapshotActionCount(t *testing.T, db *sql.DB, actionID string) int { + t.Helper() + var count int + require.NoError(t, db.QueryRowContext(t.Context(), `SELECT count(*) FROM moderation_actions WHERE id = $1`, actionID).Scan(&count)) + return count +} + +// Erasing an account deletes the snapshots of its removed content, found by +// the human author rather than by the repository that holds the record: a +// legacy post lives in its community's repository. The action rows stay, and +// other authors' snapshots are untouched. +func TestModerationContentSnapshotAuthorErasure(t *testing.T) { + db := testkit.DB(t) + communityDID, erasedDID := moderationPostActors(t, db) + keptName := testkit.UniqueIDWithPrefix(t, "snapkept") + keptDID := fixtures.DID(keptName) + fixtures.User(t, db, keptName+".test", keptDID) + service := restrictedLogService(db) + + keptPost := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, keptDID, "Kept author title", "") + legacyPost := indexedModerationPost(t, db, moderation.LegacyPostCollection, communityDID, erasedDID, "Erased legacy title", "") + erasedComment := restrictedLogComment(t, db, erasedDID, keptPost) + erasedRemovals := map[string]string{ + "legacy post": restrictedLogRemove(t, service, legacyPost, snapshotReasonSpam), + "comment": restrictedLogRemove(t, service, erasedComment, snapshotReasonSpam), + } + keptRemoval := restrictedLogRemove(t, service, keptPost, snapshotReasonSpam) + for name, id := range erasedRemovals { + _, admin := snapshotLogEntries(t, service, id) + require.Contains(t, admin, "contentSnapshot", "the %s removal must keep a snapshot before erasure", name) + } + + require.NoError(t, postgres.NewUserRepository(db).Delete(t.Context(), erasedDID)) + + for name, id := range erasedRemovals { + _, admin := snapshotLogEntries(t, service, id) + assert.NotContains(t, admin, "contentSnapshot", "the erased author's %s snapshot must be gone", name) + assert.Contains(t, admin, "privateSubject", "the %s log entry stays for admins", name) + } + _, admin := snapshotLogEntries(t, service, keptRemoval) + assert.Equal(t, map[string]any{"title": "Kept author title", "text": "indexed post content", "capturedCid": moderationPostCID}, admin["contentSnapshot"]) + + for name, id := range erasedRemovals { + assert.Equal(t, 1, snapshotActionCount(t, db, id), "the %s action row stays", name) + assert.Equal(t, 0, snapshotRowCount(t, db, id), "the %s snapshot row is deleted", name) + } + assert.Equal(t, 1, snapshotRowCount(t, db, keptRemoval), "another author's snapshot stays") +} + +// An erasure that runs while a removal of the author's post is in flight must +// still delete the snapshot that removal commits: erasure waits on the +// removal's share lock on the post row, and sweeps snapshots after that. +func TestModerationContentSnapshotErasureWaitsForInFlightRemoval(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "In-flight removal title", "") + gate, release := holdRestoreBeforeCommit(t, db) + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + removed := make(chan moderationConcurrentOutcome, 1) + go func() { + result, err := service.RemoveContent(ctx, fixtures.DID("snapshotraceadmin"), moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v0", IdempotencyKey: "snapshot-race-remove", Reason: snapshotReasonSpam, + }) + removed <- moderationConcurrentOutcome{result: result, err: err} + }() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + var parked int + err := gate.QueryRowContext(ctx, ` + SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() + AND wait_event_type = 'Lock' AND wait_event = 'advisory' + AND query LIKE '%INSERT INTO moderation_idempotency_keys%' + `).Scan(&parked) + return parked == 1, err + }, testkit.WithDescription("the removal parked at its last statement, after its snapshot write")) + + erased := make(chan error, 1) + go func() { erased <- postgres.NewUserRepository(db).Delete(ctx, authorDID) }() + testkit.WaitFor(t, 5*time.Second, func() (bool, error) { + var waiting int + err := gate.QueryRowContext(ctx, ` + SELECT count(*) FROM pg_stat_activity + WHERE datname = current_database() AND pid <> pg_backend_pid() + AND wait_event_type = 'Lock' AND wait_event <> 'advisory' + AND query LIKE '%DELETE FROM posts WHERE author_did%' + `).Scan(&waiting) + return waiting == 1, err + }, testkit.WithDescription("the erasure waiting on the in-flight removal's lock on the post row")) + + release() + outcome := <-removed + require.NoError(t, outcome.err) + require.NotNil(t, outcome.result) + require.Equal(t, moderation.OutcomeApplied, outcome.result.Outcome) + require.NotNil(t, outcome.result.Action) + require.NoError(t, <-erased) + + assert.Equal(t, 1, snapshotActionCount(t, db, outcome.result.Action.ID), "the removal's action row stays") + assert.Equal(t, 0, snapshotRowCount(t, db, outcome.result.Action.ID), "no snapshot of the erased author may survive the race") +} diff --git a/internal/db/postgres/moderation_content_snapshot_integration_test.go b/internal/db/postgres/moderation_content_snapshot_integration_test.go new file mode 100644 index 0000000..2b8e20e --- /dev/null +++ b/internal/db/postgres/moderation_content_snapshot_integration_test.go @@ -0,0 +1,424 @@ +//go:build integration + +package postgres_test + +import ( + "database/sql" + "encoding/json" + "testing" + + "Coves/internal/core/moderation" + "Coves/tests/fixtures" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +const ( + snapshotReasonSpam = "social.coves.moderation.defs#reasonSpam" + snapshotReasonDoxing = "social.coves.moderation.defs#reasonDoxing" + snapshotReasonIllegal = "social.coves.moderation.defs#reasonIllegalContent" + snapshotReasonDiscretion = "social.coves.moderation.defs#reasonModeratorDiscretion" + snapshotLinkURL = "https://news.example/snapshot-story" + snapshotLinkEmbed = `{"$type":"social.coves.embed.external","external":{"uri":"https://news.example/snapshot-story","title":"Snapshot story card"}}` + snapshotEditedCID = "bafyreidfcytgqwl6jyx5nhsofbvq4dibimtbx3uwckcs2wtwulwsz7jvbm" + snapshotMalformedEmbed = `{"$type":"social.coves.embed.external","external":{"uri":"not a uri","title":"Malformed link card"}}` +) + +// snapshotObject is the JSON object a log view serializes to; nil when the +// value is absent. +func snapshotObject(t *testing.T, value any) map[string]any { + t.Helper() + encoded, err := json.Marshal(value) + require.NoError(t, err) + var object map[string]any + require.NoError(t, json.Unmarshal(encoded, &object)) + return object +} + +// snapshotLogEntries returns an action's public log entry, nil when the public +// log does not list it, and its admin log entry. +func snapshotLogEntries(t *testing.T, service moderation.Service, actionID string) (public, admin map[string]any) { + t.Helper() + adminPage, err := service.ListAdminActions(t.Context(), moderation.ListAdminActionsParams{ActionID: actionID}) + require.NoError(t, err) + require.Len(t, adminPage.Actions, 1) + admin = snapshotObject(t, adminPage.Actions[0]) + publicPage, _ := restrictedLogPage(t, service, moderation.ListActionsParams{}) + for _, action := range publicPage.Actions { + if action.Ref.ActionID == actionID { + public = snapshotObject(t, action) + } + } + return public, admin +} + +// snapshotAdminSubject is the admin entry's subject reference, wherever the +// projection put it. +func snapshotAdminSubject(t *testing.T, admin map[string]any) map[string]any { + t.Helper() + if subject := snapshotObject(t, snapshotObject(t, admin["action"])["subject"]); subject != nil { + return subject + } + return snapshotObject(t, admin["privateSubject"]) +} + +func snapshotRowCount(t *testing.T, db *sql.DB, actionID string) int { + t.Helper() + var count int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_content_snapshots WHERE action_id = $1 + `, actionID).Scan(&count)) + return count +} + +// A removal keeps the text of the version the moderator saw: title, body and +// external link of a post, the body of a comment. Doxing removals are kept for +// admins only; illegal-content removals are never kept. A comment action +// records the post that roots its thread. +func TestModerationContentSnapshotCapture(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + post := func(title, embed string) moderation.StrongRef { + return indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, title, embed) + } + root := post("Snapshot thread root", "") + emptyPost := post("", "") + _, err := db.ExecContext(t.Context(), `UPDATE posts SET title = NULL, content = NULL, embed = NULL WHERE uri = $1`, emptyPost.URI) + require.NoError(t, err) + // The comment consumer only loosely validates a root URI, so an index row + // can hold one the lexicon's at-uri format rejects. + malformedRootComment := restrictedLogComment(t, db, authorDID, root) + _, err = db.ExecContext(t.Context(), `UPDATE comments SET root_uri = $2 WHERE uri = $1`, + malformedRootComment.URI, "at://did:plc:rootauthor/social.coves.community.postv2/3abc/extra") + require.NoError(t, err) + + for _, test := range []struct { + name string + subject moderation.StrongRef + reason string + snapshot map[string]any + rootURI string + public bool + }{ + { + name: "spam post removal keeps title, body and link", subject: post("Snapshot spam title", snapshotLinkEmbed), reason: snapshotReasonSpam, + snapshot: map[string]any{"title": "Snapshot spam title", "text": "indexed post content", "linkUrl": snapshotLinkURL, "capturedCid": moderationPostCID}, + public: true, + }, + { + name: "comment removal keeps its body and records its root post", subject: restrictedLogComment(t, db, authorDID, root), reason: snapshotReasonSpam, + snapshot: map[string]any{"text": "restricted log comment", "capturedCid": moderationCommentCID}, + rootURI: root.URI, public: true, + }, + { + name: "comment whose indexed root is not a valid at-uri records no root", subject: malformedRootComment, reason: snapshotReasonSpam, + snapshot: map[string]any{"text": "restricted log comment", "capturedCid": moderationCommentCID}, + public: true, + }, + { + name: "doxing removal is kept for admins only", subject: post("Snapshot doxing title", snapshotLinkEmbed), reason: snapshotReasonDoxing, + snapshot: map[string]any{"title": "Snapshot doxing title", "text": "indexed post content", "linkUrl": snapshotLinkURL, "capturedCid": moderationPostCID}, + }, + { + name: "illegal-content removal is never kept", subject: post("Snapshot illegal title", snapshotLinkEmbed), reason: snapshotReasonIllegal, + }, + { + name: "post without title, body or embed keeps only its CID", subject: emptyPost, reason: snapshotReasonSpam, + snapshot: map[string]any{"capturedCid": moderationPostCID}, public: true, + }, + { + name: "image post keeps its text and no blob reference", subject: post("Snapshot image title", moderationTwoImageEmbed()), reason: snapshotReasonSpam, + snapshot: map[string]any{"title": "Snapshot image title", "text": "indexed post content", "capturedCid": moderationPostCID}, + public: true, + }, + { + name: "external embed with a malformed uri keeps no link", subject: post("Snapshot malformed link title", snapshotMalformedEmbed), reason: snapshotReasonSpam, + snapshot: map[string]any{"title": "Snapshot malformed link title", "text": "indexed post content", "capturedCid": moderationPostCID}, + public: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + id := restrictedLogRemove(t, service, test.subject, test.reason) + public, admin := snapshotLogEntries(t, service, id) + + if test.snapshot == nil { + assert.NotContains(t, admin, "contentSnapshot") + } else { + assert.Equal(t, test.snapshot, admin["contentSnapshot"]) + } + adminSubject := snapshotAdminSubject(t, admin) + require.NotNil(t, adminSubject) + if test.rootURI == "" { + assert.NotContains(t, adminSubject, "rootUri", "only a comment with a valid root carries a root") + } else { + assert.Equal(t, test.rootURI, adminSubject["rootUri"]) + } + + require.NotNil(t, public, "every removal is listed in the public log") + if test.public { + assert.Equal(t, test.snapshot, public["contentSnapshot"]) + publicSubject := snapshotObject(t, public["subject"]) + require.NotNil(t, publicSubject) + if test.rootURI == "" { + assert.NotContains(t, publicSubject, "rootUri") + } else { + assert.Equal(t, test.rootURI, publicSubject["rootUri"]) + } + } else { + assert.NotContains(t, public, "contentSnapshot") + assert.NotContains(t, public, "subject") + } + + state, err := service.GetSubjectState(t.Context(), test.subject.URI) + require.NoError(t, err, "the subject state must still read a removal that carries a snapshot") + require.NotNil(t, state) + require.NotNil(t, state.LocalRemoval) + assert.Equal(t, id, state.LocalRemoval.ActionID) + assert.Equal(t, moderation.ModerationStateRemoved, state.Moderation.State) + + var stored string + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT stored_result::text FROM moderation_idempotency_keys WHERE stored_result->'Action'->>'ID' = $1 + `, id).Scan(&stored)) + for _, field := range []string{"title", "text", "linkUrl"} { + if value, present := test.snapshot[field]; present { + assert.NotContains(t, stored, value, "the stored idempotency result must not copy the removed %s", field) + } + } + + want := 1 + if test.snapshot == nil { + want = 0 + } + assert.Equal(t, want, snapshotRowCount(t, db, id)) + }) + } +} + +// Once its author deletes the subject, the public log still names it but no +// longer shows its snapshot; the admin log keeps it (decided 2026-10-13). A +// post already deleted by its author can still be removed, and its snapshot is +// kept for admins. A comment already deleted by its author has lost its text, +// so its removal keeps no snapshot. +func TestModerationContentSnapshotWithheldAfterAuthorDeletion(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + root := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "Author deletion thread root", "") + // The SQL the post and comment consumers run when an author deletes. + const deletePost = `UPDATE posts SET deleted_at = NOW() WHERE uri = $1` + const deleteComment = ` + UPDATE comments SET content = '', content_facets = NULL, embed = NULL, content_labels = NULL, + deleted_at = NOW(), deletion_reason = 'author', deleted_by = commenter_did + WHERE uri = $1` + + for _, test := range []struct { + name string + subject moderation.StrongRef + deleteSQL string + deletedBefore bool + snapshot map[string]any + }{ + { + name: "post deleted after its removal", + subject: indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "Deleted after removal title", snapshotLinkEmbed), + deleteSQL: deletePost, + snapshot: map[string]any{"title": "Deleted after removal title", "text": "indexed post content", "linkUrl": snapshotLinkURL, "capturedCid": moderationPostCID}, + }, + { + name: "post deleted before its removal", + subject: indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "Deleted before removal title", snapshotLinkEmbed), + deleteSQL: deletePost, deletedBefore: true, + snapshot: map[string]any{"title": "Deleted before removal title", "text": "indexed post content", "linkUrl": snapshotLinkURL, "capturedCid": moderationPostCID}, + }, + { + name: "comment deleted after its removal", + subject: restrictedLogComment(t, db, authorDID, root), + deleteSQL: deleteComment, + snapshot: map[string]any{"text": "restricted log comment", "capturedCid": moderationCommentCID}, + }, + { + name: "comment deleted before its removal", + subject: restrictedLogComment(t, db, authorDID, root), + deleteSQL: deleteComment, deletedBefore: true, + }, + } { + t.Run(test.name, func(t *testing.T) { + authorDelete := func() { + _, err := db.ExecContext(t.Context(), test.deleteSQL, test.subject.URI) + require.NoError(t, err) + } + if test.deletedBefore { + authorDelete() + } + id := restrictedLogRemove(t, service, test.subject, snapshotReasonSpam) + if !test.deletedBefore { + authorDelete() + } + + public, admin := snapshotLogEntries(t, service, id) + if test.snapshot == nil { + assert.NotContains(t, admin, "contentSnapshot") + assert.Equal(t, 0, snapshotRowCount(t, db, id), "a removal of an author-deleted comment keeps no snapshot") + } else { + assert.Equal(t, test.snapshot, admin["contentSnapshot"], "the admin log keeps the snapshot") + } + + require.NotNil(t, public, "the removal stays in the public log") + publicSubject := snapshotObject(t, public["subject"]) + require.NotNil(t, publicSubject, "the public log still names an author-deleted subject") + assert.Equal(t, test.subject.URI, publicSubject["uri"]) + assert.NotContains(t, public, "contentSnapshot", "the public log withholds an author-deleted subject's snapshot") + publicJSON, err := json.Marshal(public) + require.NoError(t, err) + for _, field := range []string{"title", "text", "linkUrl"} { + if value, present := test.snapshot[field]; present { + assert.NotContains(t, string(publicJSON), value, "the public log must not show the deleted subject's %s", field) + } + } + }) + } +} + +// A doxing or illegal-content removal deletes every earlier snapshot of its +// subject, for admins too (decided 2026-10-13): a subject first removed as spam, +// restored and removed again for doxing no longer shows the spam removal's +// text. The doxing removal keeps its own admin-only snapshot. +func TestModerationContentSnapshotHiddenRemovalDeletesEarlierSnapshots(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + actor := fixtures.DID("snapshothiddenadmin") + + for _, test := range []struct { + name string + title string + reason string + snapshot map[string]any + rows int + }{ + { + name: "doxing removal", title: "Spam then doxing title", reason: snapshotReasonDoxing, + snapshot: map[string]any{"title": "Spam then doxing title", "text": "indexed post content", "linkUrl": snapshotLinkURL, "capturedCid": moderationPostCID}, + rows: 1, + }, + {name: "illegal-content removal", title: "Spam then illegal title", reason: snapshotReasonIllegal}, + } { + t.Run(test.name, func(t *testing.T) { + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, test.title, snapshotLinkEmbed) + spamID := restrictedLogRemove(t, service, subject, snapshotReasonSpam) + restored, err := service.RestoreContent(t.Context(), actor, moderation.RestoreContentRequest{ + ActionID: spamID, ReviewedSubject: &subject, ExpectedVersion: "v1", + IdempotencyKey: testkit.UniqueIDWithPrefix(t, "restore"), Reason: snapshotReasonDiscretion, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + removed, err := service.RemoveContent(t.Context(), actor, moderation.RemoveContentRequest{ + Subject: subject, ExpectedVersion: "v2", IdempotencyKey: testkit.UniqueIDWithPrefix(t, "remove"), Reason: test.reason, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, removed.Outcome) + require.NotNil(t, removed.Action) + + spamPublic, spamAdmin := snapshotLogEntries(t, service, spamID) + assert.NotContains(t, spamAdmin, "contentSnapshot", "the spam removal's snapshot is gone for admins") + require.NotNil(t, spamPublic, "the spam removal stays in the public log") + assert.NotContains(t, spamPublic, "contentSnapshot", "the spam removal's snapshot is gone from the public log") + assert.Equal(t, 0, snapshotRowCount(t, db, spamID)) + + _, hiddenAdmin := snapshotLogEntries(t, service, removed.Action.ID) + if test.snapshot == nil { + assert.NotContains(t, hiddenAdmin, "contentSnapshot") + } else { + assert.Equal(t, test.snapshot, hiddenAdmin["contentSnapshot"], "the doxing removal keeps its own admin-only snapshot") + } + + var subjectRows int + require.NoError(t, db.QueryRowContext(t.Context(), ` + SELECT count(*) FROM moderation_content_snapshots s + JOIN moderation_actions a ON a.id = s.action_id + WHERE a.subject_uri = $1 + `, subject.URI).Scan(&subjectRows)) + assert.Equal(t, test.rows, subjectRows) + }) + } +} + +// Only removals keep a snapshot, and a removal's snapshot outlives its restore +// and later edits of the indexed post: it stays the version the moderator saw. +func TestModerationContentSnapshotOnlyRemovalsKeepItAndEditsDoNotChangeIt(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + actor := fixtures.DID("snapshotlifecycleadmin") + subject := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "Original snapshot title", snapshotLinkEmbed) + removalID := restrictedLogRemove(t, service, subject, snapshotReasonSpam) + + _, err := db.ExecContext(t.Context(), ` + UPDATE posts SET title = 'Edited snapshot title', content = 'edited post content', embed = NULL, cid = $2 WHERE uri = $1 + `, subject.URI, snapshotEditedCID) + require.NoError(t, err) + edited := moderation.StrongRef{URI: subject.URI, CID: snapshotEditedCID} + restored, err := service.RestoreContent(t.Context(), actor, moderation.RestoreContentRequest{ + ActionID: removalID, ReviewedSubject: &edited, ExpectedVersion: "v1", + IdempotencyKey: "snapshot-restore", Reason: snapshotReasonDiscretion, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + require.NotNil(t, restored.Action) + labelled := labelPost(t, service, actor, edited, "v2", "snapshot-label") + retracted, err := service.RetractContentLabel(t.Context(), actor, moderation.RetractContentLabelRequest{ + ActionID: labelled.Action.ID, ReviewedSubject: &edited, ExpectedVersion: "v3", IdempotencyKey: "snapshot-retract", + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, retracted.Outcome) + require.NotNil(t, retracted.Action) + + public, admin := snapshotLogEntries(t, service, removalID) + original := map[string]any{"title": "Original snapshot title", "text": "indexed post content", "linkUrl": snapshotLinkURL, "capturedCid": moderationPostCID} + assert.Equal(t, original, admin["contentSnapshot"], "a restore keeps the snapshot and later edits do not change it") + assert.Equal(t, original, public["contentSnapshot"]) + + others := map[string]string{ + moderation.ActionRestore: restored.Action.ID, moderation.ActionLabel: labelled.Action.ID, + moderation.ActionRetractLabel: retracted.Action.ID, + } + for kind, id := range others { + _, admin := snapshotLogEntries(t, service, id) + assert.NotContains(t, admin, "contentSnapshot", kind) + } + assert.Equal(t, 1, snapshotRowCount(t, db, removalID)) + for kind, id := range others { + assert.Equal(t, 0, snapshotRowCount(t, db, id), "a %s action keeps no snapshot", kind) + } +} + +// A restore records the root of the comment it restores, read from its +// removal, so it keeps the root after the comment row is gone. +func TestModerationContentSnapshotRestoreKeepsCommentRoot(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + root := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "Restore root post", "") + comment := restrictedLogComment(t, db, authorDID, root) + removalID := restrictedLogRemove(t, service, comment, snapshotReasonSpam) + _, err := db.ExecContext(t.Context(), `DELETE FROM comments WHERE uri = $1`, comment.URI) + require.NoError(t, err) + + restored, err := service.RestoreContent(t.Context(), fixtures.DID("snapshotrootadmin"), moderation.RestoreContentRequest{ + ActionID: removalID, ExpectedVersion: "v1", IdempotencyKey: "snapshot-root-restore", Reason: snapshotReasonDiscretion, + }) + require.NoError(t, err) + require.Equal(t, moderation.OutcomeApplied, restored.Outcome) + require.NotNil(t, restored.Action) + + _, admin := snapshotLogEntries(t, service, restored.Action.ID) + subject := snapshotAdminSubject(t, admin) + require.NotNil(t, subject) + assert.Equal(t, comment.URI, subject["uri"]) + assert.Equal(t, root.URI, subject["rootUri"]) + assert.NotContains(t, admin, "contentSnapshot") +} diff --git a/internal/db/postgres/moderation_content_snapshot_visibility_integration_test.go b/internal/db/postgres/moderation_content_snapshot_visibility_integration_test.go new file mode 100644 index 0000000..74dd25c --- /dev/null +++ b/internal/db/postgres/moderation_content_snapshot_visibility_integration_test.go @@ -0,0 +1,126 @@ +//go:build integration + +package postgres_test + +import ( + "database/sql" + "encoding/json" + "testing" + + "Coves/internal/core/comments" + "Coves/internal/core/moderation" + "Coves/internal/db/postgres" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// Once its author deletes a removed comment, the comment's snapshot stays out +// of the public log, even after a new record re-creates the comment's URI; the +// admin log keeps it (decided 2026-10-13). +func TestModerationContentSnapshotWithheldAfterAuthorDeletedCommentIsRecreated(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + repository, ok := postgres.NewCommentRepository(db).(comments.RepositoryTx) + require.True(t, ok) + root := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "Recreated comment thread root", "") + + for _, test := range []struct { + name string + inTransaction bool + }{ + {name: "deleted inside the consumer's transaction", inTransaction: true}, + {name: "deleted without a transaction"}, + } { + t.Run(test.name, func(t *testing.T) { + subject := restrictedLogComment(t, db, authorDID, root) + id := restrictedLogRemove(t, service, subject, snapshotReasonSpam) + + if test.inTransaction { + tx, err := db.BeginTx(t.Context(), nil) + require.NoError(t, err) + defer tx.Rollback() + deleted, err := repository.SoftDeleteWithReasonTx(t.Context(), tx, subject.URI, comments.DeletionReasonAuthor, authorDID) + require.NoError(t, err) + require.Equal(t, int64(1), deleted) + require.NoError(t, tx.Commit()) + } else { + deleted, err := repository.SoftDeleteWithReasonTx(t.Context(), nil, subject.URI, comments.DeletionReasonAuthor, authorDID) + require.NoError(t, err) + require.Equal(t, int64(1), deleted) + } + // The fields the comment consumer's resurrection sets when a new + // record re-creates a deleted comment's URI. + _, err := db.ExecContext(t.Context(), ` + UPDATE comments SET cid = $2, content = 'recreated comment text', + deleted_at = NULL, deletion_reason = NULL, deleted_by = NULL + WHERE uri = $1 + `, subject.URI, snapshotEditedCID) + require.NoError(t, err) + + public, admin := snapshotLogEntries(t, service, id) + assert.Equal(t, map[string]any{"text": "restricted log comment", "capturedCid": moderationCommentCID}, admin["contentSnapshot"], + "the admin log keeps the snapshot") + require.NotNil(t, public, "the removal stays in the public log") + publicSubject := snapshotObject(t, public["subject"]) + require.NotNil(t, publicSubject) + assert.Equal(t, subject.URI, publicSubject["uri"]) + assert.NotContains(t, public, "contentSnapshot", "the re-created comment must not bring back the deleted text") + publicJSON, err := json.Marshal(public) + require.NoError(t, err) + assert.NotContains(t, string(publicJSON), "restricted log comment") + }) + } +} + +// The public log omits a comment's rootUri when the root post has a doxing or +// illegal-content action or is no longer indexed; the admin log carries it in +// privateSubject (decided 2026-10-13). +func TestModerationContentSnapshotRootURIWithheldForHiddenOrUnindexedRoot(t *testing.T) { + db := testkit.DB(t) + communityDID, authorDID := moderationPostActors(t, db) + service := restrictedLogService(db) + + for _, test := range []struct { + name string + withholdRoot func(t *testing.T, db *sql.DB, root moderation.StrongRef) + }{ + { + name: "root post removed for doxing", + withholdRoot: func(t *testing.T, _ *sql.DB, root moderation.StrongRef) { + restrictedLogRemove(t, service, root, snapshotReasonDoxing) + }, + }, + { + name: "root post no longer indexed", + withholdRoot: func(t *testing.T, db *sql.DB, root moderation.StrongRef) { + _, err := db.ExecContext(t.Context(), `DELETE FROM posts WHERE uri = $1`, root.URI) + require.NoError(t, err) + }, + }, + } { + t.Run(test.name, func(t *testing.T) { + root := indexedModerationPost(t, db, moderation.PostV2Collection, communityDID, authorDID, "Withheld root title", "") + subject := restrictedLogComment(t, db, authorDID, root) + id := restrictedLogRemove(t, service, subject, snapshotReasonSpam) + test.withholdRoot(t, db, root) + + public, admin := snapshotLogEntries(t, service, id) + require.NotNil(t, public, "the comment's removal stays in the public log") + publicSubject := snapshotObject(t, public["subject"]) + require.NotNil(t, publicSubject, "the public log still names the comment") + assert.Equal(t, subject.URI, publicSubject["uri"]) + assert.NotContains(t, publicSubject, "rootUri") + publicJSON, err := json.Marshal(public) + require.NoError(t, err) + assert.NotContains(t, string(publicJSON), root.URI, "the public log must not name the withheld root") + + privateSubject := snapshotObject(t, admin["privateSubject"]) + require.NotNil(t, privateSubject, "the admin log carries the withheld root") + assert.Equal(t, subject.URI, privateSubject["uri"]) + assert.Equal(t, root.URI, privateSubject["rootUri"]) + }) + } +} diff --git a/internal/db/postgres/moderation_repo.go b/internal/db/postgres/moderation_repo.go index b5a7a73..401c630 100644 --- a/internal/db/postgres/moderation_repo.go +++ b/internal/db/postgres/moderation_repo.go @@ -57,7 +57,7 @@ func (r *ModerationRepository) SubjectModeration(ctx context.Context, authorityD row := tx.QueryRowContext(ctx, ` SELECT COALESCE(s.version, 0), a.id, a.actor_did, a.authority_did, a.scope_kind, a.scope_community_did, - a.subject_uri, a.subject_collection, a.subject_community_did, a.observed_cid, + a.subject_uri, a.subject_collection, a.subject_community_did, a.subject_root_uri, a.observed_cid, a.action, a.label_value, a.reason, a.private_classification, a.private_note, a.reverses_action_id, a.origin, a.created_at FROM (SELECT $2::text AS uri) AS target @@ -168,12 +168,11 @@ func (t *moderationTransaction) LockSubject(ctx context.Context, subjectURI stri func (t *moderationTransaction) ReadIndexedPost(ctx context.Context, subjectURI string) (*moderation.IndexedPost, error) { var post moderation.IndexedPost - var authorDID string - var embed sql.NullString + var embed, title, content sql.NullString err := t.tx.QueryRowContext(ctx, ` - SELECT uri, cid, author_did, community_did, (deleted_at IS NOT NULL), embed + SELECT uri, cid, author_did, community_did, (deleted_at IS NOT NULL), embed, title, content FROM posts WHERE uri = $1 FOR SHARE - `, subjectURI).Scan(&post.URI, &post.CID, &authorDID, &post.CommunityDID, &post.AuthorDeleted, &embed) + `, subjectURI).Scan(&post.URI, &post.CID, &post.AuthorDID, &post.CommunityDID, &post.AuthorDeleted, &embed, &title, &content) if errors.Is(err, sql.ErrNoRows) { return nil, moderation.ErrSubjectNotIndexed } @@ -184,7 +183,8 @@ func (t *moderationTransaction) ReadIndexedPost(ctx context.Context, subjectURI if err != nil { return nil, err } - post.OwnerDID = authorDID + post.Title, post.Text = title.String, content.String + post.OwnerDID = post.AuthorDID if uri.Collection().String() == moderation.LegacyPostCollection { post.OwnerDID = post.CommunityDID } @@ -195,24 +195,41 @@ func (t *moderationTransaction) ReadIndexedPost(ctx context.Context, subjectURI if err := json.Unmarshal([]byte(embed.String), &decoded); err == nil { if object, isObject := decoded.(map[string]any); isObject { post.BlobCIDs = embeds.PostBlobCIDs(object) + post.LinkURL = externalEmbedURI(object) } } } return &post, nil } +// externalEmbedURI returns the link URI of an external link embed, or empty +// for any other or malformed embed or a uri that is not a valid URI. +func externalEmbedURI(embed map[string]any) string { + if embedType, _ := embed["$type"].(string); embedType != embeds.TypeExternal { + return "" + } + external, _ := embed["external"].(map[string]any) + uri, _ := external["uri"].(string) + // Keep only a value the lexicon's "uri" format accepts, so the snapshot + // view's linkUrl always validates. + if _, err := syntax.ParseURI(uri); err != nil { + return "" + } + return uri +} + func (t *moderationTransaction) ReadIndexedComment(ctx context.Context, subjectURI string) (*moderation.IndexedComment, error) { var comment moderation.IndexedComment - var communityDID, embed sql.NullString + var communityDID, embed, rootURI, content sql.NullString err := t.tx.QueryRowContext(ctx, ` SELECT c.uri, c.cid, c.commenter_did, (c.deleted_at IS NOT NULL AND c.deletion_reason = 'author'), - p.community_did, c.embed + p.community_did, c.embed, c.root_uri, c.content FROM comments c LEFT JOIN posts p ON p.uri = c.root_uri WHERE c.uri = $1 FOR SHARE OF c `, subjectURI).Scan(&comment.URI, &comment.CID, &comment.OwnerDID, - &comment.AuthorDeleted, &communityDID, &embed) + &comment.AuthorDeleted, &communityDID, &embed, &rootURI, &content) if errors.Is(err, sql.ErrNoRows) { return nil, moderation.ErrSubjectNotIndexed } @@ -220,6 +237,14 @@ func (t *moderationTransaction) ReadIndexedComment(ctx context.Context, subjectU return nil, err } comment.CommunityDID = communityDID.String + comment.Text = content.String + // The indexed root is stored unvalidated. Keep only a value the lexicon's + // "at-uri" format accepts, so the subject's rootUri always + // validates; an invalid root records no root and the comment stays + // removable. + if _, err := syntax.ParseATURI(rootURI.String); err == nil { + comment.RootURI = rootURI.String + } if embed.Valid { // The embed is author-controlled and stored unvalidated. A shape the // comment view cannot serve images from has no images to block, and @@ -236,7 +261,7 @@ func (t *moderationTransaction) ReadIndexedComment(ctx context.Context, subjectU const moderationActionColumns = ` a.id, a.actor_did, a.authority_did, a.scope_kind, a.scope_community_did, - a.subject_uri, a.subject_collection, a.subject_community_did, a.observed_cid, + a.subject_uri, a.subject_collection, a.subject_community_did, a.subject_root_uri, a.observed_cid, a.action, a.label_value, a.reason, a.private_classification, a.private_note, a.reverses_action_id, a.origin, a.created_at` @@ -246,13 +271,13 @@ type moderationRow interface { func scanModerationAction(row moderationRow, version *int64, reversedReason *sql.NullString) (*moderation.Action, error) { var id, actorDID, authorityDID, scopeKind, scopeCommunityDID sql.NullString - var subjectURI, subjectCollection, subjectCommunityDID, observedCID sql.NullString + var subjectURI, subjectCollection, subjectCommunityDID, subjectRootURI, observedCID sql.NullString var actionKind, labelValue, reason, privateClassification, privateNote sql.NullString var reversesActionID, origin sql.NullString var createdAt sql.NullTime dest := []any{ &id, &actorDID, &authorityDID, &scopeKind, &scopeCommunityDID, - &subjectURI, &subjectCollection, &subjectCommunityDID, &observedCID, + &subjectURI, &subjectCollection, &subjectCommunityDID, &subjectRootURI, &observedCID, &actionKind, &labelValue, &reason, &privateClassification, &privateNote, &reversesActionID, &origin, &createdAt, } @@ -272,8 +297,9 @@ func scanModerationAction(row moderationRow, version *int64, reversedReason *sql ID: id.String, ActorDID: actorDID.String, AuthorityDID: authorityDID.String, ScopeKind: scopeKind.String, ScopeCommunityDID: scopeCommunityDID.String, SubjectURI: subjectURI.String, SubjectCollection: subjectCollection.String, - SubjectCommunityDID: subjectCommunityDID.String, ObservedCID: observedCID.String, - Action: actionKind.String, LabelValue: labelValue.String, Reason: reason.String, + SubjectCommunityDID: subjectCommunityDID.String, SubjectRootURI: subjectRootURI.String, + ObservedCID: observedCID.String, Action: actionKind.String, + LabelValue: labelValue.String, Reason: reason.String, PrivateNote: privateNote.String, ReversesActionID: reversesActionID.String, Origin: origin.String, CreatedAt: createdAt.Time, } @@ -365,19 +391,44 @@ func (t *moderationTransaction) InsertAction(ctx context.Context, action moderat INSERT INTO moderation_actions (id, actor_did, authority_did, scope_kind, scope_community_did, subject_uri, subject_collection, subject_community_did, observed_cid, - action, label_value, reason, private_note, reverses_action_id, origin, created_at) + action, label_value, reason, private_note, reverses_action_id, origin, created_at, + subject_root_uri) VALUES ($1, $2, $3, $4, NULLIF($5, ''), $6, $7, NULLIF($8, ''), NULLIF($9, ''), - $10, NULLIF($11, ''), NULLIF($12, ''), NULLIF($13, ''), NULLIF($14, ''), $15, $16) + $10, NULLIF($11, ''), NULLIF($12, ''), NULLIF($13, ''), NULLIF($14, ''), $15, $16, + NULLIF($17, '')) `, action.ID, action.ActorDID, action.AuthorityDID, action.ScopeKind, action.ScopeCommunityDID, action.SubjectURI, action.SubjectCollection, action.SubjectCommunityDID, action.ObservedCID, action.Action, action.LabelValue, action.Reason, action.PrivateNote, action.ReversesActionID, - action.Origin, action.CreatedAt) + action.Origin, action.CreatedAt, action.SubjectRootURI) if err != nil { return nil, err } return &action, nil } +func (t *moderationTransaction) InsertContentSnapshot(ctx context.Context, actionID, authorDID string, snapshot moderation.ContentSnapshot) error { + _, err := t.tx.ExecContext(ctx, ` + INSERT INTO moderation_content_snapshots (action_id, author_did, title, text, link_url, captured_cid) + VALUES ($1, $2, NULLIF($3, ''), NULLIF($4, ''), NULLIF($5, ''), $6) + `, actionID, authorDID, snapshot.Title, snapshot.Text, snapshot.LinkURL, snapshot.CapturedCID) + if err != nil { + return fmt.Errorf("insert content snapshot: %w", err) + } + return nil +} + +func (t *moderationTransaction) DeleteSubjectContentSnapshots(ctx context.Context, subjectURI string) error { + _, err := t.tx.ExecContext(ctx, ` + DELETE FROM moderation_content_snapshots s + USING moderation_actions a + WHERE a.id = s.action_id AND a.subject_uri = $1 + `, subjectURI) + if err != nil { + return fmt.Errorf("delete subject content snapshots: %w", err) + } + return nil +} + func (t *moderationTransaction) SetRemovalDecision(ctx context.Context, authorityDID, subjectURI, actionID string, active bool) error { if !active { result, err := t.tx.ExecContext(ctx, ` @@ -677,7 +728,12 @@ func (r *ModerationRepository) BindTransaction(tx *sql.Tx) moderation.MediaTrans // Each row's SubjectAccess is decided now, not when the action was recorded: a // subject whose own post or comment row is still indexed is public, whatever // its community admission status and even when the author soft-deleted it, and -// one whose row is gone (account erasure hard-deletes it) is restricted. ExcludeRestricted filters in the WHERE +// one whose row is gone (account erasure hard-deletes it) is restricted. +// SubjectAuthorDeleted is decided now too: the author has soft-deleted the +// subject's post, or its comment row was deleted for reason 'author', or the +// author once deleted the removed comment (snapshot.author_deleted_at), which a +// re-created comment does not undo. SubjectRootWithheld is set when a comment's +// root post has a hidden-reason action or is no longer indexed. ExcludeRestricted filters in the WHERE // clause, before LIMIT, so a filtered page is never short. func (r *ModerationRepository) ListActions(ctx context.Context, query moderation.ActionListQuery) ([]moderation.Action, error) { var beforeCreatedAt, beforeID any @@ -685,14 +741,26 @@ func (r *ModerationRepository) ListActions(ctx context.Context, query moderation beforeCreatedAt, beforeID = query.Before.CreatedAt, query.Before.ID } rows, err := r.db.QueryContext(ctx, ` - SELECT `+moderationActionColumns+`, reversed.reason, subject_access.access + SELECT `+moderationActionColumns+`, reversed.reason, subject_access.access, + subject_access.author_deleted OR snapshot.author_deleted_at IS NOT NULL, subject_access.root_withheld, + snapshot.title, snapshot.text, snapshot.link_url, snapshot.captured_cid FROM moderation_actions a LEFT JOIN moderation_actions reversed ON reversed.id = a.reverses_action_id + LEFT JOIN moderation_content_snapshots snapshot ON snapshot.action_id = a.id CROSS JOIN LATERAL ( SELECT CASE WHEN EXISTS (SELECT 1 FROM comments c WHERE a.subject_collection = $18 AND c.uri = a.subject_uri) OR EXISTS (SELECT 1 FROM posts p WHERE a.subject_collection <> $18 AND p.uri = a.subject_uri) - THEN $20::text ELSE $21::text END AS access + THEN $20::text ELSE $21::text END AS access, + EXISTS (SELECT 1 FROM comments c WHERE a.subject_collection = $18 AND c.uri = a.subject_uri + AND c.deleted_at IS NOT NULL AND c.deletion_reason = 'author') + OR EXISTS (SELECT 1 FROM posts p WHERE a.subject_collection <> $18 AND p.uri = a.subject_uri + AND p.deleted_at IS NOT NULL) AS author_deleted, + NULLIF(a.subject_root_uri, '') IS NOT NULL AND ( + NOT EXISTS (SELECT 1 FROM posts p WHERE p.uri = a.subject_root_uri) + OR EXISTS (SELECT 1 FROM moderation_actions root_action + WHERE root_action.subject_uri = a.subject_root_uri AND root_action.reason = ANY($15::text[])) + ) AS root_withheld ) subject_access WHERE ($2::timestamptz IS NULL OR (a.created_at, a.id) < ($2::timestamptz, $3::text)) AND (NULLIF($4::text, '') IS NULL OR a.subject_uri = $4) @@ -724,9 +792,12 @@ func (r *ModerationRepository) ListActions(ctx context.Context, query moderation defer rows.Close() var actions []moderation.Action for rows.Next() { - var reversedReason sql.NullString + var reversedReason, snapshotTitle, snapshotText, snapshotLinkURL, snapshotCID sql.NullString var access string - action, err := scanModerationAction(trailingColumns{row: rows, extra: []any{&access}}, nil, &reversedReason) + var authorDeleted, rootWithheld bool + action, err := scanModerationAction(trailingColumns{row: rows, extra: []any{ + &access, &authorDeleted, &rootWithheld, &snapshotTitle, &snapshotText, &snapshotLinkURL, &snapshotCID, + }}, nil, &reversedReason) if err != nil { return nil, fmt.Errorf("scan moderation action: %w", err) } @@ -734,6 +805,14 @@ func (r *ModerationRepository) ListActions(ctx context.Context, query moderation return nil, errors.New("scan moderation action: row has no action id") } action.SubjectAccess = moderation.SubjectAccess(access) + action.SubjectAuthorDeleted = authorDeleted + action.SubjectRootWithheld = rootWithheld + if snapshotCID.Valid { + action.ContentSnapshot = &moderation.ContentSnapshot{ + Title: snapshotTitle.String, Text: snapshotText.String, + LinkURL: snapshotLinkURL.String, CapturedCID: snapshotCID.String, + } + } actions = append(actions, *action) } if err := rows.Err(); err != nil { diff --git a/internal/db/postgres/notification_migration_test.go b/internal/db/postgres/notification_migration_test.go index 22177be..7aadfcd 100644 --- a/internal/db/postgres/notification_migration_test.go +++ b/internal/db/postgres/notification_migration_test.go @@ -56,6 +56,8 @@ func TestNotificationMigration053_ActivationSingleton(t *testing.T) { func TestNotificationMigration053_DownAndUpRoundTrip(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), @@ -81,6 +83,8 @@ func TestNotificationMigration053_DownAndUpRoundTrip(t *testing.T) { func TestNotificationMigration055_DownAndUpRoundTrip(t *testing.T) { t.Parallel() db := testkit.DB(t) + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "this must exercise 055's Down section, not an earlier migration") var relation sql.NullString diff --git a/internal/db/postgres/read_path_indexes_migration_test.go b/internal/db/postgres/read_path_indexes_migration_test.go index 097c612..26695a8 100644 --- a/internal/db/postgres/read_path_indexes_migration_test.go +++ b/internal/db/postgres/read_path_indexes_migration_test.go @@ -37,6 +37,9 @@ func TestMigration049ReadPathIndexes(t *testing.T) { } assertReadPathIndexDefinitions(t, db, "after 049 Up") + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") + require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/rematerialize_ledger_schema_test.go b/internal/db/postgres/rematerialize_ledger_schema_test.go index c81c653..a9f1fe7 100644 --- a/internal/db/postgres/rematerialize_ledger_schema_test.go +++ b/internal/db/postgres/rematerialize_ledger_schema_test.go @@ -196,6 +196,8 @@ func TestRematerializeLedgerMigration_RollsBack(t *testing.T) { // 037 and come off first, one asserted step at a time. Asserting which // migration rolled back is what keeps this pointed at 037's Down rather than // drifting onto a newer one later. + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/user_repo.go b/internal/db/postgres/user_repo.go index 8a60e5a..5fc951c 100644 --- a/internal/db/postgres/user_repo.go +++ b/internal/db/postgres/user_repo.go @@ -408,7 +408,16 @@ func (r *postgresUserRepo) Delete(ctx context.Context, did string) error { return fmt.Errorf("failed to delete posts for did=%s: %w", did, err) } - // 11. Delete notifications for this recipient or actor after content deletes. + // 11. Delete the modlog content snapshots of this author's removed posts + // and comments; their moderation actions stay. This must follow the + // comments and posts deletes: an in-flight removal holds a share lock on + // the post or comment row until it commits its snapshot, so those deletes + // wait for it, and this statement then sees the committed snapshot. + if _, err := tx.ExecContext(ctx, `DELETE FROM moderation_content_snapshots WHERE author_did = $1`, did); err != nil { + return fmt.Errorf("failed to delete moderation content snapshots for did=%s: %w", did, err) + } + + // 12. Delete notifications for this recipient or actor after content deletes. // A notification consumer locks the recipient's content row (count/reply_count // UPDATE) before upserting the notification. Deleting notifications first // could hold the notification row it needs while waiting for the content row @@ -425,12 +434,12 @@ func (r *postgresUserRepo) Delete(ctx context.Context, did string) error { return fmt.Errorf("failed to delete notification public post withdrawals for did=%s: %w", did, err) } - // 12. Delete notification state + // 13. Delete notification state if _, err := tx.ExecContext(ctx, `DELETE FROM notification_state WHERE did = $1`, did); err != nil { return fmt.Errorf("failed to delete notification_state for did=%s: %w", did, err) } - // 13. Delete user + // 14. Delete user result, err := tx.ExecContext(ctx, `DELETE FROM users WHERE did = $1`, did) if err != nil { return fmt.Errorf("failed to delete user did=%s: %w", did, err) diff --git a/internal/db/postgres/vote_drift_recount_migration_test.go b/internal/db/postgres/vote_drift_recount_migration_test.go index 2243fa1..a859d36 100644 --- a/internal/db/postgres/vote_drift_recount_migration_test.go +++ b/internal/db/postgres/vote_drift_recount_migration_test.go @@ -80,6 +80,8 @@ func TestMigration040_RecountsVoteDriftAndSweepsLegacyOrphans(t *testing.T) { // point of a repair migration and cannot be observed by seeding after it has // run. Asserting the version that came off is the tripwire that keeps this // pointed at 040 when later migrations land. + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/internal/db/postgres/votes_upvote_history_index_migration_test.go b/internal/db/postgres/votes_upvote_history_index_migration_test.go index da9b0bd..08c76ca 100644 --- a/internal/db/postgres/votes_upvote_history_index_migration_test.go +++ b/internal/db/postgres/votes_upvote_history_index_migration_test.go @@ -17,6 +17,8 @@ func TestVotesUpvoteHistoryIndexMigration054_DownAndUpRoundTrip(t *testing.T) { db := testkit.DB(t) assertVotesUpvoteHistoryIndex(t, db, "after 054 Up") + require.EqualValues(t, 56, testkit.MigrateDownOne(t, db, 56), + "056 (moderation content snapshots) must be rolled back before testing earlier migrations") require.EqualValues(t, 55, testkit.MigrateDownOne(t, db, 55), "055 (notification public post withdrawals) must be rolled back before testing earlier migrations") require.EqualValues(t, 54, testkit.MigrateDownOne(t, db, 54), diff --git a/tests/lexicon-test-data/moderation/action-view-invalid-root-uri-not-at-uri.json b/tests/lexicon-test-data/moderation/action-view-invalid-root-uri-not-at-uri.json new file mode 100644 index 0000000..4847944 --- /dev/null +++ b/tests/lexicon-test-data/moderation/action-view-invalid-root-uri-not-at-uri.json @@ -0,0 +1,20 @@ +{ + "$type": "social.coves.moderation.defs#actionView", + "ref": { + "serviceDid": "did:plc:abc123", + "actionId": "action-013" + }, + "action": "remove", + "authorityDid": "did:plc:abc123", + "scope": { + "kind": "instance" + }, + "createdAt": "2026-10-13T10:03:00Z", + "origin": "local", + "subject": { + "uri": "at://did:plc:ghi789/social.coves.community.comment/3k7a3dmb5bk2d", + "cid": "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi", + "rootUri": "https://coves.example/c/community/post/3k7a3dmb5bk2c" + }, + "reason": "social.coves.moderation.defs#reasonSpam" +} diff --git a/tests/lexicon-test-data/moderation/action-view-invalid-snapshot-missing-captured-cid.json b/tests/lexicon-test-data/moderation/action-view-invalid-snapshot-missing-captured-cid.json new file mode 100644 index 0000000..4b5cde0 --- /dev/null +++ b/tests/lexicon-test-data/moderation/action-view-invalid-snapshot-missing-captured-cid.json @@ -0,0 +1,22 @@ +{ + "$type": "social.coves.moderation.defs#actionView", + "ref": { + "serviceDid": "did:plc:abc123", + "actionId": "action-012" + }, + "action": "remove", + "authorityDid": "did:plc:abc123", + "scope": { + "kind": "instance" + }, + "createdAt": "2026-10-13T10:02:00Z", + "origin": "local", + "subject": { + "uri": "at://did:plc:ghi789/social.coves.community.comment/3k7a3dmb5bk2d", + "cid": "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + }, + "reason": "social.coves.moderation.defs#reasonSpam", + "contentSnapshot": { + "text": "Buy cheap watches at the link in my profile" + } +} diff --git a/tests/lexicon-test-data/moderation/action-view-valid-content-snapshot.json b/tests/lexicon-test-data/moderation/action-view-valid-content-snapshot.json new file mode 100644 index 0000000..359664c --- /dev/null +++ b/tests/lexicon-test-data/moderation/action-view-valid-content-snapshot.json @@ -0,0 +1,25 @@ +{ + "$type": "social.coves.moderation.defs#actionView", + "ref": { + "serviceDid": "did:plc:abc123", + "actionId": "action-010" + }, + "action": "remove", + "authorityDid": "did:plc:abc123", + "scope": { + "kind": "instance" + }, + "createdAt": "2026-10-13T10:00:00Z", + "origin": "local", + "subject": { + "uri": "at://did:plc:ghi789/social.coves.community.comment/3k7a3dmb5bk2d", + "cid": "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi", + "communityDid": "did:plc:def456", + "rootUri": "at://did:plc:def456/social.coves.community.postv2/3k7a3dmb5bk2c" + }, + "reason": "social.coves.moderation.defs#reasonSpam", + "contentSnapshot": { + "text": "Buy cheap watches at the link in my profile", + "capturedCid": "bafyreihgdyzzpkkzq2izfnhcmm77ycuacvkuziwbnqxfxtqsz7tmxwhnshi" + } +} diff --git a/tests/lexicon-test-data/moderation/admin-action-view-valid-content-snapshot.json b/tests/lexicon-test-data/moderation/admin-action-view-valid-content-snapshot.json new file mode 100644 index 0000000..209edff --- /dev/null +++ b/tests/lexicon-test-data/moderation/admin-action-view-valid-content-snapshot.json @@ -0,0 +1,29 @@ +{ + "$type": "social.coves.moderation.defs#adminActionView", + "action": { + "ref": { + "serviceDid": "did:plc:abc123", + "actionId": "action-011" + }, + "action": "remove", + "authorityDid": "did:plc:abc123", + "scope": { + "kind": "instance" + }, + "createdAt": "2026-10-13T10:01:00Z", + "origin": "local", + "reason": "social.coves.moderation.defs#reasonDoxing" + }, + "actorDid": "did:plc:ghi789", + "privateSubject": { + "uri": "at://did:plc:def456/social.coves.community.postv2/3k7a3dmb5bk2c", + "cid": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm", + "communityDid": "did:plc:def456" + }, + "contentSnapshot": { + "title": "Found their home address", + "text": "Posting the address here so everyone can visit.", + "linkUrl": "https://maps.example/place/123", + "capturedCid": "bafyreib6tbnql2ux3whnfysbzabthaj2vvck53nimhbi5g5a7jgvgr5eqm" + } +} diff --git a/tests/lexicon_fixtures_test.go b/tests/lexicon_fixtures_test.go index dff7ce7..b2145b5 100644 --- a/tests/lexicon_fixtures_test.go +++ b/tests/lexicon_fixtures_test.go @@ -75,6 +75,9 @@ var invalidFixtureExpectedErrors = map[string]string{ "post-viewer-state-invalid-moderation-item-not-string.json": "expected a string", "comment-viewer-state-invalid-moderation-item-not-string.json": "expected a string", "moderated-post-invalid-viewer-not-object.json": "expected an object", + // Modlog content snapshots and comment thread roots. + "action-view-invalid-snapshot-missing-captured-cid.json": "required field missing: capturedCid", + "action-view-invalid-root-uri-not-at-uri.json": "AT-URI syntax", } // expectedFixtureFamilies is the closed list of top-level fixture directories diff --git a/tests/lexicon_moderation_test.go b/tests/lexicon_moderation_test.go index 3d34c52..f352287 100644 --- a/tests/lexicon_moderation_test.go +++ b/tests/lexicon_moderation_test.go @@ -78,6 +78,7 @@ func moderationObjectExpectations() map[string]moderationDefinitionExpectation { "uri": formattedString("at-uri"), "cid": formattedString("cid"), "communityDid": formattedString("did"), + "rootUri": formattedString("at-uri"), }, }, "actorRef": { @@ -89,27 +90,38 @@ func moderationObjectExpectations() map[string]moderationDefinitionExpectation { "actionView": { required: []string{"ref", "action", "authorityDid", "scope", "createdAt", "origin"}, properties: map[string]moderationPropertyExpectation{ - "ref": reference("#actionRef"), - "action": stringProperty(64, "remove", "restore", "apply-removal", "retract-removal", "label", "retract-label"), - "authorityDid": formattedString("did"), - "scope": reference("#scopeView"), - "createdAt": formattedString("datetime"), - "origin": stringProperty(64, "local", "inherited"), - "subject": reference("#subjectRef"), - "reason": reference("#reasonType"), - "labelValue": {schemaType: "string", minLength: 1, maxLength: 128, knownValues: []string{"nsfw"}}, - "actor": reference("#actorRef"), - "reverses": reference("#actionRef"), - "issuedAt": formattedString("datetime"), + "ref": reference("#actionRef"), + "action": stringProperty(64, "remove", "restore", "apply-removal", "retract-removal", "label", "retract-label"), + "authorityDid": formattedString("did"), + "scope": reference("#scopeView"), + "createdAt": formattedString("datetime"), + "origin": stringProperty(64, "local", "inherited"), + "subject": reference("#subjectRef"), + "reason": reference("#reasonType"), + "labelValue": {schemaType: "string", minLength: 1, maxLength: 128, knownValues: []string{"nsfw"}}, + "actor": reference("#actorRef"), + "reverses": reference("#actionRef"), + "issuedAt": formattedString("datetime"), + "contentSnapshot": reference("#contentSnapshot"), }, }, "adminActionView": { required: []string{"action"}, properties: map[string]moderationPropertyExpectation{ - "action": reference("#actionView"), - "actorDid": formattedString("did"), - "privateSubject": reference("#subjectRef"), - "privateNote": {schemaType: "string", maxLength: 10000, maxGraphemes: 1000}, + "action": reference("#actionView"), + "actorDid": formattedString("did"), + "privateSubject": reference("#subjectRef"), + "privateNote": {schemaType: "string", maxLength: 10000, maxGraphemes: 1000}, + "contentSnapshot": reference("#contentSnapshot"), + }, + }, + "contentSnapshot": { + required: []string{"capturedCid"}, + properties: map[string]moderationPropertyExpectation{ + "title": {schemaType: "string"}, + "text": {schemaType: "string"}, + "linkUrl": formattedString("uri"), + "capturedCid": formattedString("cid"), }, }, "sourceView": { -- 2.51.2