diff --git a/cmd/rematerialize-posts/main.go b/cmd/rematerialize-posts/main.go index 8bac684..8fa2b15 100644 --- a/cmd/rematerialize-posts/main.go +++ b/cmd/rematerialize-posts/main.go @@ -5,8 +5,32 @@ // the AUTHOR's repo, plus the community's acceptance that pins it, and then — // and only then — deletes the old record. // -// It is a THIN WRAPPER. All of the safety logic lives in posts.Rematerializer; -// this file only wires the production seams the state machine drives: +// # THIS COMMAND DELETES PRODUCTION USER DATA IRREVERSIBLY +// +// It is run by hand, once, during a maintenance window, by someone who has been +// awake too long. Everything in this file that is not wiring exists because of +// that sentence: +// +// - it prints WHAT IT IS ABOUT TO TOUCH — database host, PDS, instance DID, +// community count, record count — and refuses to write anything until the +// operator passes -yes; +// - -dry-run walks the identical code path with only the mutations replaced, +// so the rehearsal really resolves credentials, really re-reads records, +// really fetches blobs, and really reports what it would delete; +// - it holds a Postgres ADVISORY LOCK, so a second invocation refuses to start +// rather than racing the first through guarded transitions and reporting +// "the ledger and the tool have diverged", which reads like corruption; +// - it logs one line per record and a rolling n/N, so a slow run is +// distinguishable from a hung one; +// - it ALWAYS prints the census, including on the error path, because "did it +// delete 0 records or 4,131?" is the only question that matters after a +// failure; +// - SIGINT and SIGTERM cancel the run between records rather than killing it +// mid-transition, and every outbound call is bounded by a timeout. +// +// It is a THIN WRAPPER otherwise. All of the safety logic lives in +// posts.Rematerializer; this file wires the production seams the state machine +// drives: // // - the ledger (migration 037), so the run is resumable and idempotent; // - the author-repo factory, so each postv2 is signed by its own author (an @@ -14,13 +38,11 @@ // admin signature); // - the DIRECT community acceptance writer, so a since-banned author's live // post is preserved rather than re-adjudicated; +// - the community-repo factory, so the acceptance is READ BACK from the +// community's own repo before anything is deleted; // - a LegacySource over the real community repos: listRecords to discover the -// deprecated posts, deleteRecord to remove them once verified. -// -// It is run by hand during the deploy window (§11 step 4), against a database -// whose migrations are already applied, and it reports a census that refuses to -// declare the migration complete while any post was left as legacy — the gate on -// the separate, irreversible legacy-removal follow-up. +// deprecated posts, getRecord for the pre-delete re-read, and a +// swap-guarded deleteRecord to remove them once verified. package main import ( @@ -32,6 +54,9 @@ import ( "fmt" "log" "os" + "os/signal" + "strings" + "syscall" "time" "Coves/internal/atproto/oauth" @@ -56,9 +81,37 @@ const legacyPostCollection = posts.LegacyPostCollection // catalogue is enumerated in bounded queries rather than one unbounded read. const listPageSize = 100 +// rematerializeAdvisoryLock is the Postgres advisory-lock key this tool holds +// for the whole run. +// +// TWO CONCURRENT RUNS ARE NOT MERELY WASTEFUL — they interleave on the ledger's +// guarded transitions, so the loser of each race is told "no row in the expected +// prior state (the ledger and the tool have diverged)". That message is correct +// and terrifying, and an operator reading it at 3am has no way to tell it from +// real corruption. Refusing the second run is the only version of this that +// stays legible. +const rematerializeAdvisoryLock int64 = 0x52454d4154 // "REMAT" + +// perRecordTimeout bounds everything one record's processing does — several PDS +// round trips and a handful of ledger writes. Without it a single half-open +// socket stalls the entire migration with no output. +const perRecordTimeout = 5 * time.Minute + func main() { communityFilter := flag.String("community", "", "restrict the run to a single community DID (a staged rollout); empty means every hosted community") + dryRun := flag.Bool("dry-run", false, + "rehearse: walk the identical code path — resolving credentials, re-reading records, fetching blobs — but write and delete nothing") + confirm := flag.Bool("yes", false, + "required for a real run: confirm the target printed in the banner before anything is written or deleted") + acceptFallbacks := flag.Bool("accept-fallbacks", false, + "proceed even if the credential census leaves posts as legacy (default: stop before mutating anything and report them)") + reopenFallbacks := flag.Bool("reopen-fallbacks", false, + "move rows previously left as legacy back to 'discovered' so this run retries them, then exit; use after re-authorizing the affected authors") + recordDelay := flag.Duration("delay", 0, + "pause between records, to rate-limit the PDS during the run (e.g. 100ms)") + runTimeout := flag.Duration("timeout", 6*time.Hour, + "hard deadline for the whole run; the run cancels cleanly between records when it expires") flag.Parse() cfg, err := config.Load() @@ -72,7 +125,33 @@ func main() { } defer func() { _ = db.Close() }() - ctx := context.Background() + // SIGINT/SIGTERM cancel between records rather than killing the process + // mid-transition; the ledger then holds a coherent checkpoint to resume from. + ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) + defer stop() + ctx, cancelDeadline := context.WithTimeout(ctx, *runTimeout) + defer cancelDeadline() + + // The advisory lock is taken on a DEDICATED connection held for the whole run: + // a session-scoped lock released the instant the pool recycles its connection + // is not a lock. + unlock, err := takeAdvisoryLock(ctx, db) + if err != nil { + log.Fatalf("rematerialize-posts: %v", err) + } + defer unlock() + + ledger := postgresRepo.NewRematerializeLedger(db) + + if *reopenFallbacks { + moved, err := ledger.ReopenFallback(ctx, *communityFilter) + if err != nil { + log.Fatalf("rematerialize-posts: reopening fallback rows: %v", err) + } + log.Printf("rematerialize-posts: moved %d row(s) out of %s back to %s; re-run the tool to retry them", + moved, posts.RematerializeFallbackLeftLegacy, posts.RematerializeDiscovered) + return + } // The OAuth client is the credential seam for BOTH kinds of author: a human's // browser session is not available in a batch tool, so the only authors this @@ -98,58 +177,132 @@ func main() { // The DIRECT acceptance writer over the production community-repo factory — // the same credential-presence hosting test the acceptance engine uses. The - // tool holds this writer and no decider, so it cannot re-run admission. + // tool holds this writer and no decider, so it cannot re-run admission. The + // same factory is handed to the Rematerializer for the acceptance READ-BACK. repoFactory := posts.NewCommunityRepoFactory(communityService) writer := posts.NewCommunityRecordWriter(repoFactory, time.Now) authorFactory := posts.NewAuthorRepoFactory(oauthClient.ClientApp, aggregators.DefaultSessionID) source := &realLegacySource{ - communities: communityService, - creds: communityService, communityFilter: *communityFilter, + hostedDIDs: postgresRepo.NewHostedCommunityQuery(db).HostedCommunityDIDs, + openRepo: communityRepoOpener(communityService), } + progress := newProgressLogger() tool := &posts.Rematerializer{ - Source: source, - Ledger: postgresRepo.NewRematerializeLedger(db), - AuthorRepos: authorFactory, - Acceptances: writer, + Source: source, + Ledger: ledger, + AuthorRepos: authorFactory, + Acceptances: writer, + CommunityRepos: repoFactory, + CommunityScope: *communityFilter, + Progress: progress.log, + PerRecordTimeout: perRecordTimeout, + AbortOnFallback: !*acceptFallbacks, } - report, err := tool.Run(ctx) + // The banner is printed BEFORE the confirmation and from real queries, so the + // operator confirms the target they were shown rather than the one they + // assumed. Enumerating the source here also means a misconfigured run — wrong + // database, wrong PDS, zero hosted communities — is visible before it writes. + scope, err := describeTarget(ctx, cfg, source) if err != nil { - log.Fatalf("rematerialize-posts: the run failed: %v", err) + log.Fatalf("rematerialize-posts: describing the target: %v", err) + } + printBanner(cfg, scope, *dryRun, *communityFilter) + + if *dryRun { + tool = posts.DryRunOf(tool) + } else if !*confirm { + log.Printf("rematerialize-posts: REFUSING TO RUN. This deletes %d legacy record(s) from %d community repo(s) IRREVERSIBLY.", + scope.records, scope.communities) + log.Printf("rematerialize-posts: rehearse it with -dry-run, or confirm the target above with -yes.") + os.Exit(2) + } + + if *recordDelay > 0 { + source.delay = *recordDelay } - logCensus(report) + report, runErr := tool.Run(ctx) - // A surviving fallback means at least one post still lives only as a legacy - // record, so the operator must NOT proceed to the legacy-removal step. Exiting - // non-zero makes that a machine-checkable gate rather than a line of output an - // operator might skim past. - if !report.Complete { - log.Printf("rematerialize-posts: INCOMPLETE — %d post(s) left as legacy; do not run the legacy-removal step", report.Fallbacks) + // THE CENSUS IS ALWAYS PRINTED, including on the error path. A run that fails + // on record 900 of 4,131 has already deleted 899 records, and an operator who + // sees only the error has no way to know that. + logCensus(report, *dryRun) + if deletes, isDry := posts.DryRunDeletes(tool); isDry { + log.Printf("rematerialize-posts: DRY RUN — %d record(s) would have been deleted; nothing was written or removed", deletes) + } + + if runErr != nil { + log.Printf("rematerialize-posts: THE RUN FAILED: %v", runErr) + log.Printf("rematerialize-posts: the ledger above is the truth about what completed; re-running is safe and resumes from it") os.Exit(1) } - log.Printf("rematerialize-posts: complete — every discovered post was re-materialized") + + // TWO SIGNALS, REPORTED SEPARATELY. A staged -community run finishing its own + // scope is a success even though the migration as a whole is not done, and + // collapsing the two taught the operator to ignore a red exit code on every + // staged run — which would leave §11 step 6 with no machine-checkable gate at + // all. + if !report.ScopeComplete { + log.Printf("rematerialize-posts: SCOPE INCOMPLETE — %d of %d row(s) in scope reached done, %d fallback(s), %d legacy record(s) still standing", + report.Done, report.Discovered, report.Fallbacks, report.RemainingLegacy) + os.Exit(1) + } + log.Printf("rematerialize-posts: scope complete — every post in %s was re-materialized", scopeName(*communityFilter)) + + if !report.Complete { + log.Printf("rematerialize-posts: THE MIGRATION AS A WHOLE IS NOT COMPLETE — %d of %d ledger row(s) done, %d fallback(s), %d legacy record(s) still standing.", + report.GlobalDone, report.GlobalDiscovered, report.GlobalFallbacks, report.RemainingLegacy) + log.Printf("rematerialize-posts: DO NOT run the legacy-removal follow-up (PRD §11 step 6) until this line says complete.") + return + } + log.Printf("rematerialize-posts: MIGRATION COMPLETE — every discovered post was re-materialized and no legacy record remains") +} + +// repoClient is the narrow PDS surface the legacy source needs: enumerate, +// re-read, and delete UNDER A GUARD. +// +// It is declared here rather than taken as pds.Client so that the guarded delete +// is a REQUIREMENT of the type. A transport that cannot express swapRecord fails +// to satisfy this interface at compile time instead of quietly deleting whatever +// stands. +type repoClient interface { + ListRecords(ctx context.Context, collection string, limit int, cursor string) (*pds.ListRecordsResponse, error) + GetRecord(ctx context.Context, collection, rkey string) (*pds.RecordResponse, error) + DeleteRecordWithSwap(ctx context.Context, collection, rkey, swapRecord string) error } // realLegacySource enumerates the deprecated community.post records across the -// hosted communities and deletes them from their community repos. +// hosted communities, re-reads one on demand, and deletes them from their +// community repos under a swap guard. // -// It reaches the PDS through a full pds.Client rather than the narrowed -// CommunityRepo the acceptance writer uses, because discovery needs listRecords -// and deletion needs deleteRecord — neither of which the write-narrowed surface -// carries. The credentials come from the same source the acceptance writer's -// factory uses, so the two never disagree about which communities are hosted. +// It reaches the PDS through a full client rather than the narrowed CommunityRepo +// the acceptance writer uses, because discovery needs listRecords and deletion +// needs deleteRecord — neither of which the write-narrowed surface carries. type realLegacySource struct { - communities communities.Service - creds posts.CommunityCredentialSource + // communityFilter scopes the run. It gates DISCOVERY and, independently, the + // DELETE: the ledger reconcile pass reaches records discovery never listed, so + // a filter applied only at discovery would let a staged run for community A + // delete community B's posts. communityFilter string + + // hostedDIDs answers which communities this AppView can sign for — credential + // presence, never a claimed profile field. + hostedDIDs func(ctx context.Context) ([]string, error) + + // openRepo opens one community's repo over freshly-renewed stored credentials. + openRepo func(ctx context.Context, did string) (repoClient, error) + + // delay paces the run, so a migration over thousands of records does not + // saturate the PDS the rest of the instance is still using. + delay time.Duration } -// ListLegacyPosts walks every hosted community and lists its remaining +// ListLegacyPosts walks every hosted community in scope and lists its remaining // social.coves.community.post records as LegacyPosts. func (s *realLegacySource) ListLegacyPosts(ctx context.Context) ([]posts.LegacyPost, error) { dids, err := s.hostedCommunityDIDs(ctx) @@ -159,13 +312,16 @@ func (s *realLegacySource) ListLegacyPosts(ctx context.Context) ([]posts.LegacyP var legacy []posts.LegacyPost for _, did := range dids { - client, err := s.communityClient(ctx, did) + client, err := s.openRepo(ctx, did) if err != nil { return nil, fmt.Errorf("opening the repo of %s to list legacy posts: %w", did, err) } cursor := "" for { + if err := ctx.Err(); err != nil { + return nil, err + } page, err := client.ListRecords(ctx, legacyPostCollection, listPageSize, cursor) if err != nil { return nil, fmt.Errorf("listing %s in %s: %w", legacyPostCollection, did, err) @@ -186,76 +342,160 @@ func (s *realLegacySource) ListLegacyPosts(ctx context.Context) ([]posts.LegacyP return legacy, nil } -// DeleteLegacyPost removes the old community.post from its community repo. A -// delete of an already-gone record is success — it is the step a crash after the -// migrated checkpoint retries, so idempotence is the contract. -func (s *realLegacySource) DeleteLegacyPost(ctx context.Context, legacy posts.LegacyPost) error { - client, err := s.communityClient(ctx, legacy.CommunityDID) +// ReadLegacyPost re-reads one record as it stands right now — the read the +// pre-delete CID check is made against. +func (s *realLegacySource) ReadLegacyPost(ctx context.Context, uri string) (posts.LegacyPost, bool, error) { + communityDID, rkey, err := splitLegacyURI(uri) + if err != nil { + return posts.LegacyPost{}, false, err + } + if err := s.inScope(communityDID, uri); err != nil { + return posts.LegacyPost{}, false, err + } + + client, err := s.openRepo(ctx, communityDID) + if err != nil { + return posts.LegacyPost{}, false, fmt.Errorf("opening the repo of %s to re-read %s: %w", communityDID, uri, err) + } + record, err := client.GetRecord(ctx, legacyPostCollection, rkey) + if err != nil { + if errors.Is(err, pds.ErrNotFound) { + return posts.LegacyPost{}, false, nil + } + return posts.LegacyPost{}, false, fmt.Errorf("re-reading %s: %w", uri, err) + } + post, err := legacyPostFromEntry(communityDID, pds.RecordEntry{URI: record.URI, CID: record.CID, Value: record.Value}) + if err != nil { + return posts.LegacyPost{}, false, err + } + return post, true, nil +} + +// DeleteLegacyPost removes the old community.post from its community repo, +// GUARDED by swapCID. +// +// The guard is what makes this safe rather than merely careful: the tool checks +// the record's CID before deleting, but the check and the delete are two +// moments, and only the PDS can evaluate them as one. A delete of an +// already-gone record is success — it is the step a crash after the migrated +// checkpoint retries, so idempotence is the contract — but a LOST SWAP is not: +// it means the record changed under us, which is exactly what the guard exists +// to catch. +func (s *realLegacySource) DeleteLegacyPost(ctx context.Context, legacy posts.LegacyPost, swapCID string) error { + if swapCID == "" { + return fmt.Errorf( + "refusing to delete %s: no source CID to guard the delete with. An unguarded delete removes whatever stands, including an edit "+ + "that landed after the postv2 was built", legacy.URI) + } + if err := s.inScope(legacy.CommunityDID, legacy.URI); err != nil { + return err + } + + client, err := s.openRepo(ctx, legacy.CommunityDID) if err != nil { return fmt.Errorf("opening the repo of %s to delete %s: %w", legacy.CommunityDID, legacy.URI, err) } - rkey := legacy.URI[lastSlash(legacy.URI)+1:] - if err := client.DeleteRecord(ctx, legacyPostCollection, rkey); err != nil { + _, rkey, err := splitLegacyURI(legacy.URI) + if err != nil { + return err + } + if err := client.DeleteRecordWithSwap(ctx, legacyPostCollection, rkey, swapCID); err != nil { if errors.Is(err, pds.ErrNotFound) { return nil } + if errors.Is(err, pds.ErrSwapConflict) { + return fmt.Errorf( + "refusing to delete %s: the record changed since the postv2 was built from CID %s, so the PDS rejected the guarded delete. "+ + "Something is still writing to this repo; stop the writer and re-run: %w", legacy.URI, swapCID, err) + } return fmt.Errorf("deleting %s: %w", legacy.URI, err) } + if s.delay > 0 { + // Pacing the destructive step is the one place a deliberate pause belongs: + // it keeps a multi-thousand-record drain from saturating the PDS the rest of + // the instance is still serving from. + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(s.delay): + } + } + return nil +} + +// inScope refuses any operation on a community outside a staged run's filter. +func (s *realLegacySource) inScope(communityDID, uri string) error { + if s.communityFilter != "" && communityDID != s.communityFilter { + return fmt.Errorf( + "refusing to touch %s: it belongs to %s but this run is scoped to %s", + uri, communityDID, s.communityFilter) + } return nil } // hostedCommunityDIDs returns the DIDs of the communities this AppView can sign // for — the only ones whose posts it can re-materialize and whose old records it -// can delete. A --community filter narrows the run to one for a staged rollout. +// can delete. A -community filter narrows the run to one for a staged rollout. +// +// HOSTING IS CREDENTIAL PRESENCE, asked for directly. The obvious version of +// this — walk the community listing and test the refresh-token field — is +// silently empty, because the listing does not select the credential columns: +// every community is filtered out and the run migrates nothing while reporting +// success. See postgres.HostedCommunitySource. func (s *realLegacySource) hostedCommunityDIDs(ctx context.Context) ([]string, error) { - if s.communityFilter != "" { - return []string{s.communityFilter}, nil + hosted, err := s.hostedDIDs(ctx) + if err != nil { + return nil, fmt.Errorf("listing the communities this AppView can sign for: %w", err) } - var dids []string - offset := 0 - for { - page, err := s.communities.ListCommunities(ctx, communities.ListCommunitiesRequest{ - Limit: listPageSize, - Offset: offset, - }) - if err != nil { - return nil, fmt.Errorf("listing communities: %w", err) - } - for _, community := range page { - // Hosting is credential presence, never a claimed profile field: only a - // community whose refresh token this AppView holds can be written to, and - // the repo factory would refuse the rest with ErrCommunityNotHosted. - if community.PDSRefreshToken != "" { - dids = append(dids, community.DID) + if s.communityFilter != "" { + for _, did := range hosted { + if did == s.communityFilter { + return []string{did}, nil } } - if len(page) < listPageSize { - break - } - offset += listPageSize + return nil, fmt.Errorf( + "the run is scoped to %s, but this AppView holds no PDS credentials for it: nothing could be written to that repo and nothing may be deleted from it", + s.communityFilter) } - return dids, nil -} -// communityClient opens a full PDS client bound to one community's repo, over -// freshly-renewed stored credentials. -func (s *realLegacySource) communityClient(ctx context.Context, did string) (pds.Client, error) { - community, err := s.creds.GetByDID(ctx, did) - if err != nil { - return nil, fmt.Errorf("reading the credentials of %s: %w", did, err) - } - if community == nil { - return nil, fmt.Errorf("reading the credentials of %s: no such community is indexed", did) - } - fresh, err := s.creds.EnsureFreshToken(ctx, community) - if err != nil { - return nil, fmt.Errorf("renewing the credentials of %s: %w", did, err) + if len(hosted) == 0 { + return nil, errors.New( + "this AppView hosts no communities (no stored PDS refresh tokens), so there is nothing this tool could migrate. " + + "Check the database the tool is pointed at before assuming the migration is done") } - if fresh == nil || fresh.PDSAccessToken == "" { - return nil, fmt.Errorf("renewing the credentials of %s: no access token came back", did) + return hosted, nil +} + +// communityRepoOpener builds the production repo opener: a full PDS client bound +// to one community's repo, over freshly-renewed stored credentials. +func communityRepoOpener(creds posts.CommunityCredentialSource) func(context.Context, string) (repoClient, error) { + return func(ctx context.Context, did string) (repoClient, error) { + community, err := creds.GetByDID(ctx, did) + if err != nil { + return nil, fmt.Errorf("reading the credentials of %s: %w", did, err) + } + if community == nil { + return nil, fmt.Errorf("reading the credentials of %s: no such community is indexed", did) + } + fresh, err := creds.EnsureFreshToken(ctx, community) + if err != nil { + return nil, fmt.Errorf("renewing the credentials of %s: %w", did, err) + } + if fresh == nil || fresh.PDSAccessToken == "" { + return nil, fmt.Errorf("renewing the credentials of %s: no access token came back", did) + } + client, err := pds.NewFromAccessToken(fresh.PDSURL, fresh.DID, fresh.PDSAccessToken) + if err != nil { + return nil, err + } + guarded, ok := client.(repoClient) + if !ok { + return nil, fmt.Errorf( + "the PDS client for %s does not support the swap-guarded delete; an unguarded delete would remove whatever stands, so the run stops here", did) + } + return guarded, nil } - return pds.NewFromAccessToken(fresh.PDSURL, fresh.DID, fresh.PDSAccessToken) } // legacyPostFromEntry decodes one listRecords entry into a LegacyPost. @@ -276,6 +516,10 @@ func legacyPostFromEntry(communityDID string, entry pds.RecordEntry) (posts.Lega if record.Author == "" { return posts.LegacyPost{}, fmt.Errorf("record %s carries no author field to re-author under", entry.URI) } + if entry.CID == "" { + return posts.LegacyPost{}, fmt.Errorf( + "record %s was listed without a CID; with no CID there is nothing to guard its eventual delete on", entry.URI) + } return posts.LegacyPost{ URI: entry.URI, CID: entry.CID, @@ -287,22 +531,173 @@ func legacyPostFromEntry(communityDID string, entry pds.RecordEntry) (posts.Lega }, nil } -// logCensus prints the run's per-state tally. -func logCensus(report posts.RematerializeReport) { - log.Printf("rematerialize-posts: census — discovered=%d done=%d fallbacks=%d complete=%v", - report.Discovered, report.Done, report.Fallbacks, report.Complete) - for state, n := range report.ByState { - log.Printf(" %-22s %d", state, n) +// splitLegacyURI pulls the repo authority and the record key out of an at:// URI. +func splitLegacyURI(uri string) (repoDID, rkey string, err error) { + const scheme = "at://" + if !strings.HasPrefix(uri, scheme) { + return "", "", fmt.Errorf("%q is not an at:// URI", uri) + } + parts := strings.Split(strings.TrimPrefix(uri, scheme), "/") + if len(parts) != 3 || parts[0] == "" || parts[2] == "" { + return "", "", fmt.Errorf("%q is not a // record URI", uri) + } + return parts[0], parts[2], nil +} + +// targetScope is what the banner reports: the size of what is about to be +// touched, measured rather than assumed. +type targetScope struct { + communities int + records int +} + +// describeTarget enumerates the source so the banner states facts. +func describeTarget(ctx context.Context, cfg *config.Config, source *realLegacySource) (targetScope, error) { + _ = cfg + dids, err := source.hostedCommunityDIDs(ctx) + if err != nil { + return targetScope{}, err } + legacy, err := source.ListLegacyPosts(ctx) + if err != nil { + return targetScope{}, err + } + return targetScope{communities: len(dids), records: len(legacy)}, nil } -func lastSlash(s string) int { - for i := len(s) - 1; i >= 0; i-- { - if s[i] == '/' { - return i +// printBanner states what this invocation is pointed at, before it is allowed to +// touch any of it. +// +// The database URL is printed WITHOUT its credentials: an operator needs to see +// which host and database they are about to migrate, and nobody needs the +// password in a terminal scrollback that will be pasted into an incident channel. +func printBanner(cfg *config.Config, scope targetScope, dryRun bool, communityFilter string) { + mode := "LIVE RUN — WRITES AND IRREVERSIBLE DELETES" + if dryRun { + mode = "DRY RUN — nothing will be written or deleted" + } + log.Printf("──────────────────────────────────────────────────────────────") + log.Printf(" rematerialize-posts %s", mode) + log.Printf(" database %s", redactDatabaseURL(cfg.Database.URL)) + log.Printf(" PDS %s", cfg.PDS.URL) + log.Printf(" instance %s (%s)", cfg.Instance.DID, cfg.Instance.Domain) + log.Printf(" scope %s", scopeName(communityFilter)) + log.Printf(" communities %d", scope.communities) + log.Printf(" legacy %s %d", legacyPostCollection, scope.records) + log.Printf("──────────────────────────────────────────────────────────────") +} + +func scopeName(communityFilter string) string { + if communityFilter == "" { + return "every hosted community" + } + return communityFilter +} + +// redactDatabaseURL renders a Postgres URL as host/database only. +func redactDatabaseURL(raw string) string { + if at := strings.LastIndex(raw, "@"); at != -1 { + if scheme := strings.Index(raw, "://"); scheme != -1 && scheme+3 < at { + return raw[:scheme+3] + "***@" + raw[at+1:] } } - return -1 + return raw +} + +// progressLogger turns the Rematerializer's transitions into one line each, plus +// a rolling n/N — so the operator can see a slow run is still moving, and so a +// crash leaves a record of exactly how far it got. +type progressLogger struct { + lastIndex int + lastTotal int +} + +func newProgressLogger() *progressLogger { return &progressLogger{} } + +func (p *progressLogger) log(event posts.RematerializeProgress) { + position := "" + if event.Total > 0 { + p.lastIndex, p.lastTotal = event.Index, event.Total + position = fmt.Sprintf(" [%d/%d]", event.Index, event.Total) + } + switch { + case event.Note != "": + log.Printf(" %s%s %s → %s: %s", event.OldURI, position, orDash(event.From), orDash(event.To), event.Note) + default: + log.Printf(" %s%s %s → %s", event.OldURI, position, orDash(event.From), orDash(event.To)) + } +} + +func orDash(state posts.RematerializeState) string { + if state == "" { + return "-" + } + return string(state) +} + +// logCensus prints the run's per-state tally, for the run's scope and for the +// migration as a whole. +func logCensus(report posts.RematerializeReport, dryRun bool) { + prefix := "census" + if dryRun { + prefix = "census (dry run — no state was persisted)" + } + log.Printf("rematerialize-posts: %s — scope=%s discovered=%d done=%d fallbacks=%d remaining-legacy=%d scope-complete=%v", + prefix, scopeName(report.CommunityScope), report.Discovered, report.Done, report.Fallbacks, + report.RemainingLegacy, report.ScopeComplete) + for _, state := range censusOrder { + if n, ok := report.ByState[state]; ok { + log.Printf(" %-22s %d", state, n) + } + } + if report.CommunityScope != "" { + log.Printf("rematerialize-posts: whole migration — discovered=%d done=%d fallbacks=%d complete=%v", + report.GlobalDiscovered, report.GlobalDone, report.GlobalFallbacks, report.Complete) + } +} + +// censusOrder prints the states in machine order rather than map order, so two +// runs' output can be diffed. +var censusOrder = []posts.RematerializeState{ + posts.RematerializeDiscovered, + posts.RematerializePostV2Written, + posts.RematerializeVerified, + posts.RematerializeMigrated, + posts.RematerializeDone, + posts.RematerializeFallbackLeftLegacy, +} + +// takeAdvisoryLock holds a session-scoped Postgres advisory lock for the whole +// run, on a DEDICATED connection. +// +// It must be a dedicated connection: a session lock taken through the pool is +// released the moment that connection is recycled, which is a lock that protects +// nothing and reports that it does. pg_try_advisory_lock rather than the +// blocking form, because "another run is in progress" is information the +// operator needs immediately, not after an unbounded wait. +func takeAdvisoryLock(ctx context.Context, db *sql.DB) (release func(), err error) { + conn, err := db.Conn(ctx) + if err != nil { + return nil, fmt.Errorf("reserving a connection for the run lock: %w", err) + } + var acquired bool + if err := conn.QueryRowContext(ctx, `SELECT pg_try_advisory_lock($1)`, rematerializeAdvisoryLock).Scan(&acquired); err != nil { + _ = conn.Close() + return nil, fmt.Errorf("taking the run lock: %w", err) + } + if !acquired { + _ = conn.Close() + return nil, errors.New( + "another rematerialize-posts run is already in progress against this database. " + + "Two concurrent runs interleave on the ledger's guarded transitions and report 'the ledger and the tool have diverged', " + + "which is indistinguishable from corruption. Wait for the other run to finish") + } + return func() { + // Best-effort: the lock is session-scoped, so closing the connection + // releases it even if the explicit unlock cannot run. + _, _ = conn.ExecContext(context.Background(), `SELECT pg_advisory_unlock($1)`, rematerializeAdvisoryLock) + _ = conn.Close() + }, nil } // openDatabase opens the AppView Postgres the ledger and community catalogue @@ -346,7 +741,8 @@ func buildOAuthClient(cfg *config.Config, db *sql.DB) (*oauth.OAuthClient, error // oauthScopes is the granted scope set an aggregator's resumed session must // carry to write a postv2. It mirrors cmd/server's list, which is the authority; // the two must agree, so a divergence here shows up as a scope the resumed -// session lacks at the first write. +// session lacks at the first write. main_test.go asserts the two lists are +// equal, because cmd/server's own test cannot — it is a different package. func oauthScopes() []string { return []string{ "atproto", diff --git a/cmd/rematerialize-posts/main_test.go b/cmd/rematerialize-posts/main_test.go new file mode 100644 index 0000000..ffdadd9 --- /dev/null +++ b/cmd/rematerialize-posts/main_test.go @@ -0,0 +1,355 @@ +package main + +import ( + "context" + "fmt" + "strings" + "testing" + + "Coves/internal/atproto/pds" + "Coves/internal/core/posts" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The production LegacySource and the tool's operator surface. +// +// Everything in this file was previously untested: the tool that DELETES +// production posts had coverage of its state machine and none at all of the +// three seams that actually touch the PDS — the discovery decode, the guarded +// delete, and the not-found idempotence a resumed run leans on. Those are also +// the seams whose bugs are invisible in the state machine's fakes, because the +// fakes are written to the contract rather than to what the PDS does. + +// fakeRepoClient is a minimal pds.Client over an in-memory repo, plus the +// swap-guarded delete the real client provides. +type fakeRepoClient struct { + did string + host string + records map[string]*pds.RecordResponse // rkey -> record + listErr error + deleteOp []deleteCall + deleteFn func(collection, rkey, swap string) error +} + +type deleteCall struct { + collection string + rkey string + swap string +} + +func newFakeRepoClient(did string) *fakeRepoClient { + return &fakeRepoClient{did: did, host: "http://pds.invalid", records: map[string]*pds.RecordResponse{}} +} + +func (c *fakeRepoClient) put(collection, rkey, cid string, value map[string]any) string { + uri := "at://" + c.did + "/" + collection + "/" + rkey + c.records[rkey] = &pds.RecordResponse{URI: uri, CID: cid, Value: value} + return uri +} + +func (c *fakeRepoClient) DID() string { return c.did } +func (c *fakeRepoClient) HostURL() string { return c.host } + +func (c *fakeRepoClient) GetRecord(_ context.Context, _ string, rkey string) (*pds.RecordResponse, error) { + rec, ok := c.records[rkey] + if !ok { + return nil, fmt.Errorf("getRecord: %w: no such record", pds.ErrNotFound) + } + return rec, nil +} + +func (c *fakeRepoClient) ListRecords(_ context.Context, _ string, _ int, cursor string) (*pds.ListRecordsResponse, error) { + if c.listErr != nil { + return nil, c.listErr + } + if cursor != "" { + return &pds.ListRecordsResponse{}, nil + } + out := &pds.ListRecordsResponse{} + for _, rec := range c.records { + out.Records = append(out.Records, pds.RecordEntry{URI: rec.URI, CID: rec.CID, Value: rec.Value}) + } + return out, nil +} + +func (c *fakeRepoClient) DeleteRecordWithSwap(_ context.Context, collection, rkey, swap string) error { + c.deleteOp = append(c.deleteOp, deleteCall{collection: collection, rkey: rkey, swap: swap}) + if c.deleteFn != nil { + return c.deleteFn(collection, rkey, swap) + } + delete(c.records, rkey) + return nil +} + +// sourceOver builds a realLegacySource whose community clients are the supplied +// fakes, bypassing the credential plumbing that needs a database. +func sourceOver(clients map[string]*fakeRepoClient, scope string) *realLegacySource { + return &realLegacySource{ + communityFilter: scope, + hostedDIDs: func(context.Context) ([]string, error) { + dids := make([]string, 0, len(clients)) + for did := range clients { + dids = append(dids, did) + } + return dids, nil + }, + openRepo: func(_ context.Context, did string) (repoClient, error) { + c, ok := clients[did] + if !ok { + return nil, fmt.Errorf("no fake client for %s", did) + } + return c, nil + }, + } +} + +// ---- legacyPostFromEntry: the lossless-conversion source ------------------- + +// RawRecord is where the postv2's body comes from. If it were rebuilt from the +// decoded PostRecord instead, langs/tags/crosspostOf/crosspostChain/bridgedStats +// would be dropped before the only copy of the record was deleted. +func TestLegacyPostFromEntry_CarriesTheRawRecordThroughVerbatim(t *testing.T) { + value := map[string]any{ + "$type": "social.coves.community.post", + "community": "did:plc:community2222222222222222", + "author": "did:plc:author11111111111111111", + "title": "a post", + "createdAt": "2026-01-02T03:04:05Z", + "langs": []any{"en"}, + "tags": []any{"golang"}, + "crosspostOf": map[string]any{"uri": "at://did:plc:x/social.coves.community.postv2/abc", "cid": "bafyx"}, + "crosspostChain": []any{map[string]any{"uri": "at://did:plc:x/social.coves.community.postv2/abc", "cid": "bafyx"}}, + "bridgedStats": map[string]any{"upvotes": float64(7)}, + } + + got, err := legacyPostFromEntry("did:plc:community2222222222222222", pds.RecordEntry{ + URI: "at://did:plc:community2222222222222222/social.coves.community.post/3kabc", + CID: "bafylegacy", + Value: value, + }) + require.NoError(t, err) + + assert.Equal(t, "did:plc:author11111111111111111", got.AuthorDID) + assert.Equalf(t, "bafylegacy", got.CID, + "the entry CID must be carried: it is the value the pre-delete re-read is checked against and the swap guard the delete is sent under") + for field, want := range value { + assert.Equalf(t, want, got.RawRecord[field], + "RawRecord dropped or altered %q. The postv2 is built from this map; a field lost here is lost from the record before the original is deleted", field) + } +} + +func TestLegacyPostFromEntry_RefusesARecordWithNoAuthor(t *testing.T) { + _, err := legacyPostFromEntry("did:plc:community2222222222222222", pds.RecordEntry{ + URI: "at://did:plc:community2222222222222222/social.coves.community.post/3kabc", + CID: "bafylegacy", + Value: map[string]any{"$type": "social.coves.community.post", "title": "orphan"}, + }) + require.Errorf(t, err, + "a legacy record with no author field must fail discovery: there is no repo to re-author it under, and guessing one is the forgery the whole flip removes") +} + +func TestLegacyPostFromEntry_RefusesARecordWithNoCID(t *testing.T) { + _, err := legacyPostFromEntry("did:plc:community2222222222222222", pds.RecordEntry{ + URI: "at://did:plc:community2222222222222222/social.coves.community.post/3kabc", + Value: map[string]any{"$type": "social.coves.community.post", "author": "did:plc:author11111111111111111"}, + }) + require.Errorf(t, err, + "a legacy record listed without a CID must fail discovery: with no CID there is nothing to guard the delete on, and the tool would fall back to deleting whatever stands") +} + +// ---- DeleteLegacyPost ------------------------------------------------------ + +// The delete is the irreversible step, and it must be GUARDED. An unguarded +// delete removes whatever stands — including an edit that landed after the +// postv2 was built from an earlier version. +func TestDeleteLegacyPost_SendsTheSourceCIDAsTheSwapGuard(t *testing.T) { + community := newFakeRepoClient("did:plc:community2222222222222222") + uri := community.put(legacyPostCollection, "3kabc", "bafylegacy", map[string]any{"title": "t"}) + source := sourceOver(map[string]*fakeRepoClient{community.did: community}, "") + + err := source.DeleteLegacyPost(context.Background(), posts.LegacyPost{ + URI: uri, CID: "bafylegacy", CommunityDID: community.did, + }, "bafylegacy") + require.NoError(t, err) + + require.Lenf(t, community.deleteOp, 1, "exactly one delete must have been issued") + assert.Equalf(t, "bafylegacy", community.deleteOp[0].swap, + "the delete was sent WITHOUT the source CID as swapRecord. The PDS is the only place the CID check and the delete happen atomically; "+ + "without the guard, an edit landing between the tool's check and its delete is destroyed") + assert.Equal(t, "3kabc", community.deleteOp[0].rkey) + assert.Equal(t, legacyPostCollection, community.deleteOp[0].collection) +} + +func TestDeleteLegacyPost_RefusesAnEmptySwapCID(t *testing.T) { + community := newFakeRepoClient("did:plc:community2222222222222222") + uri := community.put(legacyPostCollection, "3kabc", "bafylegacy", map[string]any{"title": "t"}) + source := sourceOver(map[string]*fakeRepoClient{community.did: community}, "") + + err := source.DeleteLegacyPost(context.Background(), posts.LegacyPost{ + URI: uri, CID: "bafylegacy", CommunityDID: community.did, + }, "") + require.Errorf(t, err, + "a delete with no swap CID must be refused outright; 'I have no CID to guard on' is exactly the state in which a delete must not proceed") + assert.Emptyf(t, community.deleteOp, "no delete may reach the PDS when there is nothing to guard it with") +} + +// A delete of an already-gone record is SUCCESS. It is the step a crash after +// the migrated checkpoint retries, so idempotence is the contract — and a +// resumed run that treated not-found as a failure could never finish. +func TestDeleteLegacyPost_NotFoundIsSuccess(t *testing.T) { + community := newFakeRepoClient("did:plc:community2222222222222222") + community.deleteFn = func(string, string, string) error { + return fmt.Errorf("deleteRecord: %w: no such record", pds.ErrNotFound) + } + source := sourceOver(map[string]*fakeRepoClient{community.did: community}, "") + + err := source.DeleteLegacyPost(context.Background(), posts.LegacyPost{ + URI: "at://" + community.did + "/social.coves.community.post/3kgone", + CID: "bafylegacy", + CommunityDID: community.did, + }, "bafylegacy") + assert.NoErrorf(t, err, + "a delete of an already-gone record must report success: it is the step a crash after the migrated checkpoint retries, and a resumed run that "+ + "treated not-found as failure could never reach done") +} + +// A LOST SWAP IS NOT SUCCESS. It means the record changed under us, which is the +// exact case the guard exists to catch. +func TestDeleteLegacyPost_SwapConflictIsAnError(t *testing.T) { + community := newFakeRepoClient("did:plc:community2222222222222222") + community.deleteFn = func(string, string, string) error { + return fmt.Errorf("deleteRecord: %w: InvalidSwap", pds.ErrSwapConflict) + } + source := sourceOver(map[string]*fakeRepoClient{community.did: community}, "") + + err := source.DeleteLegacyPost(context.Background(), posts.LegacyPost{ + URI: "at://" + community.did + "/social.coves.community.post/3kabc", + CID: "bafylegacy", + CommunityDID: community.did, + }, "bafystale") + require.Errorf(t, err, + "a lost swap must surface as an error: the record carries a different CID than the postv2 was built from, so the delete would destroy unmigrated content") + assert.Containsf(t, err.Error(), "changed", + "the error must say the record changed, so a 3am operator is not left decoding 'InvalidSwap'") +} + +// ---- scope enforcement ----------------------------------------------------- + +// The -community filter must narrow DISCOVERY, and it must also make a delete +// outside the scope impossible: the ledger reconcile pass reaches records the +// discovery pass never listed. +func TestListLegacyPosts_CommunityFilterNarrowsDiscovery(t *testing.T) { + inScope := newFakeRepoClient("did:plc:inscope22222222222222222") + outOfScope := newFakeRepoClient("did:plc:outscope3333333333333333") + inScope.put(legacyPostCollection, "3kin", "bafyin", map[string]any{ + "$type": legacyPostCollection, "author": "did:plc:author11111111111111111", "title": "in", + }) + outOfScope.put(legacyPostCollection, "3kout", "bafyout", map[string]any{ + "$type": legacyPostCollection, "author": "did:plc:author11111111111111111", "title": "out", + }) + + source := sourceOver(map[string]*fakeRepoClient{ + inScope.did: inScope, + outOfScope.did: outOfScope, + }, inScope.did) + + found, err := source.ListLegacyPosts(context.Background()) + require.NoError(t, err) + require.Len(t, found, 1) + assert.Equalf(t, inScope.did, found[0].CommunityDID, + "a scoped run listed a record outside its scope; discovery is where the staged rollout's boundary starts") +} + +func TestDeleteLegacyPost_RefusesACommunityOutsideTheScope(t *testing.T) { + inScope := newFakeRepoClient("did:plc:inscope22222222222222222") + outOfScope := newFakeRepoClient("did:plc:outscope3333333333333333") + outOfScope.put(legacyPostCollection, "3kout", "bafyout", map[string]any{"title": "out"}) + + source := sourceOver(map[string]*fakeRepoClient{ + inScope.did: inScope, + outOfScope.did: outOfScope, + }, inScope.did) + + err := source.DeleteLegacyPost(context.Background(), posts.LegacyPost{ + URI: "at://" + outOfScope.did + "/social.coves.community.post/3kout", + CID: "bafyout", + CommunityDID: outOfScope.did, + }, "bafyout") + require.Errorf(t, err, + "a staged run deleted a record belonging to a community outside its scope. The ledger reconcile pass reaches rows discovery never listed, so the "+ + "scope has to be enforced at the delete, not only at discovery") + assert.Emptyf(t, outOfScope.deleteOp, "no delete may reach a community outside the run's scope") +} + +// ---- ReadLegacyPost -------------------------------------------------------- + +func TestReadLegacyPost_ReportsTheCurrentCIDAndAbsence(t *testing.T) { + community := newFakeRepoClient("did:plc:community2222222222222222") + uri := community.put(legacyPostCollection, "3kabc", "bafycurrent", map[string]any{ + "$type": legacyPostCollection, "author": "did:plc:author11111111111111111", "title": "t", + }) + source := sourceOver(map[string]*fakeRepoClient{community.did: community}, "") + + got, found, err := source.ReadLegacyPost(context.Background(), uri) + require.NoError(t, err) + require.True(t, found) + assert.Equalf(t, "bafycurrent", got.CID, + "the fresh read must report the CID the record carries NOW; it is the whole point of re-reading before the delete") + + _, found, err = source.ReadLegacyPost(context.Background(), "at://"+community.did+"/social.coves.community.post/3kgone") + require.NoErrorf(t, err, "a record that is simply gone is not an error — a resumed run whose delete already landed meets exactly this") + assert.Falsef(t, found, "an absent record must be reported as absent, not as an error and not as an empty record") +} + +// ---- the duplicated scope list --------------------------------------------- + +// The tool resumes the SAME sessions cmd/server mints, so the two scope lists +// must agree. A comment in main.go already claims they must; nothing checked it, +// and cmd/server's own test cannot — it is a different package. +func TestOAuthScopes_MatchTheServerScopeList(t *testing.T) { + toolScopes := oauthScopes() + + require.NotEmpty(t, toolScopes) + assert.Equalf(t, serverOAuthScopesForComparison(), toolScopes, + "the tool's OAuth scope list has drifted from cmd/server's. A session this tool resumes carries the scopes the SERVER granted, so a scope the "+ + "tool believes it has and the server never asked for is refused at the first write — mid-migration, after records have already been deleted.\n"+ + "tool: %v\nserver: %v", toolScopes, serverOAuthScopesForComparison()) +} + +// serverOAuthScopesForComparison is cmd/server's oauthScopes() transcribed. It +// is a literal copy on purpose: the two binaries are different packages, so the +// only way to compare them in a test is to state one of them here and let this +// assertion fail when either moves. +func serverOAuthScopesForComparison() []string { + return []string{ + "atproto", + "blob:*/*", + "repo:social.coves.community.postv2?action=create&action=update&action=delete", + "repo:social.coves.community.post?action=create&action=update&action=delete", + "repo:social.coves.community.comment?action=create&action=update&action=delete", + "repo:social.coves.community.profile?action=create&action=update&action=delete", + "repo:social.coves.community.subscription?action=create&action=update&action=delete", + "repo:social.coves.actor.profile?action=create&action=update&action=delete", + "repo:social.coves.feed.vote?action=create&action=delete", + "repo:social.coves.actor.block?action=create&action=delete", + } +} + +// The scope that lets the tool DELETE the legacy records has to be there, and it +// is the one a well-meaning cleanup of the deprecated collection would remove +// first. +func TestOAuthScopes_GrantTheLegacyDeleteTheDrainDependsOn(t *testing.T) { + var legacy string + for _, s := range oauthScopes() { + if strings.HasPrefix(s, "repo:"+posts.LegacyPostCollection) { + legacy = s + } + } + require.NotEmptyf(t, legacy, + "the tool has no scope for %s. Every legacy record is deleted through a session minted with these scopes, so without it the drain strands "+ + "the entire corpus undeleteable", posts.LegacyPostCollection) + assert.Containsf(t, legacy, "action=delete", + "the legacy-post scope %q grants no delete; the drain's final step is exactly that delete", legacy) +} diff --git a/internal/atproto/pds/delete_swap.go b/internal/atproto/pds/delete_swap.go new file mode 100644 index 0000000..709ec55 --- /dev/null +++ b/internal/atproto/pds/delete_swap.go @@ -0,0 +1,72 @@ +package pds + +import ( + "context" + "fmt" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +// The GUARDED delete: com.atproto.repo.deleteRecord with a swapRecord CID. +// +// # WHY THIS EXISTS SEPARATELY FROM DeleteRecord +// +// Client.DeleteRecord sends no swap guard, which is right for the ordinary case: +// a user deleting their own post means "whatever is there, remove it", and +// making every caller carry a CID would turn a delete into a read-then-write +// race with itself. +// +// It is exactly wrong for a MIGRATION. The re-materialization tool reads a +// legacy record, converts it, writes the replacement, and deletes the original +// minutes or hours later. In that gap an edit can land — a cron the maintenance +// window did not stop, a mobile session on a cached token — and an unguarded +// delete destroys the newer content with no trace, having verified a replacement +// for the OLDER content. The tool checks the CID itself before deleting, but a +// check is a moment and the write that follows it is another; only a guard the +// PDS evaluates atomically with the delete closes that window. +// +// So this is a separate, opt-in surface: callers that want "remove whatever +// stands" keep the plain method, and callers that must not destroy an unseen +// version ask for the guard by type. + +// GuardedDeleter is the swap-guarded delete surface. +// +// It is an interface (satisfied by the concrete client) rather than a method on +// Client so that a caller REQUIRING the guard states that requirement in its own +// types and fails to compile against a transport that cannot provide it, instead +// of silently falling back to an unguarded delete at 3am. +type GuardedDeleter interface { + // DeleteRecordWithSwap deletes a record only if it currently carries + // swapRecord as its CID. A mismatch comes back as ErrSwapConflict; a record + // that is already gone comes back as ErrNotFound, which an idempotent caller + // may treat as success. + // + // swapRecord is REQUIRED. Passing an empty string is an error rather than a + // silent unguarded delete, because "I have no CID to guard on" is precisely + // the state in which a delete must not proceed. + DeleteRecordWithSwap(ctx context.Context, collection, rkey, swapRecord string) error +} + +// Ensure the concrete client provides the guarded delete. +var _ GuardedDeleter = (*client)(nil) + +// DeleteRecordWithSwap deletes a record under an optimistic-concurrency guard. +func (c *client) DeleteRecordWithSwap(ctx context.Context, collection, rkey, swapRecord string) error { + if swapRecord == "" { + return fmt.Errorf("deleteRecord: refusing to delete %s/%s in %s without a swapRecord guard: "+ + "an unguarded delete removes whatever stands, including a version this caller has never seen", + collection, rkey, c.did) + } + + payload := map[string]any{ + "repo": c.did, + "collection": collection, + "rkey": rkey, + "swapRecord": swapRecord, + } + + if err := c.apiClient.Post(ctx, syntax.NSID("com.atproto.repo.deleteRecord"), payload, nil); err != nil { + return wrapAPIError(err, "deleteRecord") + } + return nil +} diff --git a/internal/core/posts/author_repo_factory.go b/internal/core/posts/author_repo_factory.go index fd75e3d..1e232d5 100644 --- a/internal/core/posts/author_repo_factory.go +++ b/internal/core/posts/author_repo_factory.go @@ -2,11 +2,13 @@ package posts import ( "context" + "errors" "fmt" "github.com/bluesky-social/indigo/atproto/auth/oauth" "github.com/bluesky-social/indigo/atproto/syntax" + covesoauth "Coves/internal/atproto/oauth" "Coves/internal/atproto/pds" ) @@ -37,6 +39,20 @@ import ( // running, correctly configured, and completely unable to post. Collapsing them // would have a revoked aggregator grant diagnosed as a PDS outage, or a // signed-out user told to file a ticket. +// +// # AND WHY "MISSING" IS NARROWER THAN "FAILED" (classifyResumeFailure) +// +// ErrNoAuthorCredentials is a TERMINAL verdict to one caller: the +// re-materialization census writes it as fallback_left_legacy, a state nothing +// in the tool can move a row back out of. Reporting every ResumeSession failure +// under it therefore turns a network blip, a PDS 5xx, or a DPoP nonce failure +// into a permanent sentence — and since one aggregator authors most of the +// corpus, into a permanent sentence over most of the corpus, in seconds. +// +// So only "the store holds no live grant for this DID" keeps the terminal +// sentinel. Every other failure is ErrAuthorCredentialsUnavailable, which is +// RETRYABLE and which the tool answers by failing the run loudly rather than by +// writing a verdict it cannot take back. func NewAuthorRepoFactory(oauthClient *oauth.ClientApp, storedSessionID string) AuthorRepoFactory { return func(ctx context.Context, authorDID string, session *oauth.ClientSessionData) (AuthorRepo, error) { if oauthClient == nil { @@ -57,8 +73,7 @@ func NewAuthorRepoFactory(oauthClient *oauth.ClientApp, storedSessionID string) // to resume" is answered in the vocabulary the boundary needs. resumed, resumeErr := oauthClient.ResumeSession(ctx, did, storedSessionID) if resumeErr != nil { - return nil, fmt.Errorf("resuming the stored session of %s: %w: %w", - authorDID, ErrNoAuthorCredentials, resumeErr) + return nil, classifyResumeFailure(authorDID, resumeErr) } if resumed == nil || resumed.Data == nil { return nil, fmt.Errorf("resuming the stored session of %s: the store returned nothing: %w", @@ -91,3 +106,41 @@ func NewAuthorRepoFactory(oauthClient *oauth.ClientApp, storedSessionID string) return repo, nil } } + +// ErrAuthorCredentialsUnavailable reports that the author's credentials could +// not be resolved RIGHT NOW — and says nothing about whether a grant exists. +// +// It is the counterpart to ErrNoAuthorCredentials, and the distinction is the +// whole point: ErrNoAuthorCredentials means "there is nothing to resume, and a +// batch tool cannot make there be", which is a terminal outcome; this one means +// "ask again", which must never be recorded as a verdict. A caller that cannot +// act on the difference should treat this one as fatal to its run, because +// continuing past it silently narrows the work it believes is left. +var ErrAuthorCredentialsUnavailable = errors.New("the author's credentials could not be resolved right now") + +// classifyResumeFailure decides whether a failed session resume is a verdict or +// a retry. +// +// THE ONLY TERMINAL CASE IS AN ABSENT GRANT. The session store answers a DID it +// holds no live row for with ErrSessionNotFound, and that is the condition a +// batch tool genuinely cannot resolve on its own: nobody is at the keyboard to +// re-authorize. (A stored session past its expiry reads as the same absence, +// which is correct — an expired grant also needs a human — and it is exactly why +// the ledger has an operator-driven way back out of the fallback state.) +// +// Everything else — a refused dial, a 5xx from the PDS, a DPoP nonce dance that +// did not converge, a database error reading the store — is transport. Those are +// reported as retryable so the run stops and says so, instead of writing a +// terminal fallback for every post by an author whose session happened to be +// mid-blip. +func classifyResumeFailure(authorDID string, resumeErr error) error { + if resumeErr == nil { + return nil + } + if errors.Is(resumeErr, covesoauth.ErrSessionNotFound) { + return fmt.Errorf("resuming the stored session of %s: %w: %w", + authorDID, ErrNoAuthorCredentials, resumeErr) + } + return fmt.Errorf("resuming the stored session of %s: %w: %w", + authorDID, ErrAuthorCredentialsUnavailable, resumeErr) +} diff --git a/internal/core/posts/rematerialize.go b/internal/core/posts/rematerialize.go index a252429..84db331 100644 --- a/internal/core/posts/rematerialize.go +++ b/internal/core/posts/rematerialize.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "net/http" + "sync" "time" "github.com/bluesky-social/indigo/atproto/syntax" @@ -23,12 +24,27 @@ import ( // (docs/PRD_AUTHOR_OWNED_POSTS.md §10.1 step 5 and §11 the rev-2.8 deploy // runbook). // -// # THIS FILE IS A RED STUB +// # THIS CODE DELETES PRODUCTION USER DATA IRREVERSIBLY // -// Every exported symbol here exists so tests/e2e/rematerialize_contract_test.go -// and the T1 state-machine tests compile and FAIL for the right reason. The -// bodies return the not-implemented sentinel; GREEN fills them in. Nothing in -// this file decides anything yet. +// Read that sentence before changing anything below it. The tool's only defence +// against destroying a post is that it VERIFIES, with fresh reads, that a +// replacement stands — and it re-verifies on every path, including a resumed +// one, because a ledger row is a memory of a past truth and a delete needs a +// present one. +// +// It is exercised by: +// - internal/core/posts/rematerialize_rkey_test.go (T0: the rkey derivation, +// including the golden values that pin it ACROSS processes) +// - internal/core/posts/rematerialize_credentials_test.go (T0: the terminal +// vs. retryable credential split) +// - internal/core/posts/rematerialize_test.go (T1: the state machine +// against a real migration-037 ledger and faked repos) +// - internal/core/posts/rematerialize_guard_test.go (T1: the pre-delete +// verification guards and the crash/resume boundaries) +// - internal/core/posts/rematerialize_outer_test.go (T1: the whole tool +// against a REAL PDS and a real ledger) +// - cmd/rematerialize-posts/*_test.go (T0: the production +// LegacySource, the scope wiring and the operator surface) // // # WHY THE LOGIC LIVES IN PACKAGE posts (not a subpackage) // @@ -53,7 +69,7 @@ import ( // RematerializeState is one legacy record's position in the ledger state machine // (migration 037). The happy path is discovered → postv2_written → verified → -// migrated → done; the two fallback states are terminal. +// migrated → done; the fallback state is terminal. type RematerializeState string const ( @@ -67,12 +83,15 @@ const ( // RematerializeVerified: the acceptance stands in the COMMUNITY's repo and // both records have been read back and confirmed to pin the same CID. + // + // IT IS NOT A LICENCE TO DELETE. It records that verification passed ONCE, + // at a moment now in the past; the delete re-verifies from scratch. RematerializeVerified RematerializeState = "verified" // RematerializeMigrated is the CHECKPOINT BEFORE DELETE, distinct from done // on purpose: it means "verified safe to delete, and the OLD record is still // present". A crash after this checkpoint resumes by retrying ONLY the delete - // (§11 step 4). + // (§11 step 4) — but that retry re-verifies first, for the same reason. RematerializeMigrated RematerializeState = "migrated" // RematerializeDone: the old community.post record has been deleted. @@ -84,22 +103,24 @@ const ( // the fallback NEVER forges authorship (no admin-signed postv2), because // forging reintroduces the exact §2 impersonation liability the whole flip // removes. + // + // IT IS TERMINAL BUT NOT IRREVERSIBLE. RematerializeLedger.ReopenFallback + // moves such rows back to discovered, which is the supported recovery once + // the operator has re-authorized the author — see its doc, and the header of + // migration 037. + // + // There is exactly ONE fallback state on purpose. An earlier revision also + // declared fallback_no_creds, and no code path ever wrote it: a vocabulary + // entry nothing produces is a trap for whoever writes recovery SQL at 2am + // against a state that cannot exist. RematerializeFallbackLeftLegacy RematerializeState = "fallback_left_legacy" - - // RematerializeFallbackNoCreds is reserved terminal vocabulary for - // distinguishing "the author-repo factory reported no credentials" from a - // human-operator-flagged leave-as-legacy. Cycle 1 pins only the behaviour - // (a no-creds author is left legacy and the run is not complete); which of - // the two strings a given cause writes is a GREEN/owner decision — see the - // ambiguity flag in the RED report. - RematerializeFallbackNoCreds RematerializeState = "fallback_no_creds" ) -// IsFallback reports whether a state is one of the terminal fallback states, so -// the census can gate "complete" on any of them surviving without enumerating -// each string at every call site. +// IsFallback reports whether a state is a terminal fallback state, so the census +// can gate "complete" on any of them surviving without enumerating each string +// at every call site. func IsFallback(state RematerializeState) bool { - return state == RematerializeFallbackLeftLegacy || state == RematerializeFallbackNoCreds + return state == RematerializeFallbackLeftLegacy } // LegacyPost is one deprecated social.coves.community.post record discovered for @@ -110,9 +131,16 @@ type LegacyPost struct { // rkey is derived from. URI string - // CID is the old record's content CID, carried for audit only. Authorship - // and content come from the author's repo once the postv2 is written; the - // old CID is never pinned by the new acceptance. + // CID is the old record's content CID AS OF THE READ THAT PRODUCED THIS + // VALUE, and it is LOAD-BEARING, not audit trim. + // + // The postv2 is built from a body read at one instant; the delete happens + // minutes to hours later. If an aggregator cron or a cached mobile session + // lands an edit in between, deleting on the strength of the earlier read + // destroys the newer content with no trace. So this CID is persisted on the + // ledger row at the postv2 write, re-checked against a FRESH read + // immediately before the delete, and passed to the PDS as the delete's swap + // guard so the PDS itself refuses a stale delete. CID string // CommunityDID is the repo the old record lived in — the community whose @@ -129,8 +157,8 @@ type LegacyPost struct { // DEPRECATED, LOSSY: PostRecord omits published fields — langs, tags, // crosspostOf, crosspostChain, bridgedStats — so converting through it before // deleting the old record IRREVERSIBLY drops them (whole-branch review, P5). - // The conversion must run off RawRecord instead; this field is retained only - // until GREEN removes the lossy path. + // The conversion runs off RawRecord instead; this field is carried only + // because callers already populate it and the author DID is read from it. Record PostRecord // RawRecord is the legacy record EXACTLY as it stands in the community repo — @@ -142,17 +170,35 @@ type LegacyPost struct { RawRecord map[string]any } -// LegacySource enumerates and deletes the deprecated community.post records. +// LegacySource enumerates, re-reads and deletes the deprecated community.post +// records. // // It is a seam so the T1 state machine runs against an in-memory source while // the T2 contract and production run it against real community repos on the PDS -// (listRecords over social.coves.community.post, delete via the community's own -// credentials). The DELETE is idempotent by contract — a delete of an -// already-gone record reports success — because it is the resumed step a crash -// after the migrated checkpoint retries. +// (listRecords over social.coves.community.post, getRecord for the pre-delete +// re-read, delete via the community's own credentials). type LegacySource interface { + // ListLegacyPosts enumerates every legacy record in scope. The bodies it + // returns are a SNAPSHOT: by the time a given record is processed they may be + // stale, which is why ReadLegacyPost exists. ListLegacyPosts(ctx context.Context) ([]LegacyPost, error) - DeleteLegacyPost(ctx context.Context, legacy LegacyPost) error + + // ReadLegacyPost re-reads ONE record as it stands right now. found is false + // when the record is gone — which is a legitimate outcome on a resumed run + // whose delete already landed, and a fatal surprise on a fresh one. + ReadLegacyPost(ctx context.Context, uri string) (post LegacyPost, found bool, err error) + + // DeleteLegacyPost removes the old record, GUARDED by swapCID: the PDS must + // refuse the delete if the record no longer carries that exact CID, so a + // concurrent edit cannot be destroyed even if every check above it somehow + // passed. An empty swapCID is not permitted and implementations must refuse + // it — an unguarded delete is the failure mode this parameter exists to make + // unrepresentable. + // + // The delete is idempotent by contract — a delete of an already-gone record + // reports success — because it is the step a crash after the migrated + // checkpoint retries. + DeleteLegacyPost(ctx context.Context, legacy LegacyPost, swapCID string) error } // RematerializeLedgerRow is one row of the migration-037 ledger. @@ -161,6 +207,17 @@ type RematerializeLedgerRow struct { State RematerializeState AuthorDID string + // CommunityDID is the repo the legacy record lives in. It is stored rather + // than parsed back out of OldURI because the whole destructive half of the + // tool is scoped by it: a staged run for one community must not resume, and + // must not DELETE, a row belonging to another. + CommunityDID string + + // SourceCID is the legacy record's CID as read at the moment the postv2 was + // built from it. The delete is refused unless a fresh read still reports this + // exact CID, and it is the swap guard the delete is sent under. + SourceCID string + // NewURI, NewCID, NewRkey identify the postv2 the tool wrote. Populated at // the postv2_written transition and never recomputed on resume — the resumed // run reads them back rather than deriving a fresh CID. @@ -178,26 +235,37 @@ type RematerializeLedgerRow struct { // RematerializeLedger is the migration-037 Postgres table, behind an interface // so the state-machine tests drive a real ledger while the source and repos are // faked. +// +// Every method that names a community takes it as an explicit scope rather than +// reading an ambient filter, because a scope that can be forgotten at one call +// site is a scope that lets a staged run delete another community's posts. type RematerializeLedger interface { // Discover upserts the row for oldURI in state discovered, idempotently: a // re-run finds the existing row (whatever state it is in) rather than // resetting it. - Discover(ctx context.Context, oldURI, authorDID string) (RematerializeLedgerRow, error) + Discover(ctx context.Context, oldURI, communityDID, authorDID string) (RematerializeLedgerRow, error) // Get reads one row. found is false when the URI has never been discovered. Get(ctx context.Context, oldURI string) (row RematerializeLedgerRow, found bool, err error) // ListResumable returns every row in a non-terminal state (not done, not a - // fallback). It is what makes crash-resume drive off the LEDGER rather than - // the source listing (whole-branch review, P7): a record whose delete - // succeeded but whose MarkDone crashed is GONE from the community repo, so a - // re-run's listRecords can never rediscover it — only the ledger row proves it - // is owed a final MarkDone. - ListResumable(ctx context.Context) ([]RematerializeLedgerRow, error) + // fallback), restricted to communityDID when it is non-empty. + // + // It is what makes crash-resume drive off the LEDGER rather than the source + // listing (whole-branch review, P7): a record whose delete succeeded but whose + // MarkDone crashed is GONE from the community repo, so a re-run's listRecords + // can never rediscover it — only the ledger row proves it is owed a final + // MarkDone. + // + // THE SCOPE IS NOT COSMETIC. Unscoped, a staged run for community A resumes — + // and deletes — rows belonging to community B, which is the one thing a staged + // rollout exists to prevent. + ListResumable(ctx context.Context, communityDID string) ([]RematerializeLedgerRow, error) - // RecordPostV2Written moves discovered → postv2_written and records the - // postv2 coordinates. - RecordPostV2Written(ctx context.Context, oldURI, newURI, newCID, newRkey string) error + // RecordPostV2Written moves discovered → postv2_written, recording both the + // postv2 coordinates and the SOURCE CID the postv2 was built from — the value + // the pre-delete re-read is checked against on every later pass. + RecordPostV2Written(ctx context.Context, oldURI, sourceCID, newURI, newCID, newRkey string) error // MarkVerified moves postv2_written → verified. MarkVerified(ctx context.Context, oldURI string) error @@ -211,23 +279,85 @@ type RematerializeLedger interface { // MarkFallback moves the row to a terminal fallback state with a reason. MarkFallback(ctx context.Context, oldURI string, state RematerializeState, reason string) error - // CountByState is the census: how many rows sit in each state, so the run - // can refuse "complete" while any fallback survives. - CountByState(ctx context.Context) (map[RematerializeState]int, error) + // ReopenFallback moves fallback rows back to discovered so a later run can + // retry them, restricted to communityDID when it is non-empty, and returns how + // many rows moved. + // + // THIS IS THE RECOVERY PATH, and it exists because the fallback state is + // otherwise a one-way door: an author whose grant was missing at census time + // can be re-authorized, and without this the operator's only remedy is + // hand-written UPDATE statements against a production table at 2am. It only + // ever moves a row from a fallback state to discovered — it cannot resurrect a + // done row, and it writes nothing to any repo. + ReopenFallback(ctx context.Context, communityDID string) (int, error) + + // CountByState is the census: how many rows sit in each state, restricted to + // communityDID when it is non-empty, so a run can report on its own scope and + // on the whole migration separately. + CountByState(ctx context.Context, communityDID string) (map[RematerializeState]int, error) } // RematerializeReport is the census a run returns. +// +// IT CARRIES TWO DIFFERENT COMPLETION SIGNALS because they answer two different +// questions and conflating them trains the operator to ignore both. A staged +// `-community` run can finish everything it was asked to do (ScopeComplete) +// while the migration as a whole still has thousands of posts to go (Complete). +// Reporting only the second makes every staged run look like a failure, and an +// operator who has learned to ignore a red exit code is an operator with no gate +// on §11 step 6 at all. type RematerializeReport struct { + // CommunityScope is the community DID this run was restricted to, or "" for + // every hosted community. + CommunityScope string + + // Discovered, Done and Fallbacks describe THIS RUN'S SCOPE. Discovered int Done int Fallbacks int ByState map[RematerializeState]int - // Complete is false while any fallback row survives — the gate on the - // separate, manual legacy-removal follow-up (§11 step 6). + // RemainingLegacy is how many legacy records a FINAL RE-SCAN of the source + // still saw that are not accounted for by a fallback row. It is what turns + // "the ledger says we are done" into "the source agrees" — a ledger-only + // completion check cannot see a record written after the run began, or one + // the discovery pass never listed. + RemainingLegacy int + + // ScopeComplete: every row in this run's scope reached done AND the final + // re-scan found nothing left in scope. + ScopeComplete bool + + // GlobalByState, GlobalDiscovered, GlobalDone and GlobalFallbacks are the + // UNSCOPED census — the whole migration, regardless of what this run touched. + GlobalByState map[RematerializeState]int + GlobalDiscovered int + GlobalDone int + GlobalFallbacks int + + // Complete is the gate on the separate, manual and IRREVERSIBLE legacy-removal + // follow-up (§11 step 6). It requires the whole migration — not this run's + // scope — to have reached done, with no fallback surviving and nothing left in + // the source. Complete bool } +// RematerializeProgress is one observable transition, handed to the caller's +// Progress hook so a batch run says what it is doing while it does it. +// +// It exists because a tool that deletes production data in silence for an hour +// gives the operator exactly one bit of information — the exit code — and no way +// to tell a hung run from a slow one, or to answer "how far did it get?" after a +// crash. +type RematerializeProgress struct { + OldURI string + From RematerializeState + To RematerializeState + Index int + Total int + Note string +} + // Rematerializer drives the cutover. // // It holds a CommunityRecordWriter — the DIRECT acceptance writer — and NOT an @@ -240,6 +370,73 @@ type Rematerializer struct { Ledger RematerializeLedger AuthorRepos AuthorRepoFactory Acceptances CommunityRecordWriter + + // CommunityRepos opens the COMMUNITY's repo for READING — specifically to + // read the acceptance back after writing it. + // + // It is required, not optional. Without it the acceptance leg of "verify BOTH + // records" cannot happen at all, and an acceptance that was never read back is + // an acceptance that might not stand: the writer's own result cannot testify + // to it, because the writer computes that result from the same inputs it was + // handed. + CommunityRepos CommunityRepoFactory + + // Blobs fetches and probes blob bytes. Defaulted to a bounded HTTP client; + // injectable so the failure modes that matter (a truncated body, a probe that + // errors rather than 404s) can be exercised without a PDS. + Blobs RematerializeBlobClient + + // CommunityScope restricts the run — discovery, resume, delete and census — to + // one community DID. Empty means every hosted community. + CommunityScope string + + // Progress, when set, is called on every state transition and on notable + // non-transitions. It must not block for long: the run is serial. + Progress func(RematerializeProgress) + + // PerRecordTimeout bounds everything ONE record's processing does — several + // PDS round trips and a handful of ledger writes. Zero means no per-record + // bound, which is only appropriate in tests: in production a single half-open + // socket otherwise stalls the whole migration silently. + PerRecordTimeout time.Duration + + // AbortOnFallback stops the run after the credential census if that census + // marked any NEW row as a fallback, before a single repo is mutated. + // + // It is the operator's answer to the failure this tool's worst day looks + // like: one aggregator session expires, the census sentences most of the + // corpus, and a run that "succeeded" has migrated almost nothing. With this + // set, the run stops and names the authors instead. + AbortOnFallback bool + + // credentials caches ONE resolution per distinct author DID for the lifetime + // of this Rematerializer. Each resolution is a refresh-token rotation against + // the PDS; an aggregator with 5,000 posts would otherwise rotate 5,000 times + // in a single run, which is both minutes of avoidable load and thousands of + // extra chances to hit the transient failure that used to be recorded as a + // terminal verdict. + credentialsMu sync.Mutex + repoCache map[string]AuthorRepo + repoErrCache map[string]error +} + +// RematerializeBlobClient reads blob bytes out of a repo, and reports whether a +// repo serves one. +// +// It is an interface rather than two package functions because both of its +// failure modes are silent by default and both destroy data: a fetch that +// truncates at a size cap uploads DIFFERENT bytes under a DIFFERENT CID, and a +// probe that reports "absent" for a transport error turns a network blip into a +// refusal — or, if the polarity were ever flipped, a missing blob into a +// go-ahead to delete the only copy. +type RematerializeBlobClient interface { + // Fetch returns the blob's bytes. It MUST fail rather than truncate. + Fetch(ctx context.Context, host, did, cid string) ([]byte, error) + + // Present reports whether the repo serves the blob. A transport failure is an + // ERROR, never a false: "I could not ask" and "it is not there" are different + // facts and only one of them is about the data. + Present(ctx context.Context, host, did, cid string) (bool, error) } // RematerializeRkey is the postv2 record key the tool writes a legacy record at. @@ -263,6 +460,11 @@ type Rematerializer struct { // widths a TID has room for, and encoded with syntax.NewTID — the one encoder // guaranteed to agree with every ParseTID in the network. Same URI in, same TID // out; different URIs, different TIDs (SHA-256 collision resistance). +// +// THE DERIVATION IS PINNED BY GOLDEN VALUES in rematerialize_rkey_test.go, and +// those literals are not a test to update: every post already migrated sits at +// the OLD key, so changing this function writes a SECOND postv2 for every one of +// them on the next run. func RematerializeRkey(legacyPostURI string) string { digest := sha256.Sum256([]byte(legacyPostURI)) @@ -280,22 +482,28 @@ func RematerializeRkey(legacyPostURI string) string { return syntax.NewTID(micros, clockID).String() } -// Run discovers every legacy record and drives each to a terminal state, -// returning the census. +// Run discovers every legacy record in scope and drives each to a terminal +// state, returning the census. // -// It runs in THREE ordered passes: +// It runs in FOUR ordered passes: // // 1. THE CREDENTIAL CENSUS (P8). Every discovered author is resolved with NO // repo mutation, so an author whose credentials cannot be restored is marked // a fallback BEFORE a single record is written or deleted. Without this, a // mutate-as-you-go run would fully migrate and delete the early records // before ever discovering that a later author is stranded — the exact -// ordering §11 step 3 forbids. +// ordering §11 step 3 forbids. A RETRYABLE credential failure fails the run +// here rather than sentencing a row. // 2. THE SOURCE PASS. Each listed record is driven to a terminal state. // 3. THE LEDGER RECONCILE (P7). A row whose delete succeeded but whose MarkDone // crashed is GONE from the community repo, so the source's listRecords can // never rediscover it — only ListResumable can. Every non-terminal ledger -// row past the postv2 write is finished from the ledger. +// row past the postv2 write is finished from the ledger, IN SCOPE. +// 4. THE FINAL RE-SCAN. The source is enumerated again and anything still +// standing that is not accounted for by a fallback row is counted. A +// completion signal computed only from rows the run already knew about +// cannot see a record written during the run, or one the first listing +// missed — and "complete" is the gate on an irreversible step. // // A per-record error FAILS THE RUN rather than being logged and skipped: the // safety properties are all ordering ones, and continuing past a record the tool @@ -314,63 +522,122 @@ func (r *Rematerializer) Run(ctx context.Context) (RematerializeReport, error) { // earlier pass and must not be re-marked — the fallback transition is guarded on // the discovered state, so re-marking a resumed or already-fallen-back row would // fail; skipping it keeps the whole run idempotent. - for _, legacy := range legacies { - row, err := r.Ledger.Discover(ctx, legacy.URI, legacy.AuthorDID) + var stranded []string + for i, legacy := range legacies { + if err := ctx.Err(); err != nil { + return RematerializeReport{}, err + } + row, err := r.Ledger.Discover(ctx, legacy.URI, legacy.CommunityDID, legacy.AuthorDID) if err != nil { return RematerializeReport{}, err } if row.State != RematerializeDiscovered { continue } - if _, err := r.AuthorRepos(ctx, legacy.AuthorDID, nil); err != nil { + if _, err := r.authorRepo(ctx, legacy.AuthorDID); err != nil { if errors.Is(err, ErrNoAuthorCredentials) { reason := fmt.Sprintf("author %s has no restorable repo credentials: %v", legacy.AuthorDID, err) if markErr := r.Ledger.MarkFallback(ctx, legacy.URI, RematerializeFallbackLeftLegacy, reason); markErr != nil { return RematerializeReport{}, markErr } + stranded = append(stranded, legacy.AuthorDID) + r.report(RematerializeProgress{ + OldURI: legacy.URI, From: RematerializeDiscovered, To: RematerializeFallbackLeftLegacy, + Index: i + 1, Total: len(legacies), Note: reason, + }) continue } + // RETRYABLE. Failing the run is the WHOLE POINT: writing this as a + // fallback would sentence the row on the strength of a network blip, and + // nothing inside a later run would move it back. return RematerializeReport{}, fmt.Errorf("preflighting the credentials of %s: %w", legacy.AuthorDID, err) } } + if r.AbortOnFallback && len(stranded) > 0 { + return RematerializeReport{}, fmt.Errorf( + "the credential census left %d post(s) as legacy across %d author(s) (%s); stopping before any repo was mutated. "+ + "Re-authorize the author(s) and re-run with -reopen-fallbacks, or re-run with -accept-fallbacks to proceed and leave those posts as legacy", + len(stranded), len(distinct(stranded)), joinFirst(distinct(stranded), 5)) + } + // Pass 2 — the source pass. A record whose census marked it a fallback is left // untouched by RematerializeOne (it returns early on a terminal row). - for _, legacy := range legacies { - if _, err := r.RematerializeOne(ctx, legacy); err != nil { + for i, legacy := range legacies { + if err := ctx.Err(); err != nil { + return RematerializeReport{}, err + } + state, err := r.rematerializeOneBounded(ctx, legacy) + if err != nil { return RematerializeReport{}, fmt.Errorf("re-materializing %s: %w", legacy.URI, err) } + r.report(RematerializeProgress{OldURI: legacy.URI, To: state, Index: i + 1, Total: len(legacies)}) } - // Pass 3 — the ledger reconcile. Finish any row the source could not present. - resumable, err := r.Ledger.ListResumable(ctx) + // Pass 3 — the ledger reconcile, IN SCOPE. Finish any row the source could not + // present. Scoped, because an unscoped resume would drive — and delete — rows + // belonging to a community this staged run was told to leave alone. + resumable, err := r.Ledger.ListResumable(ctx, r.CommunityScope) if err != nil { return RematerializeReport{}, fmt.Errorf("listing resumable rows: %w", err) } - for _, ledgerRow := range resumable { + for i, ledgerRow := range resumable { + if err := ctx.Err(); err != nil { + return RematerializeReport{}, err + } // A row still at discovered needs the ORIGINAL record's bytes to build its - // postv2, which the ledger does not hold — only a source listing carries - // them. Such a row is genuinely incomplete and keeps Complete false; it is - // left for a pass whose source can present it, not driven off an empty body. + // postv2. RematerializeOne re-reads them from the source itself, so a row + // whose record still stands is finished here; one whose record is gone is + // genuinely incomplete and keeps Complete false. if ledgerRow.State == RematerializeDiscovered { continue } - legacy, err := legacyFromLedgerRow(ledgerRow) + legacy := legacyFromLedgerRow(ledgerRow) + state, err := r.rematerializeOneBounded(ctx, legacy) if err != nil { - return RematerializeReport{}, err - } - if _, err := r.RematerializeOne(ctx, legacy); err != nil { return RematerializeReport{}, fmt.Errorf("reconciling %s: %w", ledgerRow.OldURI, err) } + r.report(RematerializeProgress{OldURI: ledgerRow.OldURI, From: ledgerRow.State, To: state, Index: i + 1, Total: len(resumable), Note: "reconciled from the ledger"}) } - byState, err := r.Ledger.CountByState(ctx) + return r.census(ctx) +} + +// rematerializeOneBounded runs one record under PerRecordTimeout, so a single +// stalled PDS call cannot hang the whole migration with no output. The bound is +// per RECORD rather than per call because the safety properties are ordering +// ones: a timeout mid-record leaves the ledger at its last checkpoint, which a +// re-run resumes from. +func (r *Rematerializer) rematerializeOneBounded(ctx context.Context, legacy LegacyPost) (RematerializeState, error) { + if r.PerRecordTimeout <= 0 { + return r.RematerializeOne(ctx, legacy) + } + recordCtx, cancel := context.WithTimeout(ctx, r.PerRecordTimeout) + defer cancel() + return r.RematerializeOne(recordCtx, legacy) +} + +// census builds the report: the scoped tally, the global tally, and a FINAL +// RE-SCAN of the source that the completion signals are gated on. +func (r *Rematerializer) census(ctx context.Context) (RematerializeReport, error) { + scoped, err := r.Ledger.CountByState(ctx, r.CommunityScope) if err != nil { return RematerializeReport{}, fmt.Errorf("taking the census: %w", err) } + global := scoped + if r.CommunityScope != "" { + global, err = r.Ledger.CountByState(ctx, "") + if err != nil { + return RematerializeReport{}, fmt.Errorf("taking the global census: %w", err) + } + } - report := RematerializeReport{ByState: byState} - for state, n := range byState { + report := RematerializeReport{ + CommunityScope: r.CommunityScope, + ByState: scoped, + GlobalByState: global, + } + for state, n := range scoped { report.Discovered += n if state == RematerializeDone { report.Done += n @@ -379,12 +646,46 @@ func (r *Rematerializer) Run(ctx context.Context) (RematerializeReport, error) { report.Fallbacks += n } } - // COMPLETE MEANS EVERY ROW REACHED done — not merely that no fallback survives - // (P7). A row stranded in any non-terminal state, or a surviving fallback, both - // leave Done < Discovered, and the operator's irreversible legacy-removal step - // (§11 step 6) must not run while either is true. - report.Complete = report.Done == report.Discovered + for state, n := range global { + report.GlobalDiscovered += n + if state == RematerializeDone { + report.GlobalDone += n + } + if IsFallback(state) { + report.GlobalFallbacks += n + } + } + // THE FINAL RE-SCAN. Completion computed only from ledger rows the run already + // discovered is circular: it can never see a record that was written after the + // discovery pass, or one the listing missed. Asking the source again is the only + // evidence that the collection is actually drained. + remaining, err := r.Source.ListLegacyPosts(ctx) + if err != nil { + return report, fmt.Errorf("re-scanning the source to confirm the drain: %w", err) + } + for _, legacy := range remaining { + row, found, err := r.Ledger.Get(ctx, legacy.URI) + if err != nil { + return report, fmt.Errorf("checking the ledger for the re-scanned %s: %w", legacy.URI, err) + } + // A record deliberately left as legacy is accounted for, not remaining. A + // record with no row at all, or one not yet done, is remaining. + if found && IsFallback(row.State) { + continue + } + report.RemainingLegacy++ + } + + report.ScopeComplete = report.Done == report.Discovered && report.RemainingLegacy == 0 + // COMPLETE MEANS THE WHOLE MIGRATION IS DRAINED — every row everywhere at done, + // no fallback surviving, and the source re-scan agreeing. A row stranded in any + // non-terminal state, a surviving fallback, or a legacy record still standing all + // leave it false, and the operator's irreversible legacy-removal step (§11 step 6) + // must not run while any of them is true. + report.Complete = report.GlobalDone == report.GlobalDiscovered && + report.GlobalFallbacks == 0 && + report.RemainingLegacy == 0 return report, nil } @@ -393,13 +694,21 @@ func (r *Rematerializer) Run(ctx context.Context) (RematerializeReport, error) { // // The steps are guarded on the ledger state each moves FROM, so a resumed run // re-enters at exactly the step its predecessor stopped before and re-does none -// of the completed ones. The load-bearing ordering is VERIFY BEFORE DELETE: the -// old record is deleted only after the postv2, its embed blobs, and its -// acceptance are all confirmed present and consistent, and the migrated -// checkpoint is persisted BEFORE the delete so a crash there retries only the -// delete. +// of the completed ones — WITH ONE DELIBERATE EXCEPTION. The verification that +// licenses the delete is re-run from fresh reads on EVERY path, resumed or not, +// because the ledger records that verification passed at a moment now in the +// past and the delete needs it to be true now. func (r *Rematerializer) RematerializeOne(ctx context.Context, legacy LegacyPost) (RematerializeState, error) { - row, err := r.Ledger.Discover(ctx, legacy.URI, legacy.AuthorDID) + if r.CommunityScope != "" && legacy.CommunityDID != r.CommunityScope { + // A SCOPED RUN NEVER TOUCHES ANOTHER COMMUNITY'S POSTS. This is the last + // gate before a delete, and it is checked here rather than only at discovery + // because the reconcile pass reaches records the discovery pass never listed. + return "", fmt.Errorf( + "refusing to re-materialize %s: it belongs to %s but this run is scoped to %s", + legacy.URI, legacy.CommunityDID, r.CommunityScope) + } + + row, err := r.Ledger.Discover(ctx, legacy.URI, legacy.CommunityDID, legacy.AuthorDID) if err != nil { return "", err } @@ -407,141 +716,73 @@ func (r *Rematerializer) RematerializeOne(ctx context.Context, legacy LegacyPost // A row already in a terminal fallback state is left exactly as it stands: the // credential census reached its verdict on a prior pass, and re-opening it // would be the one thing §11 step 3 forbids — a second chance to forge. + // (ReopenFallback is the supported, explicit way back.) if IsFallback(row.State) { return row.State, nil } + if row.State == RematerializeDone { + return row.State, nil + } // Step 1 — postv2_written. Copy the embed blobs into the author's repo, build - // the postv2 LOSSLESSLY from the raw legacy record, and write it at the - // deterministic rkey. createAuthorRecord is create-only and converges by read, - // so a resume that re-enters here finds its own first attempt rather than - // minting a second post. + // the postv2 LOSSLESSLY from the raw legacy record AS IT STANDS NOW, and write + // it at the deterministic rkey. createAuthorRecord is create-only and converges + // by read, so a resume that re-enters here finds its own first attempt rather + // than minting a second post. if row.State == RematerializeDiscovered { - repo, err := r.AuthorRepos(ctx, legacy.AuthorDID, nil) - if err != nil { - // NO CREDENTIALS IS A TERMINAL FALLBACK, NEVER A FORGERY. An author whose - // repo cannot be restored is left as legacy — the postv2 is not written and - // the old record survives — because re-authoring under any other identity - // reintroduces the §2 impersonation the whole flip removes. - if errors.Is(err, ErrNoAuthorCredentials) { - reason := fmt.Sprintf("author %s has no restorable repo credentials: %v", legacy.AuthorDID, err) - if markErr := r.Ledger.MarkFallback(ctx, legacy.URI, RematerializeFallbackLeftLegacy, reason); markErr != nil { - return row.State, markErr - } - return RematerializeFallbackLeftLegacy, nil - } - return row.State, fmt.Errorf("opening the author repo of %s: %w", legacy.AuthorDID, err) - } - - // P5 — the conversion is built from the LOSSLESS raw record, dropping only - // the author field and re-stamping $type. Building it through PostRecord - // would silently strip langs/tags/crosspostOf/crosspostChain/bridgedStats, - // which the old record can never be recovered from once it is deleted. - intended, err := postV2Body(legacy) + newRow, err := r.writePostV2(ctx, legacy, row) if err != nil { return row.State, err } - - // P4 — the embed's blob BYTES must live in the AUTHOR's repo before the old - // record (and the community's blob store) can go, or the postv2's media - // resolves against a repo that never held it. The bytes are UPLOADED here, - // before the record that references them is written; the PDS only serves an - // uploaded blob once a record pins it, so presence is VERIFIED after the - // write, below. - if err := r.uploadEmbedBlobs(ctx, repo, legacy); err != nil { - return row.State, err + row = newRow + r.report(RematerializeProgress{OldURI: legacy.URI, From: RematerializeDiscovered, To: row.State}) + if IsFallback(row.State) { + return row.State, nil } - - rkey := RematerializeRkey(legacy.URI) - newURI, newCID, converged, err := createAuthorRecord(ctx, repo, rkey, intended) - if err != nil { - return row.State, fmt.Errorf("writing the postv2 for %s: %w", legacy.URI, err) - } - - // P6 — a converged write means a record ALREADY stood at the deterministic - // rkey. createAuthorRecord accepts it by CID, but a CID match alone would - // adopt a DIFFERENT record that merely shares the key. Confirm the standing - // record IS this legacy post's conversion before trusting it; otherwise the - // legacy original would be deleted in favour of a foreign record. - if converged { - standing, err := repo.GetRecord(ctx, PostV2Collection, rkey) - if err != nil { - return row.State, fmt.Errorf("reading the converged postv2 for %s: %w", legacy.URI, err) - } - if !sameRecordBody(standing.Value, intended) { - return row.State, fmt.Errorf( - "a different record already stands at %s in the author's repo: its body is not this legacy post's conversion, so re-materializing would adopt a foreign record and delete the real one", - rkey) - } - } - - // P4 — now the postv2 pins the blobs, so the author repo actually serves - // them. Confirm every embed blob is present BEFORE recording the postv2 (well - // before the migrated checkpoint): a blob the author repo does not serve is a - // broken image the moment the community's copy is garbage-collected. - if err := r.verifyEmbedBlobsPresent(ctx, repo, legacy); err != nil { - return row.State, err - } - - if err := r.Ledger.RecordPostV2Written(ctx, legacy.URI, newURI, newCID, rkey); err != nil { - return row.State, err - } - row.State = RematerializePostV2Written - row.NewURI, row.NewCID, row.NewRkey = newURI, newCID, rkey } - // Step 2 — verified. Write the community's acceptance DIRECT (never through the - // engine — see the type's doc) pinning the NEW postv2 CID, confirm the - // acceptance actually stands against OUR subject, then RE-READ the postv2 and - // confirm it still pins that CID. Both reads happen before anything is deleted. + // Step 2 — write the community's acceptance DIRECT (never through the engine — + // see the type's doc) pinning the NEW postv2 CID. + // + // NOTE WHAT DOES *NOT* HAPPEN HERE: the row is not checkpointed to `verified`. + // The writer's own result cannot testify that the record stands — it is + // computed from the inputs it was handed — so a checkpoint written here would + // mean "verified" on the strength of nothing. The state advances only after + // the read-back below. if row.State == RematerializePostV2Written { - res, err := r.Acceptances.WriteAcceptance(ctx, CommunityWriteCommand{ + if _, err := r.Acceptances.WriteAcceptance(ctx, CommunityWriteCommand{ CommunityDID: legacy.CommunityDID, PostURI: row.NewURI, PostCID: row.NewCID, - }) - if err != nil { + }); err != nil { return row.State, fmt.Errorf("writing the acceptance for %s: %w", row.NewURI, err) } + } - // P6 — verify the acceptance's subject strongRef. Its record key is the - // digest of the subject URI, so a matching rkey proves the acceptance is FOR - // our postv2; an empty CID would mean no record stands at all. A write that - // returned neither has not made the community's acceptance real, and deleting - // the legacy record on the strength of it would drop the post out of its - // community. - if res.CID == "" || res.RKey != SubjectRkey(row.NewURI) { - return row.State, fmt.Errorf( - "the acceptance for %s did not stand against the expected subject (got rkey %q cid %q, want rkey %q)", - row.NewURI, res.RKey, res.CID, SubjectRkey(row.NewURI)) - } - - repo, err := r.AuthorRepos(ctx, legacy.AuthorDID, nil) - if err != nil { - return row.State, fmt.Errorf("re-opening the author repo of %s to verify: %w", legacy.AuthorDID, err) - } - standing, err := repo.GetRecord(ctx, PostV2Collection, row.NewRkey) - if err != nil { - return row.State, fmt.Errorf("verifying the postv2 for %s: %w", row.NewURI, err) - } - if standing.CID != row.NewCID { - // VERIFY BEFORE DELETE fails closed: the acceptance now pins a CID the - // postv2 no longer carries, so deleting the old record would destroy the - // only copy of a post whose new attestation points at content that no - // longer stands. No checkpoint, no delete — the row stays at - // postv2_written for a later pass to re-verify. - return row.State, fmt.Errorf( - "verifying the postv2 for %s: the standing record pins %s but the acceptance pinned %s (a concurrent edit landed mid-verify)", - row.NewURI, standing.CID, row.NewCID) - } + // THE PRE-DELETE VERIFICATION. Unconditional, on every path into the delete, + // from FRESH READS ONLY. + // + // This is the load-bearing ordering of the whole tool, and it is deliberately + // NOT gated on the ledger state: a row at verified or migrated says only that + // the check passed once, before a crash whose duration nothing here knows. In + // that gap the postv2 could have been edited or deleted, the acceptance + // withdrawn, or the legacy record itself edited by a writer the maintenance + // window failed to stop. A ledger memory is not evidence; a read is. + verified, err := r.verifyBeforeDelete(ctx, legacy, row) + if err != nil { + return row.State, err + } + // Step 3 — verified. Recorded ONLY now, after the reads that make it true. + if row.State == RematerializePostV2Written { if err := r.Ledger.MarkVerified(ctx, legacy.URI); err != nil { return row.State, err } row.State = RematerializeVerified + r.report(RematerializeProgress{OldURI: legacy.URI, From: RematerializePostV2Written, To: RematerializeVerified}) } - // Step 3 — migrated. The checkpoint BEFORE the delete: postv2, blobs and + // Step 4 — migrated. The checkpoint BEFORE the delete: postv2, blobs and // acceptance verified, old record still present. Persisting it as its own state // is what lets a crash on the delete retry ONLY the delete. if row.State == RematerializeVerified { @@ -549,29 +790,305 @@ func (r *Rematerializer) RematerializeOne(ctx context.Context, legacy LegacyPost return row.State, err } row.State = RematerializeMigrated + r.report(RematerializeProgress{OldURI: legacy.URI, From: RematerializeVerified, To: RematerializeMigrated}) } - // Step 4 — done. Delete the old community.post; a delete of an already-gone - // record is success (the source's contract), so a resumed delete is idempotent. + // Step 5 — done. Delete the old community.post, GUARDED by the source CID the + // postv2 was built from, so the PDS refuses the delete if anything landed on + // the record since. A delete of an already-gone record is success (the source's + // contract), so a resumed delete is idempotent. if row.State == RematerializeMigrated { - if err := r.Source.DeleteLegacyPost(ctx, legacy); err != nil { - return row.State, fmt.Errorf("deleting the old record %s: %w", legacy.URI, err) + if verified.legacyPresent { + if err := r.Source.DeleteLegacyPost(ctx, legacy, row.SourceCID); err != nil { + return row.State, fmt.Errorf("deleting the old record %s: %w", legacy.URI, err) + } } if err := r.Ledger.MarkDone(ctx, legacy.URI); err != nil { return row.State, err } row.State = RematerializeDone + r.report(RematerializeProgress{OldURI: legacy.URI, From: RematerializeMigrated, To: RematerializeDone}) } return row.State, nil } +// writePostV2 is step 1: resolve the author, re-read the legacy record, copy its +// blobs, build the lossless conversion and write it at the deterministic rkey. +// +// IT RE-READS THE RECORD rather than trusting the listing it was handed. The +// listing snapshots every body at t0 and the pass that consumes it runs for +// minutes to hours; building the postv2 from a stale body and then deleting the +// original destroys whatever landed in between. +func (r *Rematerializer) writePostV2(ctx context.Context, legacy LegacyPost, row RematerializeLedgerRow) (RematerializeLedgerRow, error) { + repo, err := r.authorRepo(ctx, legacy.AuthorDID) + if err != nil { + // NO CREDENTIALS IS A TERMINAL FALLBACK, NEVER A FORGERY. An author whose + // repo cannot be restored is left as legacy — the postv2 is not written and + // the old record survives — because re-authoring under any other identity + // reintroduces the §2 impersonation the whole flip removes. + // + // A RETRYABLE failure is not that, and is not written to the ledger at all. + if errors.Is(err, ErrNoAuthorCredentials) { + reason := fmt.Sprintf("author %s has no restorable repo credentials: %v", legacy.AuthorDID, err) + if markErr := r.Ledger.MarkFallback(ctx, legacy.URI, RematerializeFallbackLeftLegacy, reason); markErr != nil { + return row, markErr + } + row.State = RematerializeFallbackLeftLegacy + return row, nil + } + return row, fmt.Errorf("opening the author repo of %s: %w", legacy.AuthorDID, err) + } + + fresh, found, err := r.Source.ReadLegacyPost(ctx, legacy.URI) + if err != nil { + return row, fmt.Errorf("re-reading the legacy record %s before converting it: %w", legacy.URI, err) + } + if !found { + // The row is at discovered — this tool has written nothing for it — and the + // record is gone. Something outside the tool deleted it. There is nothing to + // migrate and nothing safe to assume, so say so rather than inventing a body. + return row, fmt.Errorf( + "the legacy record %s is gone from its community repo but the ledger has never written a postv2 for it; "+ + "it was deleted by something other than this tool, so there is nothing to re-materialize", legacy.URI) + } + if fresh.CID == "" { + return row, fmt.Errorf("re-reading the legacy record %s: the community repo reported no CID, so no delete could ever be guarded on it", legacy.URI) + } + fresh.CommunityDID = legacy.CommunityDID + if fresh.AuthorDID == "" { + fresh.AuthorDID = legacy.AuthorDID + } + + // P5 — the conversion is built from the LOSSLESS raw record, dropping only + // the author field and re-stamping $type. Building it through PostRecord + // would silently strip langs/tags/crosspostOf/crosspostChain/bridgedStats, + // which the old record can never be recovered from once it is deleted. + intended, err := postV2Body(fresh) + if err != nil { + return row, err + } + + // P4 — the embed's blob BYTES must live in the AUTHOR's repo before the old + // record (and the community's blob store) can go, or the postv2's media + // resolves against a repo that never held it. The bytes are UPLOADED here, + // before the record that references them is written; the PDS only serves an + // uploaded blob once a record pins it, so presence is VERIFIED after the + // write, in verifyBeforeDelete. + if err := r.uploadEmbedBlobs(ctx, repo, fresh); err != nil { + return row, err + } + + rkey := RematerializeRkey(legacy.URI) + newURI, newCID, converged, err := createAuthorRecord(ctx, repo, rkey, intended) + if err != nil { + return row, fmt.Errorf("writing the postv2 for %s: %w", legacy.URI, err) + } + + // P6 — a converged write means a record ALREADY stood at the deterministic + // rkey. createAuthorRecord accepts it by CID, but a CID match alone would + // adopt a DIFFERENT record that merely shares the key. Confirm the standing + // record IS this legacy post's conversion before trusting it; otherwise the + // legacy original would be deleted in favour of a foreign record. + if converged { + standing, err := repo.GetRecord(ctx, PostV2Collection, rkey) + if err != nil { + return row, fmt.Errorf("reading the converged postv2 for %s: %w", legacy.URI, err) + } + if !sameRecordBody(standing.Value, intended) { + return row, fmt.Errorf( + "a different record already stands at %s in the author's repo: its body is not this legacy post's conversion, so re-materializing would adopt a foreign record and delete the real one", + rkey) + } + } + + if err := r.Ledger.RecordPostV2Written(ctx, legacy.URI, fresh.CID, newURI, newCID, rkey); err != nil { + return row, err + } + row.State = RematerializePostV2Written + row.SourceCID = fresh.CID + row.NewURI, row.NewCID, row.NewRkey = newURI, newCID, rkey + return row, nil +} + +// verificationResult reports what the pre-delete verification actually observed, +// so the delete step knows whether there is still a record to delete. +type verificationResult struct { + legacyPresent bool +} + +// verifyBeforeDelete is the guarantee. It runs immediately before every delete, +// on every path, and it reads everything it asserts. +// +// FOUR FACTS, ALL FROM FRESH READS: +// +// 1. The postv2 stands in the AUTHOR's repo and still carries the CID the +// acceptance pinned. +// 2. The ACCEPTANCE stands in the COMMUNITY's repo and its subject strongRef +// names our postv2 URI and that same CID. This is read back from the repo, +// never inferred from the writer's own result — the writer computes its +// result from the inputs it was handed, so comparing it to those inputs is a +// tautology that cannot fail and proves nothing about what the repo holds. +// 3. Every embed blob is served by the AUTHOR's repo, so the post's media does +// not break the instant the community's copy becomes collectable. +// 4. The LEGACY record either is gone already (an idempotent resumed delete) or +// still carries the exact CID the postv2 was built from. Anything else means +// a writer landed an edit the maintenance window did not stop, and the newer +// content must not be destroyed. +func (r *Rematerializer) verifyBeforeDelete(ctx context.Context, legacy LegacyPost, row RematerializeLedgerRow) (verificationResult, error) { + if row.NewURI == "" || row.NewCID == "" || row.NewRkey == "" { + return verificationResult{}, fmt.Errorf( + "refusing to verify %s for deletion: the ledger row names no postv2 (uri %q cid %q rkey %q), so there is nothing proven to replace it", + legacy.URI, row.NewURI, row.NewCID, row.NewRkey) + } + if row.SourceCID == "" { + return verificationResult{}, fmt.Errorf( + "refusing to verify %s for deletion: the ledger row records no source CID, so the delete could not be guarded against a concurrent edit", + legacy.URI) + } + + // (1) The postv2, read fresh out of the author's repo. + repo, err := r.authorRepo(ctx, legacy.AuthorDID) + if err != nil { + return verificationResult{}, fmt.Errorf("opening the author repo of %s to verify: %w", legacy.AuthorDID, err) + } + standing, err := repo.GetRecord(ctx, PostV2Collection, row.NewRkey) + if err != nil { + return verificationResult{}, fmt.Errorf("verifying the postv2 for %s: %w", row.NewURI, err) + } + if standing == nil { + return verificationResult{}, fmt.Errorf("verifying the postv2 for %s: the author's repo returned no record", row.NewURI) + } + if standing.CID != row.NewCID { + // VERIFY BEFORE DELETE fails closed: the acceptance pins a CID the postv2 no + // longer carries, so deleting the old record would destroy the only copy of a + // post whose new attestation points at content that no longer stands. No + // checkpoint, no delete. + return verificationResult{}, fmt.Errorf( + "verifying the postv2 for %s: the standing record pins %s but the acceptance pinned %s (a concurrent edit landed after the write)", + row.NewURI, standing.CID, row.NewCID) + } + + // (2) The acceptance, read fresh out of the COMMUNITY's repo. + if r.CommunityRepos == nil { + return verificationResult{}, fmt.Errorf( + "refusing to delete %s: no community-repo factory is wired, so the acceptance cannot be read back and 'verify BOTH records' is unsatisfiable", + legacy.URI) + } + communityRepo, err := r.CommunityRepos(ctx, legacy.CommunityDID) + if err != nil { + return verificationResult{}, fmt.Errorf("opening the community repo of %s to verify the acceptance: %w", legacy.CommunityDID, err) + } + acceptanceRkey := SubjectRkey(row.NewURI) + acceptance, err := communityRepo.GetRecord(ctx, AcceptanceCollection, acceptanceRkey) + if err != nil { + return verificationResult{}, fmt.Errorf( + "verifying the acceptance of %s at %s/%s: %w (the community's acceptance is what keeps the post IN the community; without it the postv2 stands orphaned)", + row.NewURI, AcceptanceCollection, acceptanceRkey, err) + } + if acceptance == nil { + return verificationResult{}, fmt.Errorf("verifying the acceptance of %s: the community's repo returned no record", row.NewURI) + } + subjectURI, subjectCID := acceptanceSubject(acceptance.Value) + if subjectURI != row.NewURI || subjectCID != row.NewCID { + return verificationResult{}, fmt.Errorf( + "the acceptance standing at %s/%s does not pin our postv2: its subject is uri %q cid %q, we need uri %q cid %q. "+ + "Deleting the legacy record now would drop the post out of its community", + AcceptanceCollection, acceptanceRkey, subjectURI, subjectCID, row.NewURI, row.NewCID) + } + + // (4) The legacy record, read fresh — done before the blob check so the blob + // refs can come from the body that actually stands. + fresh, legacyPresent, err := r.Source.ReadLegacyPost(ctx, legacy.URI) + if err != nil { + return verificationResult{}, fmt.Errorf("re-reading the legacy record %s before deleting it: %w", legacy.URI, err) + } + if legacyPresent && fresh.CID != row.SourceCID { + return verificationResult{}, fmt.Errorf( + "refusing to delete %s: it now carries CID %s but the postv2 was built from %s. "+ + "An edit landed after the conversion, so deleting would destroy content that was never re-materialized. "+ + "Re-run after the writer is stopped; the ledger row stays where it is", + legacy.URI, fresh.CID, row.SourceCID) + } + + // (3) The blobs, in the AUTHOR's repo. The refs come from the standing legacy + // body when there is one, and from the postv2 itself when the legacy record has + // already gone — a resumed run must still prove the media is there. + blobSource := standing.Value + if legacyPresent { + blobSource = fresh.RawRecord + } + if err := r.verifyEmbedBlobsPresent(ctx, repo, legacy.URI, blobSource); err != nil { + return verificationResult{}, err + } + + return verificationResult{legacyPresent: legacyPresent}, nil +} + +// authorRepo resolves ONE author's repo, caching both the repo and a terminal +// failure for the lifetime of the run. +// +// Caching is not an optimisation here so much as a correctness property: every +// resolution rotates the stored refresh token, so resolving per POST rather than +// per AUTHOR means an aggregator with 5,000 posts performs 5,000 rotations in +// one run — each one a chance to break the session the AppView itself depends +// on. A RETRYABLE failure is deliberately NOT cached: it is the caller's job to +// fail the run on it, and a later run must be free to succeed. +func (r *Rematerializer) authorRepo(ctx context.Context, authorDID string) (AuthorRepo, error) { + r.credentialsMu.Lock() + defer r.credentialsMu.Unlock() + + if repo, ok := r.repoCache[authorDID]; ok { + return repo, nil + } + if err, ok := r.repoErrCache[authorDID]; ok { + return nil, err + } + + repo, err := r.AuthorRepos(ctx, authorDID, nil) + if err != nil { + if errors.Is(err, ErrNoAuthorCredentials) { + if r.repoErrCache == nil { + r.repoErrCache = map[string]error{} + } + r.repoErrCache[authorDID] = err + } + return nil, err + } + if r.repoCache == nil { + r.repoCache = map[string]AuthorRepo{} + } + r.repoCache[authorDID] = repo + return repo, nil +} + +// report hands one transition to the caller's Progress hook, if it wired one. +func (r *Rematerializer) report(p RematerializeProgress) { + if r.Progress != nil { + r.Progress(p) + } +} + +// blobClient returns the injected blob client, or the bounded HTTP default. +func (r *Rematerializer) blobClient() RematerializeBlobClient { + if r.Blobs != nil { + return r.Blobs + } + return DefaultRematerializeBlobClient() +} + // maxRematerializeBlobBytes caps a single blob copy. It is generous — larger than // any post media the lexicons admit — because the bytes come from our own // community repo, not an untrusted origin; the cap exists to bound a corrupt or // runaway response, not to enforce a content policy. const maxRematerializeBlobBytes = 100 << 20 // 100 MiB +// rematerializeBlobFetchTimeout bounds a single blob transfer. +// +// http.DefaultClient has NO timeout, so a half-open socket to the PDS hangs the +// whole run forever — at 3am, with no output, indistinguishable from a slow one. +const rematerializeBlobFetchTimeout = 2 * time.Minute + // postV2Body builds the author-owned postv2 record from the legacy record's // LOSSLESS raw map: every published field is carried through byte-for-byte, only // the `author` field is dropped (authorship is the repo now, §3.1) and the $type @@ -586,28 +1103,45 @@ func postV2Body(legacy LegacyPost) (map[string]any, error) { return body, nil } -// uploadEmbedBlobs copies every embed blob's BYTES from the community's blob store -// into the author's repo. It runs BEFORE the postv2 record is written, because a -// record's embed may only reference blobs the repo has already received. +// acceptanceSubject reads the subject strongRef out of a decoded acceptance +// record. A record whose subject is missing or malformed yields two empty +// strings, which the caller treats as "this is not our acceptance" — the +// fail-closed reading. +func acceptanceSubject(record map[string]any) (uri, cid string) { + subject, ok := record["subject"].(map[string]any) + if !ok { + return "", "" + } + uri, _ = subject["uri"].(string) + cid, _ = subject["cid"].(string) + return uri, cid +} + +// uploadEmbedBlobs copies every embed blob's BYTES from the COMMUNITY's blob +// store into the author's repo. It runs BEFORE the postv2 record is written, +// because a record's embed may only reference blobs the repo has already +// received. // -// The bytes are fetched via com.atproto.sync.getBlob against the instance PDS -// (the host the author repo is bound to, which also hosts the community's repo on -// a Coves instance) and uploaded through the author's own credentialed -// UploadBlob. A blob left uncopied fails the record here rather than after the -// old bytes are gone. +// THE BYTES COME FROM THE COMMUNITY'S OWN PDS. A blob lives in the repo that +// holds it, so the community's blobs are fetched from the community's host — +// which is the same machine as the author's only for as long as every account on +// this instance shares one PDS. Fetching them from the author's host works today +// and silently 404s the moment that stops being true, which is exactly the kind +// of assumption a one-shot destructive tool must not carry. func (r *Rematerializer) uploadEmbedBlobs(ctx context.Context, repo AuthorRepo, legacy LegacyPost) error { refs := extractBlobRefs(cloneRecord(legacy.RawRecord)) if len(refs) == 0 { return nil } - host, ok := repoHostURL(repo) - if !ok { - return fmt.Errorf("copying blobs for %s: the author repo exposes no host URL to fetch the community's blobs from", legacy.URI) + communityHost, err := r.communityHostURL(ctx, legacy.CommunityDID) + if err != nil { + return fmt.Errorf("copying blobs for %s: %w", legacy.URI, err) } + client := r.blobClient() for _, ref := range refs { - data, err := fetchBlobBytes(ctx, host, legacy.CommunityDID, ref.cid) + data, err := client.Fetch(ctx, communityHost, legacy.CommunityDID, ref.cid) if err != nil { return fmt.Errorf("fetching embed blob %s from %s: %w", ref.cid, legacy.CommunityDID, err) } @@ -615,37 +1149,87 @@ func (r *Rematerializer) uploadEmbedBlobs(ctx context.Context, repo AuthorRepo, if mimeType == "" { mimeType = "application/octet-stream" } - if _, err := repo.UploadBlob(ctx, data, mimeType); err != nil { + uploaded, err := repo.UploadBlob(ctx, data, mimeType) + if err != nil { return fmt.Errorf("uploading embed blob %s into the author repo of %s: %w", ref.cid, repo.DID(), err) } + // A blob CID is content-addressed, so re-uploading identical bytes yields + // the identical CID and the postv2 may carry the community's ref unchanged. + // That is a PROPERTY OF THE PDS, not of this code, so it is checked rather + // than assumed: a host that re-encoded on upload would mint a different CID, + // and the postv2 would reference bytes that are not there. + if got := uploadedBlobCID(uploaded); got != "" && got != ref.cid { + return fmt.Errorf( + "the author's PDS stored embed blob %s under a DIFFERENT CID (%s); the postv2 would reference bytes the repo does not serve. "+ + "Blob CIDs are content-addressed, so this means the host re-encoded the upload and the record cannot be re-materialized unchanged", + ref.cid, got) + } } return nil } -// verifyEmbedBlobsPresent confirms every embed blob is served by the author's -// repo — the P4 guarantee that the postv2's media resolves against the author, -// not the community repo the old record is about to be deleted from. It runs -// AFTER the postv2 is written, because the PDS serves a blob only once a record -// pins it; a 200 from getBlob proves the bytes actually landed. -func (r *Rematerializer) verifyEmbedBlobsPresent(ctx context.Context, repo AuthorRepo, legacy LegacyPost) error { - refs := extractBlobRefs(cloneRecord(legacy.RawRecord)) +// uploadedBlobCID reads the CID out of an upload result, tolerating a ref shape +// the fakes leave empty. +func uploadedBlobCID(ref *blobs.BlobRef) string { + if ref == nil { + return "" + } + return ref.Ref["$link"] +} + +// verifyEmbedBlobsPresent confirms every embed blob referenced by body is served +// by the author's repo — the P4 guarantee that the postv2's media resolves +// against the author, not the community repo the old record is about to be +// deleted from. +// +// It runs as part of the pre-delete verification on EVERY path. Running it only +// in the first-pass branch meant a resumed run that re-entered at postv2_written +// deleted the legacy record — and with it the last reference keeping the +// community's blobs alive — having checked nothing about the media at all. +func (r *Rematerializer) verifyEmbedBlobsPresent(ctx context.Context, repo AuthorRepo, oldURI string, body map[string]any) error { + refs := extractBlobRefs(cloneRecord(body)) if len(refs) == 0 { return nil } host, ok := repoHostURL(repo) if !ok { - return fmt.Errorf("verifying blobs for %s: the author repo exposes no host URL", legacy.URI) + return fmt.Errorf("verifying blobs for %s: the author repo exposes no host URL", oldURI) } + client := r.blobClient() for _, ref := range refs { - if !blobPresent(ctx, host, repo.DID(), ref.cid) { + present, err := client.Present(ctx, host, repo.DID(), ref.cid) + if err != nil { + // "I could not ask" is not "it is not there". Reporting a transport + // failure as absence would refuse a healthy record; reporting it as + // presence would license deleting the only copy of the bytes. + return fmt.Errorf("checking whether embed blob %s is present in the author repo of %s: %w", ref.cid, repo.DID(), err) + } + if !present { return fmt.Errorf("embed blob %s is not present in the author repo of %s after the postv2 write; refusing to proceed toward the delete", ref.cid, repo.DID()) } } return nil } +// communityHostURL is the PDS host the COMMUNITY's repo lives on — where its +// blobs are actually served from. +func (r *Rematerializer) communityHostURL(ctx context.Context, communityDID string) (string, error) { + if r.CommunityRepos == nil { + return "", fmt.Errorf("no community-repo factory is wired, so the host holding %s's blobs cannot be resolved", communityDID) + } + repo, err := r.CommunityRepos(ctx, communityDID) + if err != nil { + return "", fmt.Errorf("opening the community repo of %s: %w", communityDID, err) + } + host, ok := hostURLOf(repo) + if !ok { + return "", fmt.Errorf("the community repo of %s exposes no host URL to fetch its blobs from", communityDID) + } + return host, nil +} + // rematerializeBlobRef is one embed blob to copy: its CID and MIME type, both // read from the blob reference in the record. type rematerializeBlobRef struct { @@ -692,17 +1276,42 @@ func blobLinkCID(blob map[string]any) string { return "" } -// fetchBlobBytes downloads a blob's bytes from a repo via com.atproto.sync.getBlob. -func fetchBlobBytes(ctx context.Context, host, did, cid string) ([]byte, error) { - blobURL := blobs.HydrateBlobURL(host, did, cid) - if blobURL == "" { - return nil, fmt.Errorf("could not build a getBlob URL for %s / %s", did, cid) - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, blobURL, nil) - if err != nil { - return nil, err - } - resp, err := http.DefaultClient.Do(req) +// httpRematerializeBlobClient is the production RematerializeBlobClient: bounded +// HTTP against com.atproto.sync.getBlob. +type httpRematerializeBlobClient struct { + client *http.Client + // maxBytes is the copy cap. It is a field rather than the package constant so + // the overrun path can be exercised without moving 100 MiB through a test. + maxBytes int +} + +// DefaultRematerializeBlobClient is the production blob client, over an HTTP +// client with its OWN timeout. +// +// http.DefaultClient has none, and a batch tool that hangs on a half-open socket +// is a batch tool the operator must kill and re-run — mid-migration, without +// knowing where it stopped. +func DefaultRematerializeBlobClient() RematerializeBlobClient { + return newRematerializeBlobClient( + &http.Client{Timeout: rematerializeBlobFetchTimeout}, + maxRematerializeBlobBytes, + ) +} + +// newRematerializeBlobClient is the constructor the tests use to shrink the cap. +func newRematerializeBlobClient(client *http.Client, maxBytes int) *httpRematerializeBlobClient { + return &httpRematerializeBlobClient{client: client, maxBytes: maxBytes} +} + +// Fetch downloads a blob's bytes via com.atproto.sync.getBlob. +// +// IT READS ONE BYTE PAST THE CAP ON PURPOSE. io.ReadAll over a LimitReader +// cannot tell "the body ended" from "the limit was reached", so a blob larger +// than the cap used to be uploaded TRUNCATED — under a different CID, referenced +// by a postv2 that would then be verified against the wrong bytes. Over-reading +// by one byte makes the overrun observable, and it is an error. +func (c *httpRematerializeBlobClient) Fetch(ctx context.Context, host, did, cid string) ([]byte, error) { + resp, err := c.get(ctx, host, did, cid) if err != nil { return nil, err } @@ -710,33 +1319,63 @@ func fetchBlobBytes(ctx context.Context, host, did, cid string) ([]byte, error) if resp.StatusCode != http.StatusOK { return nil, fmt.Errorf("getBlob returned status %d", resp.StatusCode) } - return io.ReadAll(io.LimitReader(resp.Body, maxRematerializeBlobBytes)) + + data, err := io.ReadAll(io.LimitReader(resp.Body, int64(c.maxBytes)+1)) + if err != nil { + return nil, fmt.Errorf("reading blob bytes: %w", err) + } + if len(data) > c.maxBytes { + return nil, fmt.Errorf( + "blob %s exceeds the %d-byte copy cap; a truncated copy is DIFFERENT bytes, so it would be stored under a different CID and the postv2 would point at media the repo does not serve", + cid, c.maxBytes) + } + return data, nil +} + +// Present reports whether the repo serves the blob. A transport failure is an +// error, never a false. +func (c *httpRematerializeBlobClient) Present(ctx context.Context, host, did, cid string) (bool, error) { + resp, err := c.get(ctx, host, did, cid) + if err != nil { + return false, err + } + defer func() { _ = resp.Body.Close() }() + _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, int64(c.maxBytes))) + switch { + case resp.StatusCode == http.StatusOK: + return true, nil + case resp.StatusCode == http.StatusNotFound: + return false, nil + default: + return false, fmt.Errorf("getBlob for %s in %s returned status %d, which is neither 'here' nor 'absent'", cid, did, resp.StatusCode) + } } -// blobPresent reports whether a repo serves a blob — a 200 from getBlob proves the -// repo actually holds the bytes. -func blobPresent(ctx context.Context, host, did, cid string) bool { +func (c *httpRematerializeBlobClient) get(ctx context.Context, host, did, cid string) (*http.Response, error) { blobURL := blobs.HydrateBlobURL(host, did, cid) if blobURL == "" { - return false + return nil, fmt.Errorf("could not build a getBlob URL for %s / %s on %q", did, cid, host) } req, err := http.NewRequestWithContext(ctx, http.MethodGet, blobURL, nil) if err != nil { - return false + return nil, fmt.Errorf("building the getBlob request for %s: %w", cid, err) } - resp, err := http.DefaultClient.Do(req) + resp, err := c.client.Do(req) if err != nil { - return false + return nil, fmt.Errorf("requesting blob %s from %s: %w", cid, did, err) } - defer func() { _ = resp.Body.Close() }() - _, _ = io.Copy(io.Discard, io.LimitReader(resp.Body, maxRematerializeBlobBytes)) - return resp.StatusCode == http.StatusOK + return resp, nil } // repoHostURL extracts the PDS host the author repo is bound to, when the concrete // repo exposes one. The production pds.Client does; the state-machine fakes do // not, and they never need it because their records carry no blobs. func repoHostURL(repo AuthorRepo) (string, bool) { + return hostURLOf(repo) +} + +// hostURLOf reads the PDS host off anything that exposes one. +func hostURLOf(repo any) (string, bool) { if h, ok := repo.(interface{ HostURL() string }); ok { if host := h.HostURL(); host != "" { return host, true @@ -776,34 +1415,39 @@ func sameRecordBody(a, b map[string]any) bool { return string(aj) == string(bj) } -// legacyFromLedgerRow reconstructs the minimal LegacyPost the reconcile pass needs -// to finish a row past the postv2 write: the delete step keys off the old URI, and -// the community DID is parsed back out of it. It deliberately carries no -// RawRecord — a reconciled row is past the point where the record body is read. -func legacyFromLedgerRow(row RematerializeLedgerRow) (LegacyPost, error) { - communityDID, err := communityDIDFromURI(row.OldURI) - if err != nil { - return LegacyPost{}, err - } +// legacyFromLedgerRow reconstructs the LegacyPost the reconcile pass needs to +// finish a row past the postv2 write. It carries no RawRecord — the reconcile +// path re-reads the record from the source when it needs the body — but it does +// carry the community DID and the source CID, which are what the scope check and +// the guarded delete are made of. +func legacyFromLedgerRow(row RematerializeLedgerRow) LegacyPost { return LegacyPost{ URI: row.OldURI, - CommunityDID: communityDID, + CID: row.SourceCID, + CommunityDID: row.CommunityDID, AuthorDID: row.AuthorDID, - }, nil + } } -// communityDIDFromURI extracts the repo authority (the community DID) from an -// at:// record URI. -func communityDIDFromURI(uri string) (string, error) { - const scheme = "at://" - if len(uri) <= len(scheme) || uri[:len(scheme)] != scheme { - return "", fmt.Errorf("cannot extract a community DID from %q: not an at:// URI", uri) - } - rest := uri[len(scheme):] - for i := 0; i < len(rest); i++ { - if rest[i] == '/' { - return rest[:i], nil +// distinct returns the unique values of a slice, order-preserving. +func distinct(values []string) []string { + seen := map[string]bool{} + var out []string + for _, v := range values { + if seen[v] { + continue } + seen[v] = true + out = append(out, v) + } + return out +} + +// joinFirst renders at most n values for an error message, saying how many more +// there are. +func joinFirst(values []string, n int) string { + if len(values) <= n { + return fmt.Sprint(values) } - return "", fmt.Errorf("cannot extract a community DID from %q: no collection path", uri) + return fmt.Sprintf("%v and %d more", values[:n], len(values)-n) } diff --git a/internal/core/posts/rematerialize_blob_test.go b/internal/core/posts/rematerialize_blob_test.go new file mode 100644 index 0000000..a0cfed4 --- /dev/null +++ b/internal/core/posts/rematerialize_blob_test.go @@ -0,0 +1,184 @@ +package posts + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// The blob copy leg, whose two default behaviours both destroyed data silently. + +// A blob larger than the copy cap used to be uploaded TRUNCATED. +// +// io.ReadAll over an io.LimitReader cannot distinguish "the body ended" from +// "the limit was reached" — both come back as a successful read of exactly N +// bytes with a nil error. So an oversized blob was silently cut, uploaded under +// a DIFFERENT CID (blob CIDs are content-addressed), and the postv2 kept the +// original reference — pointing at bytes the author's repo does not hold. The +// legacy record, and with it the only intact copy, was then deleted. +func TestRematerializeBlobClient_Fetch_FailsRatherThanTruncating(t *testing.T) { + const cap = 1024 + oversized := strings.Repeat("x", cap+64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(oversized)) + })) + defer server.Close() + + client := newRematerializeBlobClient(server.Client(), cap) + _, err := client.Fetch(context.Background(), server.URL, "did:plc:community2222222222222222", "bafkreioversized") + + require.Errorf(t, err, + "an oversized blob was read without complaint. A truncated read is uploaded under a different CID and the postv2 ends up referencing media the "+ + "author's repo does not serve — after the community's copy has been deleted. Read limit+1 and refuse the overrun") + assert.Containsf(t, err.Error(), "truncated", + "the error must say what the danger is: a truncated copy is DIFFERENT bytes, not fewer bytes") +} + +func TestRematerializeBlobClient_Fetch_ReturnsTheWholeBodyUnderTheCap(t *testing.T) { + payload := strings.Repeat("y", 4096) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte(payload)) + })) + defer server.Close() + + data, err := newRematerializeBlobClient(server.Client(), 8192). + Fetch(context.Background(), server.URL, "did:plc:community2222222222222222", "bafkreiok") + require.NoError(t, err) + assert.Equalf(t, payload, string(data), "a blob inside the cap must come back byte-for-byte") +} + +// "I could not ask" is not "it is not there", and neither of them is "it is +// there". The probe used to collapse a request-construction error, a transport +// error and a 404 into a single false — so a network blip refused a healthy +// record, and any future change of polarity would have licensed deleting the +// last copy of a blob. +func TestRematerializeBlobClient_Present_DistinguishesAbsentFromUnaskable(t *testing.T) { + t.Run("200 is present", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + _, _ = w.Write([]byte("bytes")) + })) + defer server.Close() + + present, err := DefaultRematerializeBlobClient().Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") + require.NoError(t, err) + assert.True(t, present) + }) + + t.Run("404 is absent, and not an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusNotFound) + })) + defer server.Close() + + present, err := DefaultRematerializeBlobClient().Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") + require.NoErrorf(t, err, "a definite 404 is an ANSWER — the blob is not there — and must not be reported as a failure to ask") + assert.False(t, present) + }) + + t.Run("a 503 is neither, and must be an error", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer server.Close() + + _, err := DefaultRematerializeBlobClient().Present(context.Background(), server.URL, "did:plc:x2222222222222222222222", "bafkrei1") + require.Errorf(t, err, + "a 503 was collapsed into 'absent'. The caller uses this answer to decide whether it is safe to delete the record that keeps the community's "+ + "only copy of the bytes alive, and a server that could not answer has told it nothing") + }) + + t.Run("an unreachable host is an error, never a false", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc(func(http.ResponseWriter, *http.Request) {})) + url := server.URL + server.Close() // nothing is listening now + + _, err := DefaultRematerializeBlobClient().Present(context.Background(), url, "did:plc:x2222222222222222222222", "bafkrei1") + require.Errorf(t, err, "a transport failure must surface; reporting it as 'absent' turns a blip into a refusal and a polarity slip into data loss") + }) + + t.Run("an unbuildable URL is an error, never a false", func(t *testing.T) { + _, err := DefaultRematerializeBlobClient().Present(context.Background(), "", "did:plc:x2222222222222222222222", "bafkrei1") + require.Errorf(t, err, "a URL that could not be built means the question was never asked") + }) +} + +// THE BLOB REFERENCE IS CARRIED THROUGH UNCHANGED, ON PURPOSE. +// +// An external review claimed the conversion is broken because the postv2 keeps +// the community-repo blob CID rather than rewriting it to the CID of the +// re-upload. It is not broken: atProto blob CIDs are CONTENT-ADDRESSED, so +// re-uploading identical bytes yields the identical CID and the reference stays +// valid — which is also why the real-PDS contract's getBlob against the author's +// repo returns 200 for the community's original CID. +// +// That is a property of the PDS, though, not of this code. A host that +// re-encoded on upload would mint a different CID and the postv2 would reference +// bytes the repo does not serve — so uploadEmbedBlobs CHECKS the returned CID +// rather than assuming it, and verifyEmbedBlobsPresent independently proves the +// referenced CID is actually served before anything is deleted. +func TestPostV2Body_CarriesTheEmbedBlobReferenceThroughUnchanged(t *testing.T) { + blobRef := map[string]any{ + "$type": "blob", + "ref": map[string]any{"$link": "bafkreicommunityblobcid"}, + "mimeType": "image/png", + "size": float64(12), + } + legacy := LegacyPost{ + URI: "at://did:plc:community2222222222222222/social.coves.community.post/3kabc", + RawRecord: map[string]any{ + "$type": LegacyPostCollection, + "author": "did:plc:author11111111111111111", + "embed": map[string]any{ + "$type": "social.coves.embed.images", + "images": []any{map[string]any{"alt": "a picture", "image": blobRef}}, + }, + }, + } + + body, err := postV2Body(legacy) + require.NoError(t, err) + + refs := extractBlobRefs(body) + require.Lenf(t, refs, 1, "the conversion lost the embed's blob reference entirely") + assert.Equalf(t, "bafkreicommunityblobcid", refs[0].cid, + "the blob reference was REWRITTEN. It must be carried through verbatim: a blob CID is the hash of its bytes, so the re-upload of identical bytes "+ + "has the identical CID, and inventing a new one would point the postv2 at media that does not exist") + assert.Equalf(t, "image/png", refs[0].mimeType, + "the MIME type must survive: it is sent as the upload's Content-Type, and a PDS enforcing the granular blob:*/* scope rejects a wildcard") +} + +// extractBlobRefs has to find a blob wherever the embed union puts it, because a +// blob it does not find is a blob that is never copied — and the record that +// referenced it is deleted anyway. +func TestExtractBlobRefs_FindsBlobsNestedAnywhereInTheRecord(t *testing.T) { + record := map[string]any{ + "embed": map[string]any{ + "$type": "social.coves.embed.external", + "external": map[string]any{ + "uri": "https://example.com", + "thumb": map[string]any{"$type": "blob", "ref": map[string]any{"$link": "bafkreithumb"}, "mimeType": "image/jpeg"}, + }, + }, + "images": []any{ + map[string]any{"image": map[string]any{"$type": "blob", "ref": map[string]any{"$link": "bafkreiimage1"}}}, + map[string]any{"image": map[string]any{"$type": "blob", "ref": "bafkreiimage2"}}, // the bare-string ref shape + }, + } + + refs := extractBlobRefs(record) + + found := map[string]bool{} + for _, ref := range refs { + found[ref.cid] = true + } + for _, want := range []string{"bafkreithumb", "bafkreiimage1", "bafkreiimage2"} { + assert.Truef(t, found[want], + "the blob %s was not found. A blob this walk misses is one whose bytes are never copied into the author's repo — and the record that held "+ + "the only reference to them is deleted regardless, so the media is lost", want) + } +} diff --git a/internal/core/posts/rematerialize_credentials_test.go b/internal/core/posts/rematerialize_credentials_test.go new file mode 100644 index 0000000..425fd32 --- /dev/null +++ b/internal/core/posts/rematerialize_credentials_test.go @@ -0,0 +1,112 @@ +package posts + +import ( + "context" + "errors" + "fmt" + "testing" + + covesoauth "Coves/internal/atproto/oauth" + + "github.com/bluesky-social/indigo/atproto/auth/oauth" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// A TRANSIENT CREDENTIAL FAILURE MUST NOT BE WRITTEN AS A TERMINAL VERDICT. +// +// ErrNoAuthorCredentials is not a diagnosis, it is a SENTENCE: the +// re-materialization census answers it by writing fallback_left_legacy, which is +// terminal three ways over — IsFallback short-circuits RematerializeOne, +// ListResumable excludes the row, and MarkFallback only accepts a row still at +// discovered, so nothing in the tool can ever move it back. +// +// The Kagi aggregator authors the overwhelming majority of production posts. If +// one network blip, one PDS 5xx, or one DPoP nonce failure while resuming ITS +// session is reported as "this author has no credentials", the census marks the +// ENTIRE CORPUS terminal in seconds and every subsequent run is a permanent +// no-op — with nothing in the tool to undo it. +// +// So the sentinel is split. "There is no grant to resume" is a genuine, terminal +// absence. Everything else is ErrAuthorCredentialsUnavailable: RETRYABLE, and +// the run fails loudly on it rather than sentencing a row. + +// resumeFailureClass is the classifier under test: it is what decides whether a +// failure to open an author's repo is a verdict or a retry. +func TestClassifyResumeFailure_MissingGrantIsTerminal(t *testing.T) { + err := classifyResumeFailure("did:plc:aggregator", covesoauth.ErrSessionNotFound) + + require.Truef(t, errors.Is(err, ErrNoAuthorCredentials), + "a session the store does not hold is the one genuinely terminal case: nobody can re-authorize an aggregator from inside a batch tool, "+ + "so the post is left as legacy rather than forged. got: %v", err) + assert.Falsef(t, errors.Is(err, ErrAuthorCredentialsUnavailable), + "an absent grant must NOT also be retryable, or the run would fail instead of recording the fallback the census exists to produce") +} + +func TestClassifyResumeFailure_TransientFailuresAreRetryableNotTerminal(t *testing.T) { + transient := []struct { + name string + err error + }{ + {"network blip", errors.New("dial tcp 10.0.0.5:443: connect: connection refused")}, + {"PDS 5xx", fmt.Errorf("token refresh: %w", errors.New("unexpected status 502"))}, + {"DPoP nonce failure", errors.New("use_dpop_nonce")}, + {"database error reading the session store", fmt.Errorf("failed to get session: %w", errors.New("driver: bad connection"))}, + } + + for _, tc := range transient { + t.Run(tc.name, func(t *testing.T) { + err := classifyResumeFailure("did:plc:aggregator", tc.err) + + assert.Falsef(t, errors.Is(err, ErrNoAuthorCredentials), + "%s was classified as ErrNoAuthorCredentials. The re-materialization census writes that as fallback_left_legacy, which is TERMINAL and "+ + "has no in-tool path back — one blip while resuming the aggregator's session would sentence every post it ever wrote. got: %v", tc.name, err) + assert.Truef(t, errors.Is(err, ErrAuthorCredentialsUnavailable), + "%s must be RETRYABLE so the run fails loudly and the operator can re-run after the cause clears. got: %v", tc.name, err) + }) + } +} + +// The two sentinels must be genuinely distinct values, not one aliased to the +// other: every caller decides "sentence or retry" by telling them apart. +func TestAuthorCredentialSentinels_AreDistinct(t *testing.T) { + assert.Falsef(t, errors.Is(ErrAuthorCredentialsUnavailable, ErrNoAuthorCredentials), + "the retryable sentinel must not satisfy errors.Is against the terminal one, or a transient failure is still written as a terminal fallback") + assert.Falsef(t, errors.Is(ErrNoAuthorCredentials, ErrAuthorCredentialsUnavailable), + "the terminal sentinel must not satisfy errors.Is against the retryable one, or an author who genuinely cannot be restored fails the whole run forever") +} + +// A nil-but-successful resume — the store answered without an error and handed +// back nothing — is an absent grant, not a transport fault. +func TestClassifyResumeFailure_NilErrorIsNotAFailure(t *testing.T) { + assert.NoErrorf(t, classifyResumeFailure("did:plc:aggregator", nil), + "classifying a nil resume error must produce no error at all") +} + +// The tool resolves credentials ONCE PER DISTINCT AUTHOR, not once per post. +// +// Each resume is a refresh-token rotation against the PDS. An aggregator with +// 5,000 posts would otherwise trigger 5,000 rotations in a single run — minutes +// of avoidable load, thousands of chances for the transient failure above, and a +// token-rotation chain any one of whose links can break the session for the +// AppView itself. +func TestRematerializer_ResolvesCredentialsOncePerAuthor(t *testing.T) { + resolutions := map[string]int{} + tool := &Rematerializer{ + AuthorRepos: func(_ context.Context, did string, _ *oauth.ClientSessionData) (AuthorRepo, error) { + resolutions[did]++ + return nil, fmt.Errorf("no repo in this unit test: %w", ErrNoAuthorCredentials) + }, + } + + ctx := context.Background() + for i := 0; i < 5; i++ { + _, _ = tool.authorRepo(ctx, "did:plc:aggregator") + _, _ = tool.authorRepo(ctx, "did:plc:human") + } + + assert.Equalf(t, 1, resolutions["did:plc:aggregator"], + "the aggregator's session was resumed %d times. Each resume rotates the refresh token; an aggregator with 5,000 posts would rotate 5,000 times "+ + "in one run", resolutions["did:plc:aggregator"]) + assert.Equalf(t, 1, resolutions["did:plc:human"], "credentials must be resolved once per distinct author DID") +} diff --git a/internal/core/posts/rematerialize_dryrun.go b/internal/core/posts/rematerialize_dryrun.go new file mode 100644 index 0000000..66da0b0 --- /dev/null +++ b/internal/core/posts/rematerialize_dryrun.go @@ -0,0 +1,568 @@ +package posts + +import ( + "context" + "crypto/sha256" + "encoding/base32" + "encoding/json" + "fmt" + "strings" + "sync" + "time" + + "github.com/bluesky-social/indigo/atproto/auth/oauth" + + "Coves/internal/atproto/pds" + "Coves/internal/core/blobs" +) + +// DRY RUN: the same code path, with only the MUTATIONS replaced. +// +// # WHY THIS IS SEAM WRAPPERS AND NOT AN `if dryRun` BRANCH +// +// A dry run whose value is a flag consulted inside the state machine tests a +// DIFFERENT program than the real one: every branch is a place the two can +// diverge, and the divergences accumulate in exactly the code the operator is +// trying to rehearse. So the state machine has no idea it is rehearsing. Every +// seam is wrapped instead, and a wrapped seam either +// +// - performs the real operation, when it is a READ (the source listing, the +// legacy re-read, credential resolution, blob fetches, record reads), or +// - records the intended mutation in memory and answers subsequent reads from +// that memory, when it is a WRITE. +// +// The consequence is that a dry run really does: resolve every author's +// credentials (and so really does discover a missing grant), re-read every +// legacy record from the PDS (and so really does discover an edit that landed), +// build every postv2 body losslessly, derive every rkey, enumerate and FETCH +// every embed blob (and so really does discover an unreachable or oversized +// one), and run the whole pre-delete verification against the records it would +// have written. What it never does is put a byte in a repo or take one out. +// +// # WHAT IT CANNOT PROVE +// +// The CIDs are synthesised from the record bodies rather than minted by a PDS, +// so a dry run cannot prove that the PDS accepts a record it will validate, and +// it cannot prove a blob upload succeeds. It proves the tool's decisions, its +// scope, its conversions and its reachability — which is what an operator is +// asking about at 2am. + +// DryRunOf returns a Rematerializer that walks the same code path as tool but +// mutates nothing. +// +// The returned tool shares no state with the original: cancel it, discard it, +// run it twice. Its ledger writes, repo writes and acceptance writes live in an +// in-memory overlay that is thrown away with it. +func DryRunOf(tool *Rematerializer) *Rematerializer { + overlay := &dryRunOverlay{ + ledgerRows: map[string]RematerializeLedgerRow{}, + records: map[string]*pds.RecordResponse{}, + blobs: map[string]bool{}, + } + + dry := &Rematerializer{ + Source: &dryRunSource{inner: tool.Source, overlay: overlay}, + Ledger: &dryRunLedger{inner: tool.Ledger, overlay: overlay}, + AuthorRepos: dryRunAuthorRepos(tool.AuthorRepos, overlay), + Acceptances: &dryRunAcceptanceWriter{overlay: overlay}, + CommunityRepos: dryRunCommunityRepos(tool.CommunityRepos, overlay), + Blobs: &dryRunBlobClient{inner: tool.blobClient(), overlay: overlay}, + CommunityScope: tool.CommunityScope, + Progress: tool.Progress, + PerRecordTimeout: tool.PerRecordTimeout, + AbortOnFallback: tool.AbortOnFallback, + } + return dry +} + +// DryRunDeletes reports how many legacy records the tool WOULD have deleted. +// It returns 0 and false for a Rematerializer that is not a dry run. +func DryRunDeletes(tool *Rematerializer) (int, bool) { + source, ok := tool.Source.(*dryRunSource) + if !ok { + return 0, false + } + source.overlay.mu.Lock() + defer source.overlay.mu.Unlock() + return source.overlay.deletes, true +} + +// dryRunOverlay is the in-memory store every wrapped write lands in and every +// wrapped read falls back to. One overlay per dry run. +type dryRunOverlay struct { + mu sync.Mutex + ledgerRows map[string]RematerializeLedgerRow + ledgerOriginal map[string]RematerializeState // the state each row stood at on disk + deletedURIs map[string]bool + records map[string]*pds.RecordResponse // "did/collection/rkey" -> record + blobs map[string]bool // content-keyed marks for the rehearsal + deletes int +} + +func (o *dryRunOverlay) recordKey(did, collection, rkey string) string { + return did + "/" + collection + "/" + rkey +} + +func (o *dryRunOverlay) putRecord(did, collection, rkey string, value map[string]any) *pds.RecordResponse { + o.mu.Lock() + defer o.mu.Unlock() + rec := &pds.RecordResponse{ + URI: "at://" + did + "/" + collection + "/" + rkey, + CID: dryRunCID(value), + Value: value, + } + o.records[o.recordKey(did, collection, rkey)] = rec + return rec +} + +func (o *dryRunOverlay) getRecord(did, collection, rkey string) (*pds.RecordResponse, bool) { + o.mu.Lock() + defer o.mu.Unlock() + rec, ok := o.records[o.recordKey(did, collection, rkey)] + return rec, ok +} + +// dryRunCID synthesises a stable, obviously-fake CID from a record body. +// +// It is derived from the canonical bytes so that the tool's own CID comparisons +// — the converged-write body check, the postv2 re-read, the acceptance's pinned +// subject — mean the same thing in a rehearsal as they do in production. The +// "bafyDRYRUN" prefix is deliberate: a synthetic CID must never be mistaken for +// one a PDS minted if it escapes into a log. +func dryRunCID(value map[string]any) string { + raw, err := json.Marshal(value) + if err != nil { + raw = []byte(fmt.Sprintf("%v", value)) + } + digest := sha256.Sum256(raw) + encoded := strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(digest[:16])) + return "bafyDRYRUN" + encoded +} + +// ---- source --------------------------------------------------------------- + +// dryRunSource performs every READ for real — the listing and the pre-delete +// re-read both go to the PDS — and counts the deletes it did not make. +type dryRunSource struct { + inner LegacySource + overlay *dryRunOverlay +} + +// ListLegacyPosts lists for real, minus the records the rehearsal has already +// "deleted". Without that subtraction the final re-scan would always see every +// record still standing, and a rehearsal of a run that WOULD complete would +// report itself incomplete — which is the one number the operator reads. +func (s *dryRunSource) ListLegacyPosts(ctx context.Context) ([]LegacyPost, error) { + all, err := s.inner.ListLegacyPosts(ctx) + if err != nil { + return nil, err + } + s.overlay.mu.Lock() + defer s.overlay.mu.Unlock() + if len(s.overlay.deletedURIs) == 0 { + return all, nil + } + out := make([]LegacyPost, 0, len(all)) + for _, p := range all { + if s.overlay.deletedURIs[p.URI] { + continue + } + out = append(out, p) + } + return out, nil +} + +func (s *dryRunSource) ReadLegacyPost(ctx context.Context, uri string) (LegacyPost, bool, error) { + s.overlay.mu.Lock() + deleted := s.overlay.deletedURIs[uri] + s.overlay.mu.Unlock() + if deleted { + // A record this run has "deleted" must read as gone, or the reconcile pass + // would rehearse a second delete of it. + return LegacyPost{}, false, nil + } + return s.inner.ReadLegacyPost(ctx, uri) +} + +func (s *dryRunSource) DeleteLegacyPost(_ context.Context, legacy LegacyPost, swapCID string) error { + if swapCID == "" { + // The rehearsal enforces the same precondition the real source does, so a + // dry run catches a missing guard rather than masking it. + return fmt.Errorf("dry run: refusing to rehearse an unguarded delete of %s", legacy.URI) + } + s.overlay.mu.Lock() + defer s.overlay.mu.Unlock() + s.overlay.deletes++ + if s.overlay.deletedURIs == nil { + s.overlay.deletedURIs = map[string]bool{} + } + s.overlay.deletedURIs[legacy.URI] = true + return nil +} + +// ---- ledger --------------------------------------------------------------- + +// dryRunLedger reads through to the real ledger — so a rehearsal resumes from +// the same place a real run would — and holds every write in the overlay. +type dryRunLedger struct { + inner RematerializeLedger + overlay *dryRunOverlay +} + +func (l *dryRunLedger) Discover(ctx context.Context, oldURI, communityDID, authorDID string) (RematerializeLedgerRow, error) { + if row, ok := l.overlayRow(oldURI); ok { + return row, nil + } + row, found, err := l.inner.Get(ctx, oldURI) + if err != nil { + return RematerializeLedgerRow{}, err + } + if !found { + row = RematerializeLedgerRow{ + OldURI: oldURI, State: RematerializeDiscovered, + CommunityDID: communityDID, AuthorDID: authorDID, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + } + l.setOverlayRow(row) + return row, nil +} + +func (l *dryRunLedger) Get(ctx context.Context, oldURI string) (RematerializeLedgerRow, bool, error) { + if row, ok := l.overlayRow(oldURI); ok { + return row, true, nil + } + return l.inner.Get(ctx, oldURI) +} + +func (l *dryRunLedger) ListResumable(ctx context.Context, communityDID string) ([]RematerializeLedgerRow, error) { + rows, err := l.inner.ListResumable(ctx, communityDID) + if err != nil { + return nil, err + } + out := make([]RematerializeLedgerRow, 0, len(rows)) + for _, row := range rows { + if overlaid, ok := l.overlayRow(row.OldURI); ok { + row = overlaid + } + if row.State == RematerializeDone || IsFallback(row.State) { + continue + } + out = append(out, row) + } + return out, nil +} + +func (l *dryRunLedger) RecordPostV2Written(_ context.Context, oldURI, sourceCID, newURI, newCID, newRkey string) error { + return l.advance(oldURI, RematerializeDiscovered, func(row *RematerializeLedgerRow) { + row.State = RematerializePostV2Written + row.SourceCID, row.NewURI, row.NewCID, row.NewRkey = sourceCID, newURI, newCID, newRkey + }) +} + +func (l *dryRunLedger) MarkVerified(_ context.Context, oldURI string) error { + return l.advance(oldURI, RematerializePostV2Written, func(row *RematerializeLedgerRow) { + row.State = RematerializeVerified + }) +} + +func (l *dryRunLedger) MarkMigrated(_ context.Context, oldURI string) error { + return l.advance(oldURI, RematerializeVerified, func(row *RematerializeLedgerRow) { + row.State = RematerializeMigrated + }) +} + +func (l *dryRunLedger) MarkDone(_ context.Context, oldURI string) error { + return l.advance(oldURI, RematerializeMigrated, func(row *RematerializeLedgerRow) { + row.State = RematerializeDone + }) +} + +func (l *dryRunLedger) MarkFallback(_ context.Context, oldURI string, state RematerializeState, reason string) error { + if !IsFallback(state) { + return fmt.Errorf("dry run: %q is not a fallback state", state) + } + return l.advance(oldURI, RematerializeDiscovered, func(row *RematerializeLedgerRow) { + row.State = state + row.Reason = reason + }) +} + +// ReopenFallback is a no-op in a rehearsal: it changes no repo, so there is +// nothing to rehearse, and applying it would make the run report progress the +// operator has not authorised. +func (l *dryRunLedger) ReopenFallback(context.Context, string) (int, error) { return 0, nil } + +func (l *dryRunLedger) CountByState(ctx context.Context, communityDID string) (map[RematerializeState]int, error) { + counts, err := l.inner.CountByState(ctx, communityDID) + if err != nil { + return nil, err + } + if counts == nil { + counts = map[RematerializeState]int{} + } + // Re-tally: an overlaid row's real state is the one on disk, and the rehearsal + // moved it somewhere else. Move the count with it. + l.overlay.mu.Lock() + defer l.overlay.mu.Unlock() + for _, row := range l.overlay.ledgerRows { + if communityDID != "" && row.CommunityDID != communityDID { + continue + } + if before, ok := l.overlay.ledgerOriginal[row.OldURI]; ok { + if counts[before] > 0 { + counts[before]-- + if counts[before] == 0 { + delete(counts, before) + } + } + } + counts[row.State]++ + } + return counts, nil +} + +func (l *dryRunLedger) overlayRow(oldURI string) (RematerializeLedgerRow, bool) { + l.overlay.mu.Lock() + defer l.overlay.mu.Unlock() + row, ok := l.overlay.ledgerRows[oldURI] + return row, ok +} + +func (l *dryRunLedger) setOverlayRow(row RematerializeLedgerRow) { + l.overlay.mu.Lock() + defer l.overlay.mu.Unlock() + if _, seen := l.overlay.ledgerOriginal[row.OldURI]; !seen { + if l.overlay.ledgerOriginal == nil { + l.overlay.ledgerOriginal = map[string]RematerializeState{} + } + l.overlay.ledgerOriginal[row.OldURI] = row.State + } + l.overlay.ledgerRows[row.OldURI] = row +} + +// advance applies a guarded transition to the overlay, enforcing the SAME +// from-state guard the real ledger does, so a rehearsal surfaces a divergence +// rather than hiding it. +func (l *dryRunLedger) advance(oldURI string, from RematerializeState, mutate func(*RematerializeLedgerRow)) error { + l.overlay.mu.Lock() + row, ok := l.overlay.ledgerRows[oldURI] + l.overlay.mu.Unlock() + if !ok { + return fmt.Errorf("dry run: no ledger row for %s", oldURI) + } + if row.State != from { + return fmt.Errorf("dry run: transitioning %s from %s: the row stands at %s (the ledger and the tool have diverged)", oldURI, from, row.State) + } + mutate(&row) + row.UpdatedAt = time.Now() + l.setOverlayRow(row) + return nil +} + +// ---- author repos --------------------------------------------------------- + +// dryRunAuthorRepos resolves credentials FOR REAL — that is most of what a +// rehearsal is for — and wraps the resulting repo so its writes land in memory. +func dryRunAuthorRepos(inner AuthorRepoFactory, overlay *dryRunOverlay) AuthorRepoFactory { + return func(ctx context.Context, authorDID string, session *oauth.ClientSessionData) (AuthorRepo, error) { + repo, err := inner(ctx, authorDID, session) + if err != nil { + return nil, err + } + return &dryRunAuthorRepo{inner: repo, overlay: overlay}, nil + } +} + +type dryRunAuthorRepo struct { + inner AuthorRepo + overlay *dryRunOverlay +} + +func (r *dryRunAuthorRepo) GetRecord(ctx context.Context, collection, rkey string) (*pds.RecordResponse, error) { + if rec, ok := r.overlay.getRecord(r.inner.DID(), collection, rkey); ok { + return rec, nil + } + return r.inner.GetRecord(ctx, collection, rkey) +} + +func (r *dryRunAuthorRepo) PutRecordWithCommit(ctx context.Context, collection, rkey string, record any, swapRecord string) (*pds.RecordCommit, error) { + // The create-only guard is rehearsed against the REAL repo first: a record + // that already stands must still produce the swap conflict that drives + // createAuthorRecord's converge-by-read, or the rehearsal would not exercise + // the branch a re-run actually takes. + if swapRecord == "" { + if _, err := r.GetRecord(ctx, collection, rkey); err == nil { + return nil, pds.ErrSwapConflict + } + } + var body map[string]any + if raw, err := json.Marshal(record); err == nil { + _ = json.Unmarshal(raw, &body) + } + rec := r.overlay.putRecord(r.inner.DID(), collection, rkey, body) + return &pds.RecordCommit{URI: rec.URI, CID: rec.CID, CommitRev: "dryrun"}, nil +} + +// DeleteRecord is a no-op: a rehearsal removes nothing. +func (r *dryRunAuthorRepo) DeleteRecord(context.Context, string, string) error { return nil } + +// UploadBlob records that the bytes WOULD have been uploaded, and reports the +// community's own CID back — which is what a content-addressed store returns for +// identical bytes, and what the caller's equality check is written against. +func (r *dryRunAuthorRepo) UploadBlob(_ context.Context, data []byte, mimeType string) (*blobs.BlobRef, error) { + r.overlay.mu.Lock() + defer r.overlay.mu.Unlock() + r.overlay.blobs[r.inner.DID()+"/"+dryRunBlobKey(data)] = true + return &blobs.BlobRef{Type: "blob", MimeType: mimeType, Size: len(data)}, nil +} + +func (r *dryRunAuthorRepo) DID() string { return r.inner.DID() } + +// HostURL forwards the wrapped repo's host so blob probing addresses the same +// PDS a real run would. +func (r *dryRunAuthorRepo) HostURL() string { + host, _ := hostURLOf(r.inner) + return host +} + +// dryRunBlobKey keys the overlay's uploaded-blob set by content, so the presence +// probe can answer for bytes this run fetched and would have uploaded. +func dryRunBlobKey(data []byte) string { + digest := sha256.Sum256(data) + return fmt.Sprintf("%x", digest[:16]) +} + +// ---- acceptances ---------------------------------------------------------- + +// dryRunAcceptanceWriter records the acceptance it would have written and serves +// it back to the verification read, so the whole "read the acceptance back and +// check its subject" leg really runs. +type dryRunAcceptanceWriter struct { + overlay *dryRunOverlay +} + +func (w *dryRunAcceptanceWriter) WriteAcceptance(_ context.Context, cmd CommunityWriteCommand) (CommunityWriteResult, error) { + rkey := SubjectRkey(cmd.PostURI) + rec := w.overlay.putRecord(cmd.CommunityDID, AcceptanceCollection, rkey, map[string]any{ + "$type": AcceptanceCollection, + "subject": map[string]any{"uri": cmd.PostURI, "cid": cmd.PostCID}, + "createdAt": time.Now().UTC().Format(time.RFC3339), + }) + return CommunityWriteResult{URI: rec.URI, RKey: rkey, CID: rec.CID, Rev: "dryrun"}, nil +} + +// The moderation writes are unreachable from the tool by construction; a +// rehearsal that somehow reached one must say so rather than pretend. +func (w *dryRunAcceptanceWriter) WriteRemoval(context.Context, CommunityRemovalCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, fmt.Errorf("dry run: the re-materialization tool must never write a removal") +} +func (w *dryRunAcceptanceWriter) RestoreAcceptance(context.Context, CommunityWriteCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, fmt.Errorf("dry run: the re-materialization tool must never restore an acceptance") +} +func (w *dryRunAcceptanceWriter) RepinAcceptance(context.Context, CommunityWriteCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, fmt.Errorf("dry run: the re-materialization tool must never repin an acceptance") +} +func (w *dryRunAcceptanceWriter) DeleteAcceptance(context.Context, CommunityAcceptanceDeleteCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, fmt.Errorf("dry run: the re-materialization tool must never delete an acceptance") +} + +// ---- community repos ------------------------------------------------------ + +// dryRunCommunityRepos opens the REAL community repo — the tool needs its host +// URL to fetch blobs and its records to fall back on — and overlays the +// acceptance the rehearsal would have written. +func dryRunCommunityRepos(inner CommunityRepoFactory, overlay *dryRunOverlay) CommunityRepoFactory { + if inner == nil { + return nil + } + return func(ctx context.Context, communityDID string) (CommunityRepo, error) { + repo, err := inner(ctx, communityDID) + if err != nil { + return nil, err + } + return &dryRunCommunityRepo{inner: repo, overlay: overlay}, nil + } +} + +type dryRunCommunityRepo struct { + inner CommunityRepo + overlay *dryRunOverlay +} + +func (r *dryRunCommunityRepo) GetRecord(ctx context.Context, collection, rkey string) (*pds.RecordResponse, error) { + if rec, ok := r.overlay.getRecord(r.inner.DID(), collection, rkey); ok { + return rec, nil + } + return r.inner.GetRecord(ctx, collection, rkey) +} + +func (r *dryRunCommunityRepo) PutRecordWithCommit(_ context.Context, collection, rkey string, record any, _ string) (*pds.RecordCommit, error) { + var body map[string]any + if raw, err := json.Marshal(record); err == nil { + _ = json.Unmarshal(raw, &body) + } + rec := r.overlay.putRecord(r.inner.DID(), collection, rkey, body) + return &pds.RecordCommit{URI: rec.URI, CID: rec.CID, CommitRev: "dryrun"}, nil +} + +func (r *dryRunCommunityRepo) ApplyWrites(context.Context, []pds.Write, string) (*pds.ApplyWritesResult, error) { + return nil, fmt.Errorf("dry run: the re-materialization tool must never batch community writes") +} + +func (r *dryRunCommunityRepo) GetLatestCommit(ctx context.Context) (*pds.LatestCommit, error) { + return r.inner.GetLatestCommit(ctx) +} + +func (r *dryRunCommunityRepo) DID() string { return r.inner.DID() } + +func (r *dryRunCommunityRepo) HostURL() string { + host, _ := hostURLOf(r.inner) + return host +} + +// ---- blobs ---------------------------------------------------------------- + +// dryRunBlobClient FETCHES FOR REAL — an unreachable or oversized blob is +// exactly the kind of thing a rehearsal exists to find — and answers the +// presence probe for bytes this run fetched, since it never uploaded them. +type dryRunBlobClient struct { + inner RematerializeBlobClient + overlay *dryRunOverlay +} + +func (c *dryRunBlobClient) Fetch(ctx context.Context, host, did, cid string) ([]byte, error) { + data, err := c.inner.Fetch(ctx, host, did, cid) + if err != nil { + return nil, err + } + c.overlay.mu.Lock() + c.overlay.blobs["fetched/"+dryRunBlobKey(data)] = true + c.overlay.blobs["cid/"+cid] = true + c.overlay.mu.Unlock() + return data, nil +} + +// Present PROBES FOR REAL — a host that cannot be reached at all is something the +// rehearsal should surface — and then answers for bytes this run fetched, since +// it never uploaded them. +// +// The real probe's ANSWER is deliberately not the rehearsal's answer when the +// bytes were fetched: the blob is legitimately absent from the author's repo +// because nothing was uploaded, and reporting that absence would stop every +// rehearsal at the first post carrying media. Its ERROR is not swallowed either +// — an unreachable host is a finding. +func (c *dryRunBlobClient) Present(ctx context.Context, host, did, cid string) (bool, error) { + present, err := c.inner.Present(ctx, host, did, cid) + + c.overlay.mu.Lock() + fetched := c.overlay.blobs["cid/"+cid] + c.overlay.mu.Unlock() + if fetched { + if err != nil { + return false, err + } + return true, nil + } + return present, err +} diff --git a/internal/core/posts/rematerialize_dryrun_test.go b/internal/core/posts/rematerialize_dryrun_test.go new file mode 100644 index 0000000..7b9b1e3 --- /dev/null +++ b/internal/core/posts/rematerialize_dryrun_test.go @@ -0,0 +1,353 @@ +package posts + +import ( + "context" + "errors" + "fmt" + "testing" + "time" + + "github.com/bluesky-social/indigo/atproto/auth/oauth" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "Coves/internal/atproto/pds" + "Coves/internal/core/blobs" +) + +// A REHEARSAL THAT DOES NOT REHEARSE IS WORSE THAN NO REHEARSAL, because the +// operator then confirms `-yes` on the strength of it. +// +// These tests pin the two halves of that: the dry run really performs every read +// and decision the real run performs, and it performs NONE of the writes. + +// ---- in-memory seams ------------------------------------------------------- + +type memAuthorRepo struct { + did string + records map[string]*pds.RecordResponse + puts int + uploads int + deletes int +} + +func newMemAuthorRepo(did string) *memAuthorRepo { + return &memAuthorRepo{did: did, records: map[string]*pds.RecordResponse{}} +} + +func (r *memAuthorRepo) GetRecord(_ context.Context, collection, rkey string) (*pds.RecordResponse, error) { + rec, ok := r.records[collection+"/"+rkey] + if !ok { + return nil, pds.ErrNotFound + } + return rec, nil +} + +func (r *memAuthorRepo) PutRecordWithCommit(_ context.Context, collection, rkey string, _ any, _ string) (*pds.RecordCommit, error) { + r.puts++ + uri := "at://" + r.did + "/" + collection + "/" + rkey + r.records[collection+"/"+rkey] = &pds.RecordResponse{URI: uri, CID: "bafyreal" + rkey} + return &pds.RecordCommit{URI: uri, CID: "bafyreal" + rkey}, nil +} + +func (r *memAuthorRepo) DeleteRecord(context.Context, string, string) error { r.deletes++; return nil } +func (r *memAuthorRepo) UploadBlob(context.Context, []byte, string) (*blobs.BlobRef, error) { + r.uploads++ + return &blobs.BlobRef{}, nil +} +func (r *memAuthorRepo) DID() string { return r.did } +func (r *memAuthorRepo) HostURL() string { return "http://author-pds.invalid" } + +type memCommunityRepo struct { + did string + records map[string]*pds.RecordResponse + puts int +} + +func (r *memCommunityRepo) GetRecord(_ context.Context, collection, rkey string) (*pds.RecordResponse, error) { + rec, ok := r.records[collection+"/"+rkey] + if !ok { + return nil, pds.ErrNotFound + } + return rec, nil +} +func (r *memCommunityRepo) PutRecordWithCommit(context.Context, string, string, any, string) (*pds.RecordCommit, error) { + r.puts++ + return &pds.RecordCommit{}, nil +} +func (r *memCommunityRepo) ApplyWrites(context.Context, []pds.Write, string) (*pds.ApplyWritesResult, error) { + return nil, errors.New("unused") +} +func (r *memCommunityRepo) GetLatestCommit(context.Context) (*pds.LatestCommit, error) { + return &pds.LatestCommit{}, nil +} +func (r *memCommunityRepo) DID() string { return r.did } +func (r *memCommunityRepo) HostURL() string { return "http://community-pds.invalid" } + +type memAcceptanceWriter struct { + repo *memCommunityRepo + calls int +} + +func (w *memAcceptanceWriter) WriteAcceptance(_ context.Context, cmd CommunityWriteCommand) (CommunityWriteResult, error) { + w.calls++ + rkey := SubjectRkey(cmd.PostURI) + w.repo.records[AcceptanceCollection+"/"+rkey] = &pds.RecordResponse{ + CID: "bafyacceptreal", + Value: map[string]any{"subject": map[string]any{"uri": cmd.PostURI, "cid": cmd.PostCID}}, + } + return CommunityWriteResult{RKey: rkey, CID: "bafyacceptreal"}, nil +} +func (w *memAcceptanceWriter) WriteRemoval(context.Context, CommunityRemovalCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, errors.New("unused") +} +func (w *memAcceptanceWriter) RestoreAcceptance(context.Context, CommunityWriteCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, errors.New("unused") +} +func (w *memAcceptanceWriter) RepinAcceptance(context.Context, CommunityWriteCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, errors.New("unused") +} +func (w *memAcceptanceWriter) DeleteAcceptance(context.Context, CommunityAcceptanceDeleteCommand) (CommunityWriteResult, error) { + return CommunityWriteResult{}, errors.New("unused") +} + +type memSource struct { + posts []LegacyPost + reads int + deletes int +} + +func (s *memSource) ListLegacyPosts(context.Context) ([]LegacyPost, error) { return s.posts, nil } +func (s *memSource) ReadLegacyPost(_ context.Context, uri string) (LegacyPost, bool, error) { + s.reads++ + for _, p := range s.posts { + if p.URI == uri { + return p, true, nil + } + } + return LegacyPost{}, false, nil +} +func (s *memSource) DeleteLegacyPost(_ context.Context, _ LegacyPost, swapCID string) error { + if swapCID == "" { + return errors.New("unguarded delete") + } + s.deletes++ + return nil +} + +type memLedger struct { + rows map[string]RematerializeLedgerRow + writes int +} + +func newMemLedger() *memLedger { return &memLedger{rows: map[string]RematerializeLedgerRow{}} } + +func (l *memLedger) Discover(_ context.Context, oldURI, communityDID, authorDID string) (RematerializeLedgerRow, error) { + if row, ok := l.rows[oldURI]; ok { + return row, nil + } + l.writes++ + row := RematerializeLedgerRow{ + OldURI: oldURI, State: RematerializeDiscovered, + CommunityDID: communityDID, AuthorDID: authorDID, + CreatedAt: time.Now(), UpdatedAt: time.Now(), + } + l.rows[oldURI] = row + return row, nil +} +func (l *memLedger) Get(_ context.Context, oldURI string) (RematerializeLedgerRow, bool, error) { + row, ok := l.rows[oldURI] + return row, ok, nil +} +func (l *memLedger) ListResumable(context.Context, string) ([]RematerializeLedgerRow, error) { + var out []RematerializeLedgerRow + for _, row := range l.rows { + if row.State != RematerializeDone && !IsFallback(row.State) { + out = append(out, row) + } + } + return out, nil +} +func (l *memLedger) RecordPostV2Written(_ context.Context, oldURI, sourceCID, newURI, newCID, newRkey string) error { + l.writes++ + row := l.rows[oldURI] + row.State = RematerializePostV2Written + row.SourceCID, row.NewURI, row.NewCID, row.NewRkey = sourceCID, newURI, newCID, newRkey + l.rows[oldURI] = row + return nil +} +func (l *memLedger) advance(oldURI string, to RematerializeState) error { + l.writes++ + row := l.rows[oldURI] + row.State = to + l.rows[oldURI] = row + return nil +} +func (l *memLedger) MarkVerified(_ context.Context, uri string) error { + return l.advance(uri, RematerializeVerified) +} +func (l *memLedger) MarkMigrated(_ context.Context, uri string) error { + return l.advance(uri, RematerializeMigrated) +} +func (l *memLedger) MarkDone(_ context.Context, uri string) error { + return l.advance(uri, RematerializeDone) +} +func (l *memLedger) MarkFallback(_ context.Context, uri string, state RematerializeState, _ string) error { + return l.advance(uri, state) +} +func (l *memLedger) ReopenFallback(context.Context, string) (int, error) { return 0, nil } +func (l *memLedger) CountByState(context.Context, string) (map[RematerializeState]int, error) { + counts := map[RematerializeState]int{} + for _, row := range l.rows { + counts[row.State]++ + } + return counts, nil +} + +type countingBlobClient struct { + fetches int + probes int + bytesFor map[string][]byte +} + +func (c *countingBlobClient) Fetch(_ context.Context, _, _, cid string) ([]byte, error) { + c.fetches++ + data, ok := c.bytesFor[cid] + if !ok { + return nil, fmt.Errorf("no such blob %s", cid) + } + return data, nil +} +func (c *countingBlobClient) Present(context.Context, string, string, string) (bool, error) { + c.probes++ + return true, nil +} + +// dryRunFixture wires a complete, working tool over in-memory seams. +func dryRunFixture() (*Rematerializer, *memSource, *memLedger, *memAuthorRepo, *memAcceptanceWriter, *countingBlobClient) { + communityDID := "did:plc:community2222222222222222" + authorDID := "did:plc:author11111111111111111" + blobCID := "bafkreiembeddedblobcid" + + legacy := LegacyPost{ + URI: "at://" + communityDID + "/" + LegacyPostCollection + "/3kdryrun", + CID: "bafylegacycid", + CommunityDID: communityDID, + AuthorDID: authorDID, + RawRecord: map[string]any{ + "$type": LegacyPostCollection, + "community": communityDID, + "author": authorDID, + "title": "a post with media", + "createdAt": "2026-01-02T03:04:05Z", + "embed": map[string]any{ + "$type": "social.coves.embed.images", + "images": []any{map[string]any{ + "alt": "a picture", + "image": map[string]any{"$type": "blob", "ref": map[string]any{"$link": blobCID}, "mimeType": "image/png"}, + }}, + }, + }, + } + + source := &memSource{posts: []LegacyPost{legacy}} + ledger := newMemLedger() + authorRepo := newMemAuthorRepo(authorDID) + communityRepo := &memCommunityRepo{did: communityDID, records: map[string]*pds.RecordResponse{}} + writer := &memAcceptanceWriter{repo: communityRepo} + blobClient := &countingBlobClient{bytesFor: map[string][]byte{blobCID: []byte("PNGDATA")}} + + tool := &Rematerializer{ + Source: source, + Ledger: ledger, + AuthorRepos: func(context.Context, string, *oauth.ClientSessionData) (AuthorRepo, error) { return authorRepo, nil }, + Acceptances: writer, + CommunityRepos: func(context.Context, string) (CommunityRepo, error) { + return communityRepo, nil + }, + Blobs: blobClient, + } + return tool, source, ledger, authorRepo, writer, blobClient +} + +// The rehearsal must reach the SAME verdict the real run does, having really +// resolved credentials, really re-read the record, and really fetched the blob. +func TestDryRun_WalksTheWholeCodePathAndMutatesNothing(t *testing.T) { + tool, source, ledger, authorRepo, writer, blobClient := dryRunFixture() + + dry := DryRunOf(tool) + report, err := dry.Run(context.Background()) + require.NoErrorf(t, err, "the dry run failed; a rehearsal that cannot complete tells the operator nothing about the real run") + + // It got all the way to the end. + assert.Equalf(t, 1, report.Done, + "the rehearsal did not carry the record to done. A dry run that stops early cannot tell the operator whether the real run would succeed") + assert.Truef(t, report.ScopeComplete, "the rehearsal must reach the same completion verdict the real run would") + + would, isDry := DryRunDeletes(dry) + require.True(t, isDry) + assert.Equalf(t, 1, would, "the rehearsal must report the delete it would have made; that count is the number the operator is being asked to authorise") + + // It really did the reads and the work. + assert.Positivef(t, source.reads, + "the rehearsal never RE-READ the legacy record. That read is where an edit landing after the listing is discovered, and rehearsing without it "+ + "means the dry run cannot find the exact problem a real run would hit") + assert.Positivef(t, blobClient.fetches, + "the rehearsal never FETCHED the embed blob. An unreachable or oversized blob is precisely the kind of failure a rehearsal exists to surface "+ + "before the destructive run") + assert.Positivef(t, blobClient.probes, "the rehearsal must still run the blob presence check that gates the delete") + + // And it mutated nothing. + assert.Zerof(t, authorRepo.puts, + "the rehearsal WROTE %d record(s) into the author's repo", authorRepo.puts) + assert.Zerof(t, authorRepo.uploads, + "the rehearsal UPLOADED %d blob(s) into the author's repo", authorRepo.uploads) + assert.Zerof(t, writer.calls, + "the rehearsal wrote %d acceptance(s) into the community's repo", writer.calls) + assert.Zerof(t, source.deletes, + "THE REHEARSAL DELETED %d LEGACY RECORD(S). This is the failure this whole flag exists to make impossible", source.deletes) + assert.Zerof(t, ledger.writes, + "the rehearsal wrote %d row(s) to the real ledger; a dry run that leaves ledger state behind makes the next REAL run skip work it never did", + ledger.writes) +} + +// A rehearsal must surface a stranded author, because that is the single most +// consequential thing an operator learns before committing. +func TestDryRun_StillFindsAStrandedAuthor(t *testing.T) { + tool, source, _, _, writer, _ := dryRunFixture() + tool.AuthorRepos = func(_ context.Context, did string, _ *oauth.ClientSessionData) (AuthorRepo, error) { + return nil, fmt.Errorf("no stored session for %s: %w", did, ErrNoAuthorCredentials) + } + + report, err := DryRunOf(tool).Run(context.Background()) + require.NoError(t, err) + + assert.Equalf(t, 1, report.Fallbacks, + "the rehearsal did not report the stranded author. Discovering at rehearsal time that an author's grant is gone is the difference between "+ + "re-authorizing them and finding out afterwards that their posts were left behind") + assert.Falsef(t, report.Complete, "a rehearsal with a stranded post must not report the migration complete") + assert.Zerof(t, source.deletes, "nothing may be deleted in a rehearsal") + assert.Zerof(t, writer.calls, "nothing may be written in a rehearsal") +} + +// A rehearsal must surface a RETRYABLE credential failure as a failed run, for +// the same reason the real one does. +func TestDryRun_StillFailsOnARetryableCredentialError(t *testing.T) { + tool, _, _, _, _, _ := dryRunFixture() + tool.AuthorRepos = func(_ context.Context, did string, _ *oauth.ClientSessionData) (AuthorRepo, error) { + return nil, fmt.Errorf("resuming %s: %w: connection refused", did, ErrAuthorCredentialsUnavailable) + } + + _, err := DryRunOf(tool).Run(context.Background()) + require.Errorf(t, err, + "the rehearsal swallowed a retryable credential failure. The operator would then confirm -yes on a rehearsal that had quietly skipped the "+ + "very authors the real run is about to fail on") +} + +// A rehearsal is not a real run, and DryRunDeletes must say so for one. +func TestDryRunDeletes_ReportsNothingForARealRun(t *testing.T) { + tool, _, _, _, _, _ := dryRunFixture() + _, isDry := DryRunDeletes(tool) + assert.Falsef(t, isDry, + "a REAL run reported itself as a dry run; the operator's console would then say 'nothing was written' after the records were deleted") +} diff --git a/internal/core/posts/rematerialize_guard_test.go b/internal/core/posts/rematerialize_guard_test.go new file mode 100644 index 0000000..b36599d --- /dev/null +++ b/internal/core/posts/rematerialize_guard_test.go @@ -0,0 +1,1001 @@ +//go:build integration + +package posts_test + +import ( + "context" + "database/sql" + "errors" + "fmt" + "testing" + + "Coves/internal/atproto/pds" + "Coves/internal/core/posts" + "Coves/internal/db/postgres" + "Coves/tests/testkit" + + "github.com/bluesky-social/indigo/atproto/auth/oauth" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// THE GUARDS ON THE IRREVERSIBLE STEP. +// +// rematerialize_test.go proves the state machine walks. This file proves the +// thing that actually matters: that the delete DOES NOT HAPPEN unless a +// replacement is provably standing, RIGHT NOW, on every path into it — including +// the resumed paths, which are the ones a real 2am run will take. +// +// Every test here is written against a specific way the tool could destroy a +// post, and each names that way in its failure message. If one of them ever goes +// red, the correct response is to stop, not to adjust the assertion. + +// guardHarness is one legacy post wired to a real ledger and faked repos, with +// every seam a test might need to break exposed. +type guardHarness struct { + t *testing.T + db *sql.DB + ledger posts.RematerializeLedger + authors *fakeAuthorFactory + writer *spyAcceptanceWriter + source *fakeLegacySource + tool *posts.Rematerializer + legacy posts.LegacyPost + rkey string + newURI string + newCID string +} + +func newGuardHarness(t *testing.T, authorDID string) *guardHarness { + t.Helper() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + authors := newFakeAuthorFactory() + authors.repo(authorDID) + writer := &spyAcceptanceWriter{} + legacy := legacyPost(t, rematCommunityDID, authorDID) + source := newFakeLegacySource(legacy) + + h := &guardHarness{ + t: t, db: db, ledger: ledger, authors: authors, writer: writer, source: source, legacy: legacy, + } + h.rkey = posts.RematerializeRkey(legacy.URI) + h.newURI = "at://" + authorDID + "/" + posts.PostV2Collection + "/" + h.rkey + h.newCID = deterministicCID(h.rkey) + h.tool = &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + } + return h +} + +func (h *guardHarness) state() posts.RematerializeState { + h.t.Helper() + row, found, err := h.ledger.Get(context.Background(), h.legacy.URI) + require.NoError(h.t, err) + require.True(h.t, found) + return row.State +} + +// dropLedgerRow removes the row entirely, forcing a genuine re-entry from +// `discovered` on the next pass. It is the only way to make the converge-by-read +// path actually execute against a repo that already holds the record. +func (h *guardHarness) dropLedgerRow() { + h.t.Helper() + _, err := h.db.ExecContext(context.Background(), + `DELETE FROM post_rematerialization_ledger WHERE old_uri = $1`, h.legacy.URI) + require.NoError(h.t, err) +} + +// ---- BLOCKER 2: the acceptance is READ BACK, never inferred ----------------- + +// The acceptance write's own result is computed from the inputs it was handed, +// so comparing it to those inputs is a tautology that cannot fail. Only a read +// of the COMMUNITY's repo can say whether the acceptance stands. +func TestRematerialize_AcceptanceThatDoesNotStand_IsNotAcceptedOnTheWritersWord(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + + // The writer reports a perfectly well-formed success — right rkey, right CID — + // and the community repo holds nothing. This is what a lost commit, a proxy + // that swallowed the write, or a bug in the writer looks like from here. + h.writer.resultOverride = &posts.CommunityWriteResult{ + URI: "at://" + rematCommunityDID + "/" + posts.AcceptanceCollection + "/" + posts.SubjectRkey(h.newURI), + RKey: posts.SubjectRkey(h.newURI), + CID: "bafyreiacceptancethatneverlanded", + } + h.writer.suppressStanding = true + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, + "the tool accepted the acceptance on the WRITER'S WORD. The writer computes its result from the arguments it was given, so checking that result "+ + "against those same arguments is SubjectRkey(X) != SubjectRkey(X) — a comparison that cannot fail. The acceptance must be read back out of the "+ + "community's repo, or the legacy record is deleted on the strength of a record that may not exist and the post drops out of its community") + assert.Equalf(t, 0, h.source.deleteCount(h.legacy.URI), + "the legacy record was deleted although no acceptance stands in the community repo") + assert.Equalf(t, posts.RematerializePostV2Written, h.state(), + "the row must stay at postv2_written: `verified` asserts the acceptance stands, and it does not") +} + +// A record standing at the deterministic acceptance key is not enough — it has +// to pin OUR postv2. The rkey is a digest of the subject URI, so a record at the +// right key naming the wrong subject means someone else's write landed there. +func TestRematerialize_AcceptanceNamingADifferentSubject_RefusesToDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + h.writer.afterWrite = func(w *spyAcceptanceWriter, cmd posts.CommunityWriteCommand) { + w.repinStanding(cmd.PostURI, "at://did:plc:someoneelse2222222222222/social.coves.community.postv2/other", "bafyreisomethingelse") + } + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, + "an acceptance naming a DIFFERENT subject was accepted. Finding a record at the deterministic key proves only that a record is there; the subject "+ + "strongRef is what says the community accepted THIS post") + assert.Equal(t, 0, h.source.deleteCount(h.legacy.URI)) + assert.Equal(t, posts.RematerializePostV2Written, h.state()) +} + +// The acceptance pinning a STALE CID is the same failure with a subtler shape: +// the community attests to a version of the post that is no longer there. +func TestRematerialize_AcceptancePinningADifferentCID_RefusesToDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + h.writer.afterWrite = func(w *spyAcceptanceWriter, cmd posts.CommunityWriteCommand) { + w.repinStanding(cmd.PostURI, cmd.PostURI, "bafyreiacidthepostnolongercarries") + } + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, + "an acceptance pinning a CID the postv2 does not carry was accepted; the community would be attesting to content that does not stand") + assert.Equal(t, 0, h.source.deleteCount(h.legacy.URI)) +} + +// TEST GAP 3: the read-back's ERROR branch. A transport failure reading the +// acceptance is not permission to proceed. +func TestRematerialize_AcceptanceReadFails_RefusesToDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + h.writer.readErr = errors.New("the community PDS returned 503 on getRecord") + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, + "a FAILED read of the acceptance was treated as a passing verification. 'I could not ask' is not 'it is there', and the difference is a deleted post") + assert.Equalf(t, 0, h.source.deleteCount(h.legacy.URI), + "the legacy record was deleted after the verification read failed") + assert.Equal(t, posts.RematerializePostV2Written, h.state()) +} + +// ---- TEST GAP 3: the postv2 read-back's error branch ------------------------ + +func TestRematerialize_PostV2ReadFails_RefusesToDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + h.authors.repo(rematAuthorDID).getErrAt = map[string]error{ + h.rkey: errors.New("the author's PDS returned 503 on getRecord"), + } + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, + "a FAILED read of the postv2 was treated as a passing verification; substituting the ledger's remembered CID for a read that did not happen "+ + "is exactly the mutation this test exists to catch") + assert.Equal(t, 0, h.source.deleteCount(h.legacy.URI)) +} + +// The postv2 DELETED between the write and the verify — an author withdrawing +// the post mid-migration, or a bug elsewhere. +func TestRematerialize_PostV2DeletedBetweenWriteAndVerify_RefusesToDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + h.authors.repo(rematAuthorDID).deleteOnGet = map[string]bool{h.rkey: true} + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, + "the postv2 was gone at verify time and the legacy record was deleted anyway; that destroys the only surviving copy of the post") + assert.Equal(t, 0, h.source.deleteCount(h.legacy.URI)) + assert.Equal(t, posts.RematerializePostV2Written, h.state()) +} + +// ---- TEST GAP 5: the acceptance-WRITE failure paths ------------------------- + +func TestRematerialize_AcceptanceWriteFails_NoCheckpointNoDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + h.writer.writeErr = errors.New("the community PDS returned 502 on putRecord") + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, "a failed acceptance write must surface as an error, not be swallowed") + assert.Equalf(t, 0, h.source.deleteCount(h.legacy.URI), + "the legacy record was deleted although the acceptance write failed") + assert.Equalf(t, posts.RematerializePostV2Written, h.state(), + "the row must stop at postv2_written — the postv2 exists, the acceptance does not") +} + +func TestRematerialize_AcceptanceWriteReportsNothing_NoDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + // An empty result: no URI, no rkey, no CID. Nothing was written. + h.writer.resultOverride = &posts.CommunityWriteResult{} + h.writer.suppressStanding = true + + _, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.Errorf(t, err, "an acceptance write reporting nothing at all must never license a delete") + assert.Equal(t, 0, h.source.deleteCount(h.legacy.URI)) +} + +// ---- BLOCKER 4: the legacy record must not have changed --------------------- + +// The listing snapshots every body at t0; the delete happens minutes to hours +// later. An edit landing in that gap is content that was never re-materialized. +func TestRematerialize_LegacyRecordEditedAfterConversion_RefusesToDelete(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + ctx := context.Background() + + // Drive to the migrated checkpoint by failing the first delete, so the row is + // staged exactly where a crash-before-delete leaves it. + h.source.deleteErr[h.legacy.URI] = errors.New("transient: 502 on delete") + _, err := h.tool.RematerializeOne(ctx, h.legacy) + require.Error(t, err) + require.Equal(t, posts.RematerializeMigrated, h.state()) + deletesBefore := h.source.deleteCount(h.legacy.URI) + + // NOW an edit lands on the legacy record: same URI, new CID. + h.source.setCurrentCID(h.legacy.URI, "bafyreianeditthatlandedaftert0") + + _, err = h.tool.RematerializeOne(ctx, h.legacy) + require.Errorf(t, err, + "the tool deleted a legacy record whose CID had changed since the postv2 was built from it. The maintenance window's 'stop writers' is never "+ + "perfect — an aggregator cron or a cached mobile session lands an edit — and deleting here destroys the newer content with no trace while the "+ + "run reports clean. Re-read the record and compare its CID before every delete") + assert.Equalf(t, deletesBefore, h.source.deleteCount(h.legacy.URI), + "no further delete may be attempted once the legacy record is known to have changed") + assert.Equalf(t, posts.RematerializeMigrated, h.state(), + "the row stays at the migrated checkpoint so a later pass can re-verify once the writer is stopped") +} + +// The delete must ALSO carry the source CID as the PDS's own swap guard: the +// tool's check and its delete are two moments, and only the PDS can make them +// one. +func TestRematerialize_Delete_IsGuardedBySourceCID(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + + state, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.NoError(t, err) + require.Equal(t, posts.RematerializeDone, state) + + guards := h.source.swapGuards() + require.Lenf(t, guards, 1, "exactly one delete must have been issued") + assert.Equalf(t, h.legacy.CID, guards[0], + "the delete was sent with swap guard %q, not the legacy record's own CID %q. The guard is what makes the PDS refuse a delete of a version the "+ + "tool never saw", guards[0], h.legacy.CID) +} + +// ---- BLOCKER 6: re-verification on EVERY resumed path ----------------------- + +// A row at `verified` records a check that passed at a moment now in the past. +// If the acceptance was withdrawn in the gap, the resumed run must notice. +func TestRematerialize_ResumeAtVerified_ReVerifiesTheAcceptance(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + ctx := context.Background() + + // Stage the row exactly at `verified`, with the postv2 standing and the + // acceptance standing — the state a crash right after MarkVerified leaves. + _, err := h.ledger.Discover(ctx, h.legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, h.ledger.RecordPostV2Written(ctx, h.legacy.URI, h.legacy.CID, h.newURI, h.newCID, h.rkey)) + require.NoError(t, h.ledger.MarkVerified(ctx, h.legacy.URI)) + h.authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, h.rkey) + h.writer.seedStandingAcceptance(rematCommunityDID, h.newURI, h.newCID) + + // ...and THEN the acceptance is withdrawn, as a moderator action or a bug + // might do while the operator was asleep. + h.writer.withdrawStanding(h.newURI) + + _, err = h.tool.RematerializeOne(ctx, h.legacy) + require.Errorf(t, err, + "a row resumed at `verified` was deleted with NO new reads. `verified` is a memory of a check, not a licence to destroy: the acceptance can be "+ + "withdrawn, the postv2 edited, in the gap the crash opened. Verification must be a fresh read on every path") + assert.Equalf(t, 0, h.source.deleteCount(h.legacy.URI), + "the legacy record was deleted although the acceptance no longer stands") +} + +// The same for `migrated`, which is one step closer to the delete and therefore +// the more dangerous of the two. +func TestRematerialize_ResumeAtMigrated_ReVerifiesThePostV2(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + ctx := context.Background() + + _, err := h.ledger.Discover(ctx, h.legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, h.ledger.RecordPostV2Written(ctx, h.legacy.URI, h.legacy.CID, h.newURI, h.newCID, h.rkey)) + require.NoError(t, h.ledger.MarkVerified(ctx, h.legacy.URI)) + require.NoError(t, h.ledger.MarkMigrated(ctx, h.legacy.URI)) + h.authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, h.rkey) + h.writer.seedStandingAcceptance(rematCommunityDID, h.newURI, h.newCID) + + // The postv2 was edited in the gap: it now carries a CID the acceptance does + // not pin. + h.authors.repo(rematAuthorDID).getCIDAt[h.rkey] = "bafyreitheposthasbeenedited" + + _, err = h.tool.RematerializeOne(ctx, h.legacy) + require.Errorf(t, err, + "a row resumed at `migrated` deleted the legacy record without re-reading the postv2. The checkpoint means 'the delete was safe when we wrote "+ + "this', and the whole reason the checkpoint exists is that the process then died") + assert.Equal(t, 0, h.source.deleteCount(h.legacy.URI)) +} + +// TEST GAP 8: every crash boundary, with the repo state the crash would have +// left, driven to its correct terminal outcome. +func TestRematerialize_ResumesCorrectlyFromEveryStepBoundary(t *testing.T) { + t.Parallel() + + boundaries := []struct { + name string + // seed stages the ledger and the repos as a crash at this boundary would + // have left them. + seed func(t *testing.T, h *guardHarness) + wantState posts.RematerializeState + // wantDeletes is how many delete attempts the resumed run should make. + wantDeletes int + // wantAcceptanceWrites is how many NEW acceptance writes it should make. + wantAcceptanceWrites int + }{ + { + name: "crashed before anything was written (discovered)", + seed: func(*testing.T, *guardHarness) {}, + wantState: posts.RematerializeDone, + wantDeletes: 1, + wantAcceptanceWrites: 1, + }, + { + name: "crashed after the postv2 write, before the acceptance (postv2_written)", + seed: func(t *testing.T, h *guardHarness) { + ctx := context.Background() + _, err := h.ledger.Discover(ctx, h.legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, h.ledger.RecordPostV2Written(ctx, h.legacy.URI, h.legacy.CID, h.newURI, h.newCID, h.rkey)) + h.authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, h.rkey) + }, + wantState: posts.RematerializeDone, + wantDeletes: 1, + wantAcceptanceWrites: 1, + }, + { + name: "crashed after the acceptance, before the verify checkpoint (postv2_written, acceptance standing)", + seed: func(t *testing.T, h *guardHarness) { + ctx := context.Background() + _, err := h.ledger.Discover(ctx, h.legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, h.ledger.RecordPostV2Written(ctx, h.legacy.URI, h.legacy.CID, h.newURI, h.newCID, h.rkey)) + h.authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, h.rkey) + h.writer.seedStandingAcceptance(rematCommunityDID, h.newURI, h.newCID) + }, + wantState: posts.RematerializeDone, + wantDeletes: 1, + // The acceptance already stands, but the tool re-fires the write; the + // real writer SKIPS in that case and mints no new CID. What must not + // happen is a second acceptance RECORD. + wantAcceptanceWrites: 1, + }, + { + name: "crashed after the verify checkpoint, before migrated (verified)", + seed: func(t *testing.T, h *guardHarness) { + ctx := context.Background() + _, err := h.ledger.Discover(ctx, h.legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, h.ledger.RecordPostV2Written(ctx, h.legacy.URI, h.legacy.CID, h.newURI, h.newCID, h.rkey)) + require.NoError(t, h.ledger.MarkVerified(ctx, h.legacy.URI)) + h.authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, h.rkey) + h.writer.seedStandingAcceptance(rematCommunityDID, h.newURI, h.newCID) + }, + wantState: posts.RematerializeDone, + wantDeletes: 1, + wantAcceptanceWrites: 0, + }, + { + name: "crashed after the migrated checkpoint, before the delete (migrated)", + seed: func(t *testing.T, h *guardHarness) { + ctx := context.Background() + _, err := h.ledger.Discover(ctx, h.legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, h.ledger.RecordPostV2Written(ctx, h.legacy.URI, h.legacy.CID, h.newURI, h.newCID, h.rkey)) + require.NoError(t, h.ledger.MarkVerified(ctx, h.legacy.URI)) + require.NoError(t, h.ledger.MarkMigrated(ctx, h.legacy.URI)) + h.authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, h.rkey) + h.writer.seedStandingAcceptance(rematCommunityDID, h.newURI, h.newCID) + }, + wantState: posts.RematerializeDone, + wantDeletes: 1, + wantAcceptanceWrites: 0, + }, + { + name: "crashed after the delete, before MarkDone (migrated, record already gone)", + seed: func(t *testing.T, h *guardHarness) { + ctx := context.Background() + _, err := h.ledger.Discover(ctx, h.legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, h.ledger.RecordPostV2Written(ctx, h.legacy.URI, h.legacy.CID, h.newURI, h.newCID, h.rkey)) + require.NoError(t, h.ledger.MarkVerified(ctx, h.legacy.URI)) + require.NoError(t, h.ledger.MarkMigrated(ctx, h.legacy.URI)) + h.authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, h.rkey) + h.writer.seedStandingAcceptance(rematCommunityDID, h.newURI, h.newCID) + h.source.markGone(h.legacy.URI) + }, + wantState: posts.RematerializeDone, + // The record is ALREADY gone. A resumed run must not issue a delete it + // does not owe — it must simply finish the ledger. + wantDeletes: 0, + wantAcceptanceWrites: 0, + }, + } + + for _, tc := range boundaries { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + tc.seed(t, h) + + state, err := h.tool.RematerializeOne(context.Background(), h.legacy) + require.NoErrorf(t, err, "a run resumed at this boundary must finish the record, not fail") + + assert.Equalf(t, tc.wantState, state, "the resumed run reached the wrong terminal state") + assert.Equalf(t, tc.wantDeletes, h.source.deleteCount(h.legacy.URI), + "the resumed run issued %d delete(s), want %d — a resume must re-do exactly the steps its predecessor owed and no others", + h.source.deleteCount(h.legacy.URI), tc.wantDeletes) + assert.Equalf(t, tc.wantAcceptanceWrites, len(h.writer.calls()), + "the resumed run made %d acceptance write(s), want %d", len(h.writer.calls()), tc.wantAcceptanceWrites) + assert.Equalf(t, 1, h.authors.repo(rematAuthorDID).recordCount(), + "exactly one postv2 must stand; a resume that mints a second dangles every strongRef built from the first") + assert.Equalf(t, 1, h.writer.acceptanceCount(), + "exactly one acceptance must stand") + }) + } +} + +// ---- TEST GAP 1: genuine re-entry, so converge-by-read actually runs -------- + +// The "re-run is a no-op" tests return early on a `done` ledger row, making ZERO +// PDS calls — so createAuthorRecord's converge-by-read and sameRecordBody are +// never executed on the success path at any tier. Dropping the ledger row is +// what forces the tool back through step 1 against a repo that already holds the +// record, which is the only way that branch runs. +func TestRematerialize_ReEntryWithNoLedgerRow_ConvergesOnTheStandingPostV2(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + ctx := context.Background() + + // Pass 1 stops after the postv2 is written, so the legacy record still stands + // and the repo already holds a postv2 at the deterministic rkey. + h.writer.writeErr = errors.New("the community PDS returned 502 on the acceptance") + _, err := h.tool.RematerializeOne(ctx, h.legacy) + require.Error(t, err) + require.Equal(t, posts.RematerializePostV2Written, h.state()) + + firstCID := h.authors.repo(rematAuthorDID).standingCID(h.rkey) + require.NotEmpty(t, firstCID) + + // Now lose the ledger entirely. The next pass re-enters at `discovered` and + // must meet its own first attempt rather than minting a second post. + h.dropLedgerRow() + + state, err := h.tool.RematerializeOne(ctx, h.legacy) + require.NoErrorf(t, err, + "a re-entry with no ledger row failed. This is the converge-by-read path: the create-only put meets ErrSwapConflict, the standing record is read "+ + "back, and its body is compared to the intended conversion. If sameRecordBody stopped comparing bodies faithfully, this is where it shows") + assert.Equal(t, posts.RematerializeDone, state) + + assert.Equalf(t, 1, h.authors.repo(rematAuthorDID).recordCount(), + "a re-entry minted a SECOND postv2. The deterministic rkey exists precisely so that a re-run converges on the record its first attempt wrote") + assert.Equalf(t, firstCID, h.authors.repo(rematAuthorDID).standingCID(h.rkey), + "the postv2's CID changed across a re-entry; every strongRef built from the first record now dangles") + assert.Equalf(t, 1, h.writer.acceptanceCount(), + "exactly one acceptance record must stand after a re-entry") +} + +// TEST GAP 9: a re-run over a completed record must make EXACTLY no new calls — +// "at least one acceptance write" is satisfied by a second one. +func TestRematerialize_ReRunOverADoneRow_MakesExactlyNoNewCalls(t *testing.T) { + t.Parallel() + h := newGuardHarness(t, rematAuthorDID) + ctx := context.Background() + + _, err := h.tool.RematerializeOne(ctx, h.legacy) + require.NoError(t, err) + + acceptancesAfterFirst := len(h.writer.calls()) + deletesAfterFirst := h.source.deleteCount(h.legacy.URI) + acceptanceCIDAfterFirst := h.writer.standingCID(h.newURI) + require.Equal(t, 1, acceptancesAfterFirst) + require.Equal(t, 1, deletesAfterFirst) + + state, err := h.tool.RematerializeOne(ctx, h.legacy) + require.NoError(t, err) + assert.Equal(t, posts.RematerializeDone, state) + + assert.Equalf(t, acceptancesAfterFirst, len(h.writer.calls()), + "a re-run over a done row wrote another acceptance (%d → %d). A second write mints a fresh record CID and invalidates every reference to the one "+ + "it replaced", acceptancesAfterFirst, len(h.writer.calls())) + assert.Equalf(t, deletesAfterFirst, h.source.deleteCount(h.legacy.URI), + "a re-run over a done row attempted another delete (%d → %d)", deletesAfterFirst, h.source.deleteCount(h.legacy.URI)) + assert.Equalf(t, acceptanceCIDAfterFirst, h.writer.standingCID(h.newURI), + "the standing acceptance's CID changed across a re-run") + assert.Equalf(t, 1, h.authors.repo(rematAuthorDID).recordCount(), "exactly one postv2 must stand") +} + +// ---- BLOCKER 5: the -community scope is enforced on the destructive path ---- + +// The reconcile pass drives rows the discovery pass never listed. Unscoped, a +// staged run for community A resumes — and deletes — community B's records. +func TestRematerialize_ScopedRun_RefusesARecordFromAnotherCommunity(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + authors := newFakeAuthorFactory() + authors.repo(rematAuthorDID) + writer := &spyAcceptanceWriter{} + + otherCommunity := "did:plc:othercommunityotherco1" + foreign := legacyPost(t, otherCommunity, rematAuthorDID) + source := newFakeLegacySource(foreign) + + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + CommunityScope: rematCommunityDID, + } + + _, err := tool.RematerializeOne(context.Background(), foreign) + require.Errorf(t, err, + "a run scoped to %s processed a record belonging to %s. The staged rollout exists so that one community's posts can be migrated and DELETED while "+ + "every other community is untouched; a scope applied only at discovery does not do that, because the ledger reconcile pass reaches rows "+ + "discovery never listed", rematCommunityDID, otherCommunity) + assert.Equalf(t, 0, source.deleteCount(foreign.URI), + "a record outside the run's scope was deleted") +} + +// A staged run finishing its own scope is a SUCCESS, reported separately from +// "the whole migration is done". Collapsing them makes every staged run exit +// non-zero and trains the operator to ignore the only machine-checkable gate on +// the irreversible §11 step 6. +func TestRematerialize_ScopedRun_ReportsScopeAndWholeMigrationSeparately(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + ctx := context.Background() + + // Another community's row is already on the ledger and NOT done: the whole + // migration is unfinished. + otherCommunity := "did:plc:othercommunityotherco2" + otherURI := "at://" + otherCommunity + "/social.coves.community.post/" + testkit.TID() + _, err := ledger.Discover(ctx, otherURI, otherCommunity, rematAuthorDID) + require.NoError(t, err) + + authors := newFakeAuthorFactory() + authors.repo(rematAuthorDID) + writer := &spyAcceptanceWriter{} + mine := legacyPost(t, rematCommunityDID, rematAuthorDID) + source := newFakeLegacySource(mine) + + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + CommunityScope: rematCommunityDID, + } + + report, err := tool.Run(ctx) + require.NoError(t, err) + + assert.Truef(t, report.ScopeComplete, + "the run finished every post in its scope but did not report ScopeComplete. A staged run that always reports failure is a staged run whose exit "+ + "code the operator learns to ignore — and that exit code is the gate on the irreversible legacy-removal step") + assert.Equalf(t, rematCommunityDID, report.CommunityScope, "the report must name the scope it describes") + assert.Equalf(t, 1, report.Discovered, "the scoped census must count only this community's rows, got %d", report.Discovered) + + assert.Falsef(t, report.Complete, + "the run reported the WHOLE MIGRATION complete while another community still has an unfinished row. Complete is what gates the irreversible "+ + "legacy-removal follow-up, and a scoped census cannot see outside its scope") + assert.GreaterOrEqualf(t, report.GlobalDiscovered, 2, + "the global census must count rows outside the run's scope, got %d", report.GlobalDiscovered) +} + +// ---- COMPLETION IS GATED ON A FINAL SOURCE RE-SCAN -------------------------- + +// A completion signal computed only from rows the run already discovered is +// circular: it cannot see a record written after the discovery pass. +func TestRematerialize_Complete_IsGatedOnARescanOfTheSource(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + authors := newFakeAuthorFactory() + authors.repo(rematAuthorDID) + writer := &spyAcceptanceWriter{} + + first := legacyPost(t, rematCommunityDID, rematAuthorDID) + late := legacyPost(t, rematCommunityDID, rematAuthorDID) + source := newFakeLegacySource(first) + // A record that appears only AFTER the discovery pass — a writer the + // maintenance window did not stop. + source.appendOnNextList(late) + + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + } + + report, err := tool.Run(context.Background()) + require.NoError(t, err) + + assert.Equalf(t, 1, report.RemainingLegacy, + "the final re-scan did not see the legacy record that appeared during the run, so 'complete' was computed from the tool's own memory of what it "+ + "had discovered — which can never contradict itself") + assert.Falsef(t, report.Complete, + "the run reported the migration complete while a legacy record still stands in the source. Complete gates the IRREVERSIBLE removal of the legacy "+ + "read/ingest surfaces; a post still living only as a community.post would silently vanish") +} + +// ---- TEST GAP 6: the never-forge property, proven against WRITABLE others --- + +// The original never-forge test held a factory with NO repos at all, so "did not +// re-author under another identity" was proven in a world where nothing was +// writable. Here the community's and the instance's repos ARE writable, and the +// assertion is positive: nothing but the author's repo received a write. +func TestRematerialize_NoCredentials_WritesNothingEvenWhenOtherIdentitiesAreWritable(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + writer := &spyAcceptanceWriter{} + + humanDID := "did:plc:humanhumanhumanhumanhum" + instanceDID := "did:plc:instanceinstanceinstan" + + // A factory that WOULD hand out a perfectly writable repo for the community + // and for the instance — the two identities a forging implementation would + // reach for — and refuses only the author's own. + authors := newFakeAuthorFactory() + authors.repo(rematCommunityDID) + authors.repo(instanceDID) + authors.noCreds[humanDID] = true + + legacy := legacyPost(t, rematCommunityDID, humanDID) + source := newFakeLegacySource(legacy) + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + } + + state, err := tool.RematerializeOne(context.Background(), legacy) + require.NoError(t, err, "a no-creds record is an expected terminal outcome, not a run-failing error") + require.Equal(t, posts.RematerializeFallbackLeftLegacy, state) + + for did, repo := range authors.repos { + assert.Equalf(t, 0, repo.recordCount(), + "a record was written into the repo of %s for a post whose own author could not be restored. Re-authoring under ANY other identity — the "+ + "community, the instance, an admin — is the §2 impersonation the whole author-owned flip exists to remove", did) + } + assert.Emptyf(t, writer.calls(), "no acceptance may be written for a post that was never re-authored") + assert.Equalf(t, 0, source.deleteCount(legacy.URI), + "the old community.post must SURVIVE: with no valid postv2 to replace it, deleting it destroys the post outright") +} + +// ---- BLOCKER 3: a retryable credential failure fails the run ---------------- + +// A transient failure resolving credentials must NOT be written to the ledger as +// a terminal fallback. One aggregator authors most of production; a single blip +// recorded as a verdict sentences the whole corpus with no in-tool way back. +func TestRematerialize_RetryableCredentialFailure_FailsTheRunAndSentencesNothing(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + writer := &spyAcceptanceWriter{} + aggregatorDID := "did:plc:aggregatoraggregatoragg" + + authors := newFakeAuthorFactory() + authors.retryable[aggregatorDID] = true + + one := legacyPost(t, rematCommunityDID, aggregatorDID) + two := legacyPost(t, rematCommunityDID, aggregatorDID) + source := newFakeLegacySource(one, two) + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + } + + _, err := tool.Run(context.Background()) + require.Errorf(t, err, + "a RETRYABLE credential failure was swallowed. It must fail the run loudly: the alternative is recording a terminal verdict on the strength of a "+ + "network blip, over every post the aggregator ever wrote") + + // The run stops at the first record, so the second may have no row at all. + // What must be true of EVERY row that does exist is that none was sentenced. + for _, legacy := range []posts.LegacyPost{one, two} { + row, found, err := ledger.Get(context.Background(), legacy.URI) + require.NoError(t, err) + if !found { + continue + } + assert.Equalf(t, posts.RematerializeDiscovered, row.State, + "a transient credential failure left %s at %s. fallback_left_legacy is TERMINAL — ListResumable excludes it and MarkFallback will not "+ + "re-open it — so a blip written there is a permanent no-op over that post, forever", legacy.URI, row.State) + assert.Falsef(t, posts.IsFallback(row.State), + "a network blip was written as a terminal verdict on %s", legacy.URI) + } +} + +// The census can be made to STOP before mutating anything, which is the +// operator's defence against a run that "succeeded" having migrated almost +// nothing. +func TestRematerialize_AbortOnFallback_StopsBeforeAnyRepoIsMutated(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + writer := &spyAcceptanceWriter{} + authors := newFakeAuthorFactory() + + goodDID := "did:plc:hascredshascredshascreds" + strandedDID := "did:plc:nocredsnocredsnocredsnoc" + authors.repo(goodDID) + authors.noCreds[strandedDID] = true + + good := legacyPost(t, rematCommunityDID, goodDID) + stranded := legacyPost(t, rematCommunityDID, strandedDID) + source := newFakeLegacySource(good, stranded) + + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + AbortOnFallback: true, + } + + _, err := tool.Run(context.Background()) + require.Errorf(t, err, "with AbortOnFallback set, a census that stranded a post must stop the run") + assert.Containsf(t, err.Error(), strandedDID, + "the abort message must name the stranded author(s); the operator's next action is to re-authorize them") + + assert.Equalf(t, 0, source.deleteCount(good.URI), + "the run mutated a repo after the census had already found a stranded post; the whole point of aborting is that nothing has happened yet") + assert.Equalf(t, 0, authors.repo(goodDID).recordCount(), "no postv2 may be written once the run has decided to abort") + assert.Emptyf(t, writer.calls(), "no acceptance may be written once the run has decided to abort") +} + +// ---- the fallback recovery path -------------------------------------------- + +// A fallback row is terminal, deliberately. It must not be IRREVERSIBLE: the +// operator re-authorizes the author and needs a supported way to retry. +func TestRematerialize_ReopenFallback_LetsAReAuthorizedAuthorBeRetried(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + ctx := context.Background() + authors := newFakeAuthorFactory() + writer := &spyAcceptanceWriter{} + + authorDID := "did:plc:reauthorizedreauthoriz1" + authors.noCreds[authorDID] = true + legacy := legacyPost(t, rematCommunityDID, authorDID) + source := newFakeLegacySource(legacy) + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + } + + _, err := tool.Run(ctx) + require.NoError(t, err) + row, _, err := ledger.Get(ctx, legacy.URI) + require.NoError(t, err) + require.Equal(t, posts.RematerializeFallbackLeftLegacy, row.State) + + // The operator re-authorizes the author, then reopens. + delete(authors.noCreds, authorDID) + authors.repo(authorDID) + + moved, err := ledger.ReopenFallback(ctx, rematCommunityDID) + require.NoError(t, err) + assert.Equalf(t, 1, moved, "ReopenFallback must report how many rows it moved, so the operator can tell it did anything at all") + + // The documented workflow is `-reopen-fallbacks` and then RE-RUN THE BINARY, so + // the retry is a fresh process: a terminal credential verdict is cached for the + // life of one run on purpose, and re-resolving it mid-run would mean thousands + // of extra token rotations for an author who has none. + retry := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), + } + state, err := retry.RematerializeOne(ctx, legacy) + require.NoErrorf(t, err, + "a reopened row could not be retried. Without a way back out of fallback_left_legacy, a single expired aggregator grant makes every subsequent "+ + "run a permanent no-op and the only remedy is hand-written SQL against a production table") + assert.Equal(t, posts.RematerializeDone, state) +} + +func TestRematerialize_ReopenFallback_NeverResurrectsADoneRow(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + h := newGuardHarness(t, rematAuthorDID) + _ = ledger + ctx := context.Background() + + _, err := h.tool.RematerializeOne(ctx, h.legacy) + require.NoError(t, err) + require.Equal(t, posts.RematerializeDone, h.state()) + + moved, err := h.ledger.ReopenFallback(ctx, rematCommunityDID) + require.NoError(t, err) + assert.Equalf(t, 0, moved, "ReopenFallback moved a row that was not a fallback") + assert.Equalf(t, posts.RematerializeDone, h.state(), + "a done row was moved back out of done. `done` means the legacy record has been DELETED; re-running against it cannot bring the record back, so "+ + "nothing may ever move a row out of it") +} + +// ---- blob handling --------------------------------------------------------- + +// The blob presence probe's error branch: "I could not ask" is not "it is not +// there", and neither is permission to delete the community's only copy. +func TestRematerialize_BlobProbeFails_RefusesToDelete(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + authors := newFakeAuthorFactory() + authors.repo(rematAuthorDID) + writer := &spyAcceptanceWriter{} + + legacy := legacyPostWithBlob(t, rematCommunityDID, rematAuthorDID) + source := newFakeLegacySource(legacy) + blobClient := &fakeBlobClient{ + bytes: map[string][]byte{embeddedBlobCID(): embeddedBlobBytes}, + presentErr: errors.New("the author's PDS returned 503 on getBlob"), + } + + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), Blobs: blobClient, + } + + _, err := tool.RematerializeOne(context.Background(), legacy) + require.Errorf(t, err, + "a FAILED blob presence probe was treated as either answer. Reported as absence it refuses a healthy record; reported as presence it licenses "+ + "deleting the last record that keeps the community's only copy of the bytes alive") + assert.Equal(t, 0, source.deleteCount(legacy.URI)) +} + +// The blob bytes come from the COMMUNITY's PDS — the repo that actually holds +// them — not from the author's host, which is the same machine only for as long +// as every account on the instance shares one PDS. +func TestRematerialize_FetchesCommunityBlobsFromTheCommunitysHost(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + authors := newFakeAuthorFactory() + authors.repo(rematAuthorDID).host = "http://author-pds.invalid" + writer := &spyAcceptanceWriter{communityHost: "http://community-pds.invalid"} + + legacy := legacyPostWithBlob(t, rematCommunityDID, rematAuthorDID) + source := newFakeLegacySource(legacy) + blobClient := &fakeBlobClient{bytes: map[string][]byte{embeddedBlobCID(): embeddedBlobBytes}} + + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), Blobs: blobClient, + } + + _, err := tool.RematerializeOne(context.Background(), legacy) + require.NoError(t, err) + + require.NotEmpty(t, blobClient.fetches, "no blob was fetched at all") + assert.Equalf(t, "http://community-pds.invalid", blobClient.fetches[0].host, + "the community's blob was fetched from %q — the AUTHOR's PDS. A blob lives in the repo that holds it, and fetching the community's bytes from "+ + "the author's host works only while both accounts happen to share one PDS", blobClient.fetches[0].host) + assert.Equalf(t, rematCommunityDID, blobClient.fetches[0].did, "the blob must be fetched from the community's repo") +} + +// The blob presence check must run on a RESUMED path too. Running it only in the +// first-pass branch meant a resume re-entering at postv2_written deleted the +// legacy record — and the last reference keeping the community's blobs alive — +// having checked nothing about the media. +func TestRematerialize_ResumeAtPostV2Written_StillVerifiesTheBlobs(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := postgres.NewRematerializeLedger(db) + ctx := context.Background() + authors := newFakeAuthorFactory() + authors.repo(rematAuthorDID) + writer := &spyAcceptanceWriter{} + + legacy := legacyPostWithBlob(t, rematCommunityDID, rematAuthorDID) + source := newFakeLegacySource(legacy) + rkey := posts.RematerializeRkey(legacy.URI) + newURI := "at://" + rematAuthorDID + "/" + posts.PostV2Collection + "/" + rkey + newCID := deterministicCID(rkey) + + _, err := ledger.Discover(ctx, legacy.URI, rematCommunityDID, rematAuthorDID) + require.NoError(t, err) + require.NoError(t, ledger.RecordPostV2Written(ctx, legacy.URI, legacy.CID, newURI, newCID, rkey)) + authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, rkey) + + // The blob is NOT in the author's repo — the first pass died before the + // upload landed. + blobClient := &fakeBlobClient{bytes: map[string][]byte{embeddedBlobCID(): embeddedBlobBytes}, absent: true} + + tool := &posts.Rematerializer{ + Source: source, Ledger: ledger, AuthorRepos: authors.factory(), + Acceptances: writer, CommunityRepos: writer.repos(), Blobs: blobClient, + } + + _, err = tool.RematerializeOne(ctx, legacy) + require.Errorf(t, err, + "a resume re-entering at postv2_written deleted the legacy record without checking that its media had been copied. The blob check ran only in "+ + "the first-pass branch, so exactly the run that crashed mid-upload was the one that skipped it — and the community's copy becomes "+ + "garbage-collectable the moment the legacy record goes") + assert.Equal(t, 0, source.deleteCount(legacy.URI)) +} + +// ---- fixtures -------------------------------------------------------------- + +// legacyPostWithBlob stages a legacy record whose embed references a blob living +// in the community's blob store. +// The blob's CID is the one a CONTENT-ADDRESSED store mints for these exact +// bytes, which is the property the whole "carry the ref through unchanged" +// design rests on: re-uploading identical bytes yields the identical CID, so the +// postv2 may keep the community's reference verbatim. +var embeddedBlobBytes = []byte("PNGDATA-embedded") + +func embeddedBlobCID() string { return blobCIDFor(embeddedBlobBytes) } + +func legacyPostWithBlob(t *testing.T, communityDID, authorDID string) posts.LegacyPost { + t.Helper() + legacy := legacyPost(t, communityDID, authorDID) + legacy.RawRecord["embed"] = map[string]any{ + "$type": "social.coves.embed.images", + "images": []any{map[string]any{ + "alt": "a picture", + "image": map[string]any{ + "$type": "blob", + "ref": map[string]any{"$link": embeddedBlobCID()}, + "mimeType": "image/png", + "size": float64(len(embeddedBlobBytes)), + }, + }}, + } + return legacy +} + +// fakeBlobClient records what was fetched from where, and can fail or answer +// absent on demand. +type fakeBlobClient struct { + bytes map[string][]byte + fetches []blobFetch + presentErr error + absent bool +} + +type blobFetch struct { + host string + did string + cid string +} + +func (c *fakeBlobClient) Fetch(_ context.Context, host, did, cid string) ([]byte, error) { + c.fetches = append(c.fetches, blobFetch{host: host, did: did, cid: cid}) + data, ok := c.bytes[cid] + if !ok { + return nil, fmt.Errorf("no such blob %s", cid) + } + return data, nil +} + +func (c *fakeBlobClient) Present(_ context.Context, _, _, _ string) (bool, error) { + if c.presentErr != nil { + return false, c.presentErr + } + return !c.absent, nil +} + +// unusedOAuthSession keeps the indigo oauth import honest for the factory +// signature the fakes satisfy. +var _ = func(_ *oauth.ClientSessionData) {} + +// unusedPDSTypes keeps the pds import honest across build-tag permutations. +var _ = pds.ErrNotFound diff --git a/internal/core/posts/rematerialize_outer_test.go b/internal/core/posts/rematerialize_outer_test.go index 44613ce..c71abba 100644 --- a/internal/core/posts/rematerialize_outer_test.go +++ b/internal/core/posts/rematerialize_outer_test.go @@ -4,6 +4,7 @@ package posts_test import ( "context" + "fmt" "net/http" "net/url" "strings" @@ -64,9 +65,42 @@ func (s *realLegacySource) ListLegacyPosts(_ context.Context) ([]posts.LegacyPos return s.staged, nil } -func (s *realLegacySource) DeleteLegacyPost(ctx context.Context, legacy posts.LegacyPost) error { +// ReadLegacyPost re-reads the record from the REAL community repo — the read the +// pre-delete CID check is made against. +func (s *realLegacySource) ReadLegacyPost(ctx context.Context, uri string) (posts.LegacyPost, bool, error) { + rkey := uri[strings.LastIndex(uri, "/")+1:] + record, err := s.community.GetRecord(ctx, postCollection, rkey) + if err != nil { + if testkit.IsNotFound(err) { + return posts.LegacyPost{}, false, nil + } + return posts.LegacyPost{}, false, err + } + for _, staged := range s.staged { + if staged.URI == uri { + // The staged shape, re-stamped with what the repo says NOW: the CID is + // the whole reason for re-reading. + staged.CID = record.CID + staged.RawRecord = record.Value + return staged, true, nil + } + } + return posts.LegacyPost{}, false, nil +} + +// DeleteLegacyPost deletes UNDER THE SWAP GUARD, exactly as production does: the +// PDS refuses the delete if the record no longer carries swapCID, so a +// concurrent edit cannot be destroyed. +func (s *realLegacySource) DeleteLegacyPost(ctx context.Context, legacy posts.LegacyPost, swapCID string) error { + if swapCID == "" { + return fmt.Errorf("refusing to delete %s without a swap guard", legacy.URI) + } rkey := legacy.URI[strings.LastIndex(legacy.URI, "/")+1:] - err := s.community.DeleteRecord(ctx, postCollection, rkey) + guarded, ok := s.community.(pds.GuardedDeleter) + if !ok { + return fmt.Errorf("the PDS client does not support the swap-guarded delete") + } + err := guarded.DeleteRecordWithSwap(ctx, postCollection, rkey, swapCID) if err != nil && testkit.IsNotFound(err) { return nil } @@ -140,7 +174,8 @@ func TestRematerialize_OuterContract_RealPDS_MovesPostAndIsIdempotent(t *testing source := &realLegacySource{community: communityGeneric, staged: []posts.LegacyPost{legacy}} ledger := postgres.NewRematerializeLedger(testkit.DB(t)) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authorFactory, Acceptances: writer} + communityRepos := func(_ context.Context, _ string) (posts.CommunityRepo, error) { return communityRepo, nil } + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authorFactory, Acceptances: writer, CommunityRepos: communityRepos} // ---- run ----------------------------------------------------------------- state, err := tool.RematerializeOne(ctx, legacy) @@ -270,7 +305,8 @@ func TestRematerialize_OuterContract_CopiesEmbedBlobToAuthorRepo(t *testing.T) { source := &realLegacySource{community: communityGeneric, staged: []posts.LegacyPost{legacy}} ledger := postgres.NewRematerializeLedger(testkit.DB(t)) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authorFactory, Acceptances: writer} + communityRepos := func(_ context.Context, _ string) (posts.CommunityRepo, error) { return communityRepo, nil } + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authorFactory, Acceptances: writer, CommunityRepos: communityRepos} _, err = tool.RematerializeOne(ctx, legacy) require.NoError(t, err) diff --git a/internal/core/posts/rematerialize_rkey_test.go b/internal/core/posts/rematerialize_rkey_test.go index a4489e2..c6c3c51 100644 --- a/internal/core/posts/rematerialize_rkey_test.go +++ b/internal/core/posts/rematerialize_rkey_test.go @@ -81,3 +81,38 @@ func TestRematerializeRkey_IsNotSubmissionRkey(t *testing.T) { assert.NotEqualf(t, submission, RematerializeRkey(oldURI), "the re-materialization key must be independent of SubmissionRkey — it is derived from the OLD record's URI, not the submission fingerprint the migration lacks") } + +// THE GOLDEN VALUES. Determinism WITHIN one process (the test above) is not the +// property production needs: a per-process salt — a package-level random seed, a +// map iteration order that leaked into the digest, a hostname — passes every +// other test in this file and still mints a SECOND postv2 for every already- +// migrated post the first time the tool is restarted. +// +// These literals are the only thing that pins the derivation ACROSS processes, +// releases and machines. They were produced by the shipped implementation and +// must never be "fixed" to match a changed one. +func TestRematerializeRkey_GoldenValues(t *testing.T) { + golden := []struct { + name string + oldURI string + wantKey string + }{ + {"plc community, tid rkey", "at://did:plc:community2222222222222222/social.coves.community.post/3kqijkl2m4c2r", "beq7r3yeigi53"}, + {"same community, adjacent rkey", "at://did:plc:community2222222222222222/social.coves.community.post/3kqijkl2m4c2s", "2o3lkuliyahmi"}, + {"different community, same rkey", "at://did:plc:community3333333333333333/social.coves.community.post/3kqijkl2m4c2r", "4kwla2cw7a4pv"}, + {"did:web community", "at://did:web:coves.social/social.coves.community.post/3kqijkl2m4c2r", "a3jez3fwk5sz2"}, + {"empty string", "", "6a6cegjsb2orv"}, + {"unicode in the rkey position", "at://did:plc:community2222222222222222/social.coves.community.post/naïve", "7g76vsnjgfeey"}, + } + + for _, tc := range golden { + t.Run(tc.name, func(t *testing.T) { + assert.Equalf(t, tc.wantKey, RematerializeRkey(tc.oldURI), + "THE RE-MATERIALIZATION RKEY DERIVATION CHANGED.\n"+ + "This is not a test to update. Every post already migrated by the shipped derivation is at the OLD key, so a run under the new one "+ + "writes a SECOND postv2 for every one of them: createAuthorRecord's converge-by-read fires against a different, empty key, the "+ + "duplicate lands, and every strongRef built from the first record dangles. Revert the derivation instead.\n"+ + "old URI: %q", tc.oldURI) + }) + } +} diff --git a/internal/core/posts/rematerialize_test.go b/internal/core/posts/rematerialize_test.go index 78a6782..a170062 100644 --- a/internal/core/posts/rematerialize_test.go +++ b/internal/core/posts/rematerialize_test.go @@ -89,8 +89,15 @@ func firstMutationIndex(events []string) int { // under the verify, an author with no credentials — are precisely the ones a real // PDS will not produce on demand. The write PRIMITIVES the tool reuses // (createAuthorRecord's converge-by-read, WriteAcceptance's skip) have their own -// real-PDS coverage; the outer real-stack proof is tests/e2e/rematerialize_ -// contract_test.go. +// real-PDS coverage; the outer real-infrastructure proof is +// rematerialize_outer_test.go in this package, which drives the whole tool +// against a REAL PDS and a real ledger (its header explains why it is T1 and not +// T2 — the tool's ledger is a table in the AppView's own database, which the e2e +// package's constitution forbids a contract from touching). +// +// The guards on the irreversible step — that the delete does not happen unless a +// replacement is provably standing RIGHT NOW, on every resumed path — live in +// rematerialize_guard_test.go. // // # THE TOOL DOES NOT RE-DECIDE (mirrors the scriptedDecider trick) // @@ -120,20 +127,50 @@ func deterministicCID(rkey string) string { return "bafyreipostv2" + rkey } type fakeAuthorRepo struct { did string + // host is the PDS this repo is bound to, exposed through HostURL so the blob + // paths address the same host a real run would. + host string + mu sync.Mutex records map[string]*pds.RecordResponse // rkey -> record putErr error // one-shot injected put failure getCIDAt map[string]string // rkey -> CID GetRecord should report (verify-window override) - blobs map[string]bool // blob CIDs uploaded into this repo (P4) - log *callLog // shared ordering log (P8), nil when unused + // getErrAt makes GetRecord FAIL for a rkey — the transport-failure branch of + // the verification read, which no test used to reach. + getErrAt map[string]error + // deleteOnGet models the record vanishing between the write and the verify. + deleteOnGet map[string]bool + blobs map[string]bool // blob CIDs uploaded into this repo (P4) + log *callLog // shared ordering log (P8), nil when unused +} + +// HostURL is what the blob paths resolve against. +func (r *fakeAuthorRepo) HostURL() string { + if r.host == "" { + return "http://author-pds.invalid" + } + return r.host +} + +// standingCID reports the CID of the record standing at a rkey, so a re-entry +// test can prove the postv2 converged rather than being re-minted. +func (r *fakeAuthorRepo) standingCID(rkey string) string { + r.mu.Lock() + defer r.mu.Unlock() + if rec, ok := r.records[rkey]; ok { + return rec.CID + } + return "" } func newFakeAuthorRepo(did string) *fakeAuthorRepo { return &fakeAuthorRepo{ - did: did, - records: map[string]*pds.RecordResponse{}, - getCIDAt: map[string]string{}, - blobs: map[string]bool{}, + did: did, + records: map[string]*pds.RecordResponse{}, + getCIDAt: map[string]string{}, + getErrAt: map[string]error{}, + deleteOnGet: map[string]bool{}, + blobs: map[string]bool{}, } } @@ -157,6 +194,13 @@ func (r *fakeAuthorRepo) writtenBody(rkey string) map[string]any { func (r *fakeAuthorRepo) GetRecord(_ context.Context, collection, rkey string) (*pds.RecordResponse, error) { r.mu.Lock() defer r.mu.Unlock() + if err, ok := r.getErrAt[rkey]; ok { + return nil, err + } + if r.deleteOnGet[rkey] { + delete(r.records, rkey) + delete(r.deleteOnGet, rkey) + } rec, ok := r.records[rkey] if !ok { return nil, pds.ErrNotFound @@ -219,7 +263,7 @@ func (r *fakeAuthorRepo) UploadBlob(_ context.Context, data []byte, mimeType str // embeds against what actually landed here. cid := blobCIDFor(data) r.blobs[cid] = true - return &blobs.BlobRef{}, nil + return &blobs.BlobRef{Type: "blob", Ref: map[string]string{"$link": cid}, MimeType: mimeType, Size: len(data)}, nil } func (r *fakeAuthorRepo) hasBlob(cid string) bool { @@ -254,11 +298,19 @@ func blobCIDFor(data []byte) string { return "bafkreiblob" + fmt.Sprintf("%x", l type fakeAuthorFactory struct { repos map[string]*fakeAuthorRepo noCreds map[string]bool - log *callLog // shared ordering log (P8), nil when unused + // retryable marks a DID whose credentials fail transiently — a network blip, + // a PDS 5xx. It is a different error class from noCreds and the tool must + // treat it differently: no verdict, fail the run. + retryable map[string]bool + log *callLog // shared ordering log (P8), nil when unused } func newFakeAuthorFactory() *fakeAuthorFactory { - return &fakeAuthorFactory{repos: map[string]*fakeAuthorRepo{}, noCreds: map[string]bool{}} + return &fakeAuthorFactory{ + repos: map[string]*fakeAuthorRepo{}, + noCreds: map[string]bool{}, + retryable: map[string]bool{}, + } } func (f *fakeAuthorFactory) repo(did string) *fakeAuthorRepo { @@ -277,6 +329,10 @@ func (f *fakeAuthorFactory) factory() posts.AuthorRepoFactory { // first pin (P8) can assert the tool resolves EVERY author before it // mutates ANY repo. f.log.note("resolve:" + authorDID) + if f.retryable[authorDID] { + return nil, fmt.Errorf("resuming the stored session of %s: %w: dial tcp: connection refused", + authorDID, posts.ErrAuthorCredentialsUnavailable) + } if f.noCreds[authorDID] { return nil, fmt.Errorf("resuming the stored session of %s: %w", authorDID, posts.ErrNoAuthorCredentials) } @@ -297,8 +353,27 @@ type spyAcceptanceWriter struct { mu sync.Mutex acceptanceCmds []posts.CommunityWriteCommand writeErr error // one-shot injected failure - otherCalled []string - log *callLog // shared ordering log (P8), nil when unused + // resultOverride replaces what WriteAcceptance reports, so a test can model a + // writer that answers success without the record standing. + resultOverride *posts.CommunityWriteResult + // suppressStanding makes the writer REPORT success without the record ever + // standing in the community repo — a lost commit, a proxy that swallowed the + // write, a bug in the writer. + suppressStanding bool + // afterWrite runs once the write has been recorded, so a test can mutate what + // stands before the verification reads it. + afterWrite func(*spyAcceptanceWriter, posts.CommunityWriteCommand) + // readErr makes the community repo's GetRecord fail. + readErr error + // communityHost is the PDS the community's repo is bound to. + communityHost string + // standing is the acceptance record the COMMUNITY repo actually serves, keyed + // by rkey. It is what the verification read-back sees, and it is deliberately + // separate from the command log: a writer that reports success without the + // record standing is exactly the case the read-back exists to catch. + standing map[string]*pds.RecordResponse + otherCalled []string + log *callLog // shared ordering log (P8), nil when unused } func (s *spyAcceptanceWriter) WriteAcceptance(_ context.Context, cmd posts.CommunityWriteCommand) (posts.CommunityWriteResult, error) { @@ -322,8 +397,33 @@ func (s *spyAcceptanceWriter) WriteAcceptance(_ context.Context, cmd posts.Commu s.acceptanceCmds = append(s.acceptanceCmds, cmd) rkey := posts.SubjectRkey(cmd.PostURI) + if s.standing == nil { + s.standing = map[string]*pds.RecordResponse{} + } + if _, already := s.standing[rkey]; !already && !s.suppressStanding { + s.standing[rkey] = &pds.RecordResponse{ + URI: "at://" + cmd.CommunityDID + "/" + posts.AcceptanceCollection + "/" + rkey, + CID: "bafyreiacceptance" + rkey, + Value: map[string]any{ + "$type": posts.AcceptanceCollection, + "subject": map[string]any{"uri": cmd.PostURI, "cid": cmd.PostCID}, + "createdAt": "2026-01-02T03:04:05Z", + }, + } + } + + if s.afterWrite != nil { + hook := s.afterWrite + s.mu.Unlock() + hook(s, cmd) + s.mu.Lock() + } + + if s.resultOverride != nil { + return *s.resultOverride, nil + } return posts.CommunityWriteResult{ - URI: "at://" + cmd.CommunityDID + "/social.coves.community.acceptance/" + rkey, + URI: "at://" + cmd.CommunityDID + "/" + posts.AcceptanceCollection + "/" + rkey, RKey: rkey, CID: "bafyreiacceptance" + rkey, Rev: "3krematacceptxx", @@ -331,6 +431,123 @@ func (s *spyAcceptanceWriter) WriteAcceptance(_ context.Context, cmd posts.Commu }, nil } +// seedStandingAcceptance stages an acceptance as if a previous run had written +// it, WITHOUT recording a WriteAcceptance call — the state a crash-resumed run +// finds. +func (s *spyAcceptanceWriter) seedStandingAcceptance(communityDID, postURI, postCID string) { + s.mu.Lock() + defer s.mu.Unlock() + if s.standing == nil { + s.standing = map[string]*pds.RecordResponse{} + } + rkey := posts.SubjectRkey(postURI) + s.standing[rkey] = &pds.RecordResponse{ + URI: "at://" + communityDID + "/" + posts.AcceptanceCollection + "/" + rkey, + CID: "bafyreiacceptance" + rkey, + Value: map[string]any{ + "$type": posts.AcceptanceCollection, + "subject": map[string]any{"uri": postURI, "cid": postCID}, + "createdAt": "2026-01-02T03:04:05Z", + }, + } +} + +// withdrawStanding removes the acceptance record from the community repo without +// touching the command log — the "the writer said yes, the repo says no" case. +func (s *spyAcceptanceWriter) withdrawStanding(postURI string) { + s.mu.Lock() + defer s.mu.Unlock() + delete(s.standing, posts.SubjectRkey(postURI)) +} + +// repinStanding rewrites the standing acceptance to name a different subject, so +// a test can prove the read-back compares the SUBJECT rather than merely finding +// a record at the deterministic key. +func (s *spyAcceptanceWriter) repinStanding(postURI, subjectURI, subjectCID string) { + s.mu.Lock() + defer s.mu.Unlock() + rkey := posts.SubjectRkey(postURI) + if rec, ok := s.standing[rkey]; ok { + rec.Value = map[string]any{ + "$type": posts.AcceptanceCollection, + "subject": map[string]any{"uri": subjectURI, "cid": subjectCID}, + } + } +} + +// acceptanceCount reports how many distinct acceptance records stand. +func (s *spyAcceptanceWriter) acceptanceCount() int { + s.mu.Lock() + defer s.mu.Unlock() + return len(s.standing) +} + +// standingCID is the CID of the acceptance record that stands for a subject. +func (s *spyAcceptanceWriter) standingCID(postURI string) string { + s.mu.Lock() + defer s.mu.Unlock() + if rec, ok := s.standing[posts.SubjectRkey(postURI)]; ok { + return rec.CID + } + return "" +} + +// repos is the CommunityRepoFactory the Rematerializer reads the acceptance back +// through. It serves ONLY what this writer actually made stand, which is the +// whole point: an acceptance the writer merely REPORTED is not one the community +// repo holds, and only a read can tell the two apart. +func (s *spyAcceptanceWriter) repos() posts.CommunityRepoFactory { + return func(_ context.Context, communityDID string) (posts.CommunityRepo, error) { + return &fakeCommunityRepo{did: communityDID, writer: s, getErr: s.readErr, host: s.communityHost}, nil + } +} + +// fakeCommunityRepo is the read side of the community's repo: it serves the +// acceptance records the spy writer made stand, and nothing else. +type fakeCommunityRepo struct { + did string + writer *spyAcceptanceWriter + // getErr, when set, makes the acceptance read-back fail — the transport + // failure whose branch had no coverage at all. + getErr error + host string +} + +// HostURL is where the community's blobs are served from. +func (r *fakeCommunityRepo) HostURL() string { + if r.host == "" { + return "http://community-pds.invalid" + } + return r.host +} + +func (r *fakeCommunityRepo) GetRecord(_ context.Context, collection, rkey string) (*pds.RecordResponse, error) { + if r.getErr != nil { + return nil, r.getErr + } + if collection != posts.AcceptanceCollection { + return nil, pds.ErrNotFound + } + r.writer.mu.Lock() + defer r.writer.mu.Unlock() + rec, ok := r.writer.standing[rkey] + if !ok { + return nil, pds.ErrNotFound + } + return rec, nil +} + +func (r *fakeCommunityRepo) PutRecordWithCommit(context.Context, string, string, any, string) (*pds.RecordCommit, error) { + return nil, fmt.Errorf("the re-materialization tool must not write to the community repo through this seam") +} +func (r *fakeCommunityRepo) ApplyWrites(context.Context, []pds.Write, string) (*pds.ApplyWritesResult, error) { + return nil, fmt.Errorf("the re-materialization tool must never batch community writes") +} +func (r *fakeCommunityRepo) GetLatestCommit(context.Context) (*pds.LatestCommit, error) { + return &pds.LatestCommit{}, nil +} +func (r *fakeCommunityRepo) DID() string { return r.did } + // The other four methods exist only to satisfy CommunityRecordWriter. The tool // must never call them: a re-materialized post is accepted, not removed, // restored, repinned, or withdrawn. Each records that it was reached so a test @@ -371,28 +588,96 @@ type fakeLegacySource struct { posts []posts.LegacyPost deleted map[string]int deleteErr map[string]error - log *callLog // shared ordering log (P8), nil when unused + gone map[string]bool + readErr map[string]error + // pending records appear on a LATER listing, modelling a write that lands + // after the discovery pass. + pending []posts.LegacyPost + listCalls int + // swaps records the CID each delete was guarded by, so a test can prove the + // guard is actually sent rather than merely accepted as a parameter. + swaps []string + log *callLog // shared ordering log (P8), nil when unused } func newFakeLegacySource(ps ...posts.LegacyPost) *fakeLegacySource { - return &fakeLegacySource{posts: ps, deleted: map[string]int{}, deleteErr: map[string]error{}} + return &fakeLegacySource{ + posts: ps, + deleted: map[string]int{}, + deleteErr: map[string]error{}, + gone: map[string]bool{}, + readErr: map[string]error{}, + } } +// ListLegacyPosts returns the records that still STAND — a deleted one is gone +// from the listing, the way a real listRecords behaves. The final re-scan the +// census gates completion on is only meaningful against a source that models +// this. func (s *fakeLegacySource) ListLegacyPosts(_ context.Context) ([]posts.LegacyPost, error) { s.mu.Lock() defer s.mu.Unlock() - return append([]posts.LegacyPost(nil), s.posts...), nil + // Pending records land on the SECOND listing and later: the first listing is + // the run's discovery pass, and the whole point is a record that appears after + // it. + s.listCalls++ + if s.listCalls > 1 && len(s.pending) > 0 { + s.posts = append(s.posts, s.pending...) + s.pending = nil + } + var out []posts.LegacyPost + for _, p := range s.posts { + if s.gone[p.URI] { + continue + } + out = append(out, p) + } + return out, nil +} + +func (s *fakeLegacySource) ReadLegacyPost(_ context.Context, uri string) (posts.LegacyPost, bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + if err, ok := s.readErr[uri]; ok { + return posts.LegacyPost{}, false, err + } + if s.gone[uri] { + return posts.LegacyPost{}, false, nil + } + for _, p := range s.posts { + if p.URI == uri { + return p, true, nil + } + } + return posts.LegacyPost{}, false, nil } -func (s *fakeLegacySource) DeleteLegacyPost(_ context.Context, legacy posts.LegacyPost) error { +// setCurrentCID models an edit landing on the legacy record after the tool read +// it: the record still stands, but under a different CID. +func (s *fakeLegacySource) setCurrentCID(uri, cid string) { + s.mu.Lock() + defer s.mu.Unlock() + for i := range s.posts { + if s.posts[i].URI == uri { + s.posts[i].CID = cid + } + } +} + +func (s *fakeLegacySource) DeleteLegacyPost(_ context.Context, legacy posts.LegacyPost, swapCID string) error { s.mu.Lock() defer s.mu.Unlock() s.log.note("delete:" + legacy.URI) s.deleted[legacy.URI]++ + s.swaps = append(s.swaps, swapCID) + if swapCID == "" { + return fmt.Errorf("refusing to delete %s without a swap guard", legacy.URI) + } if err, ok := s.deleteErr[legacy.URI]; ok { delete(s.deleteErr, legacy.URI) return err } + s.gone[legacy.URI] = true return nil } @@ -402,6 +687,29 @@ func (s *fakeLegacySource) deleteCount(uri string) int { return s.deleted[uri] } +// markGone models the record having already been deleted — the state a crash +// between the delete and MarkDone leaves behind. +func (s *fakeLegacySource) markGone(uri string) { + s.mu.Lock() + defer s.mu.Unlock() + s.gone[uri] = true +} + +// appendOnNextList stages a record that appears only on a LATER listing — a +// writer the maintenance window did not stop, landing a post mid-run. It is what +// makes the final re-scan mean anything. +func (s *fakeLegacySource) appendOnNextList(p posts.LegacyPost) { + s.mu.Lock() + defer s.mu.Unlock() + s.pending = append(s.pending, p) +} + +func (s *fakeLegacySource) swapGuards() []string { + s.mu.Lock() + defer s.mu.Unlock() + return append([]string(nil), s.swaps...) +} + // legacyPost stages one deprecated community.post, keyed by a unique rkey so // parallel tests never collide on the ledger's old_uri primary key. func legacyPost(t *testing.T, communityDID, authorDID string) posts.LegacyPost { @@ -452,7 +760,7 @@ func TestRematerialize_HappyPath_WalksToDoneVerifyBeforeDelete(t *testing.T) { legacy := legacyPost(t, rematCommunityDID, rematAuthorDID) source := newFakeLegacySource(legacy) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} state, err := tool.RematerializeOne(context.Background(), legacy) require.NoError(t, err) @@ -497,7 +805,7 @@ func TestRematerialize_ReRun_IsAPureNoOp(t *testing.T) { writer := &spyAcceptanceWriter{} legacy := legacyPost(t, rematCommunityDID, rematAuthorDID) source := newFakeLegacySource(legacy) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} first, err := tool.RematerializeOne(context.Background(), legacy) require.NoError(t, err) @@ -513,10 +821,17 @@ func TestRematerialize_ReRun_IsAPureNoOp(t *testing.T) { assert.Equalf(t, 1, authors.repo(rematAuthorDID).recordCount(), "a re-run must not mint a second postv2 — the deterministic rkey converges on the first record") + // EXACT counts. "at least one acceptance write" is satisfied by a SECOND one, + // which mints a fresh record CID and invalidates every reference to the record + // it replaced — on every retry, forever. calls := writer.calls() - require.GreaterOrEqual(t, len(calls), 1) - assert.Truef(t, calls[len(calls)-1].PostCID == deterministicCID(posts.RematerializeRkey(legacy.URI)), + require.Lenf(t, calls, 1, + "a re-run over a done row wrote %d acceptance(s); it must write exactly the one the first run did and no more", len(calls)) + assert.Equalf(t, deterministicCID(posts.RematerializeRkey(legacy.URI)), calls[0].PostCID, "a re-run's acceptance must still pin the same CID; a fresh CID would dangle every reference to the acceptance") + assert.Equalf(t, 1, source.deleteCount(legacy.URI), + "a re-run over a done row attempted a second delete; the row is terminal and the record is already gone") + assert.Equalf(t, 1, writer.acceptanceCount(), "exactly one acceptance record must stand after a re-run") // The old record was already gone; a re-run's delete (if attempted) is a no-op // success, never an error, and never resurrects the record. row, _, err := ledger.Get(context.Background(), legacy.URI) @@ -535,7 +850,7 @@ func TestRematerialize_ResumeAfterDeleteFailure_RetriesOnlyTheDelete(t *testing. legacy := legacyPost(t, rematCommunityDID, rematAuthorDID) source := newFakeLegacySource(legacy) source.deleteErr[legacy.URI] = fmt.Errorf("transient: the community PDS returned 502 on delete") - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} // First pass: everything succeeds up to the delete, which fails once. The row // must stop at migrated — the checkpoint BEFORE the delete — never done. @@ -577,7 +892,7 @@ func TestRematerialize_CIDMismatch_DoesNotCheckpointOrDelete(t *testing.T) { writer := &spyAcceptanceWriter{} legacy := legacyPost(t, rematCommunityDID, rematAuthorDID) source := newFakeLegacySource(legacy) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} // The verify re-read of the postv2 comes back with a DIFFERENT CID than the // one the acceptance pinned — a concurrent edit landing in the write→verify @@ -596,9 +911,15 @@ func TestRematerialize_CIDMismatch_DoesNotCheckpointOrDelete(t *testing.T) { row, found, err := ledger.Get(context.Background(), legacy.URI) require.NoError(t, err) require.True(t, found) - assert.NotEqualf(t, posts.RematerializeDone, row.State, "a mismatched record must never reach done") - assert.NotEqualf(t, posts.RematerializeMigrated, row.State, - "a mismatched record must never reach the migrated checkpoint — migrated asserts the delete is safe, and it is not") + // THE EXACT STATE, not merely "not those two". Asserting NotEqual leaves the + // checkpoint-before-verify mutation uncatchable: move MarkVerified above the + // read-back and the row lands on `verified` — which is neither done nor + // migrated, so a NotEqual pair stays green while the ledger now claims a + // verification that never happened. + assert.Equalf(t, posts.RematerializePostV2Written, row.State, + "a record whose postv2 CID no longer matches must stop at postv2_written. It reached %s instead: the postv2 exists and NOTHING about it has been "+ + "verified, so any state past postv2_written is a claim the ledger cannot support — and `verified`/`migrated` are what a later pass reads as "+ + "permission to delete", row.State) } func TestRematerialize_NoCredentials_LeavesLegacyNeverForges(t *testing.T) { @@ -618,7 +939,7 @@ func TestRematerialize_NoCredentials_LeavesLegacyNeverForges(t *testing.T) { authors.noCreds[humanDID] = true legacy := legacyPost(t, rematCommunityDID, humanDID) source := newFakeLegacySource(legacy) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} state, err := tool.RematerializeOne(context.Background(), legacy) require.NoError(t, err, "a no-creds record is an expected terminal outcome, not a run-failing error") @@ -654,7 +975,7 @@ func TestRematerialize_Run_CensusGatesCompletionWhileFallbackSurvives(t *testing stranded := legacyPost(t, rematCommunityDID, humanDID) source := newFakeLegacySource(migratable, stranded) writer := &spyAcceptanceWriter{} - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} report, err := tool.Run(context.Background()) require.NoError(t, err) @@ -690,7 +1011,7 @@ func TestRematerialize_UsesDirectAcceptanceWriter_NeverReDecides(t *testing.T) { // community it currently sits in. This mirrors service_writeforward_test.go's // scriptedDecider trick, made structural: the acceptance is written for the // post's content unconditionally. - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} state, err := tool.RematerializeOne(context.Background(), legacy) require.NoError(t, err) @@ -742,7 +1063,7 @@ func TestRematerialize_PreservesEveryPublishedField(t *testing.T) { RawRecord: raw, } source := newFakeLegacySource(legacy) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} _, err := tool.RematerializeOne(context.Background(), legacy) require.NoError(t, err) @@ -776,7 +1097,7 @@ func TestRematerialize_RefusesWhenADifferentRecordStandsAtTheRkey(t *testing.T) writer := &spyAcceptanceWriter{} legacy := legacyPost(t, rematCommunityDID, rematAuthorDID) source := newFakeLegacySource(legacy) - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} // A DIFFERENT record already stands at the target rkey — its CID is the one a // fresh write would get (so a CID-only verify passes), but its body is NOT this @@ -801,8 +1122,9 @@ func TestRematerialize_RefusesWhenADifferentRecordStandsAtTheRkey(t *testing.T) row, found, err := ledger.Get(context.Background(), legacy.URI) require.NoError(t, err) require.True(t, found) - assert.NotEqualf(t, posts.RematerializeDone, row.State, "a body-mismatched record must never reach done") - assert.NotEqualf(t, posts.RematerializeMigrated, row.State, "a body-mismatched record must never reach the migrated checkpoint") + assert.Equalf(t, posts.RematerializeDiscovered, row.State, + "a record whose deterministic rkey is occupied by a FOREIGN record must stay at discovered — nothing has been written for it. It reached %s "+ + "instead, and every state past discovered is read by a later pass as work already done", row.State) } // P7 — crash-resume must be driven by the LEDGER, not the source listing, and @@ -824,15 +1146,23 @@ func TestRematerialize_Run_ReconcilesStrandedMigratedRowFromLedger(t *testing.T) newRkey := posts.RematerializeRkey(strandedURI) newURI := "at://" + rematAuthorDID + "/social.coves.community.postv2/" + newRkey newCID := deterministicCID(newRkey) - _, err := ledger.Discover(ctx, strandedURI, rematAuthorDID) + _, err := ledger.Discover(ctx, strandedURI, rematCommunityDID, rematAuthorDID) require.NoError(t, err) - require.NoError(t, ledger.RecordPostV2Written(ctx, strandedURI, newURI, newCID, newRkey)) + require.NoError(t, ledger.RecordPostV2Written(ctx, strandedURI, "bafyreilegacysource", newURI, newCID, newRkey)) require.NoError(t, ledger.MarkVerified(ctx, strandedURI)) require.NoError(t, ledger.MarkMigrated(ctx, strandedURI)) + // The REPO STATE a crashed run leaves behind: the postv2 stands in the + // author's repo and the acceptance stands in the community's. The resumed run + // re-reads BOTH before it will finish the row — a ledger row at `migrated` is a + // memory of a check that passed, not a licence to delete — so a reconcile test + // that stages only the ledger proves nothing about the repos. + authors.repo(rematAuthorDID).seedStanding(posts.PostV2Collection, newRkey) + writer.seedStandingAcceptance(rematCommunityDID, newURI, newCID) + // The source does NOT list the stranded record — its community.post is gone. source := newFakeLegacySource() - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} report, err := tool.Run(ctx) require.NoError(t, err) @@ -883,8 +1213,9 @@ func TestRematerialize_Run_ResolvesAllCredentialsBeforeAnyMutation(t *testing.T) first := legacyPost(t, rematCommunityDID, withCreds) second := legacyPost(t, rematCommunityDID, noCreds) - source := &fakeLegacySource{posts: []posts.LegacyPost{first, second}, deleted: map[string]int{}, deleteErr: map[string]error{}, log: log} - tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer} + source := newFakeLegacySource(first, second) + source.log = log + tool := &posts.Rematerializer{Source: source, Ledger: ledger, AuthorRepos: authors.factory(), Acceptances: writer, CommunityRepos: writer.repos()} _, err := tool.Run(context.Background()) require.NoError(t, err) diff --git a/internal/db/migrations/037_create_post_rematerialization_ledger.sql b/internal/db/migrations/037_create_post_rematerialization_ledger.sql index 4e80b79..0234dcb 100644 --- a/internal/db/migrations/037_create_post_rematerialization_ledger.sql +++ b/internal/db/migrations/037_create_post_rematerialization_ledger.sql @@ -18,11 +18,40 @@ -- the two would either re-write the postv2 on every resume or skip the delete a -- crash owes. -- --- THE TWO FALLBACK STATES are the credential census (§11 step 3): a record whose --- author credentials cannot be restored is left as legacy — never re-authored --- under a forged signature, which would reintroduce the §2 impersonation the flip --- exists to remove — and the run refuses to report "complete" while any such row +-- NOTE THAT A CHECKPOINT IS NOT A LICENCE. `verified` and `migrated` record that +-- verification passed at a moment now in the past; the tool re-reads the postv2, +-- the acceptance, the blobs and the legacy record's CID immediately before every +-- delete, on every path including a resumed one. This table remembers progress, +-- it does not authorize destruction. +-- +-- THE FALLBACK STATE is the credential census (§11 step 3): a record whose author +-- credentials cannot be restored is left as legacy — never re-authored under a +-- forged signature, which would reintroduce the §2 impersonation the flip exists +-- to remove — and the run refuses to report "complete" while any such row -- survives, gating the operator's separate, irreversible legacy-removal step. +-- +-- ──────────────────────────────────────────────────────────────────────────── +-- OPERATOR RECOVERY: GETTING A ROW OUT OF fallback_left_legacy +-- +-- A fallback row is terminal to the state machine, deliberately, so that nothing +-- gets a second chance to forge authorship. It is NOT irreversible: once the +-- author has been re-authorized (the aggregator's grant re-issued, the human's +-- session re-established), the supported move is +-- +-- rematerialize-posts -reopen-fallbacks [-community ] +-- +-- which is the tool's ReopenFallback: it sets those rows back to `discovered` +-- and clears their reason, touching no repo. Prefer it to hand-written SQL. +-- If the binary is unavailable, the equivalent statement is: +-- +-- UPDATE post_rematerialization_ledger +-- SET state = 'discovered', reason = NULL, updated_at = NOW() +-- WHERE state = 'fallback_left_legacy' +-- AND community_did = ''; -- omit this line to reopen every community +-- +-- Never move a row out of `done` this way: `done` means the legacy record has +-- been deleted, and re-running against it cannot bring the record back. +-- ──────────────────────────────────────────────────────────────────────────── CREATE TABLE post_rematerialization_ledger ( -- The OLD community.post AT-URI is the whole key: one row per legacy record, -- so a re-run UPDATES the row it resumes from rather than accumulating a @@ -35,14 +64,19 @@ CREATE TABLE post_rematerialization_ledger ( -- schema, where every writer meets it, rather than sitting in the table as an -- unresumable row nothing would ever advance. migrated and done are BOTH -- listed and DISTINCT on purpose (see the header). + -- + -- THERE IS EXACTLY ONE FALLBACK STATE. An earlier revision also admitted + -- 'fallback_no_creds' and no code path ever wrote it: a state the vocabulary + -- permits but nothing produces is a trap for whoever writes recovery SQL + -- against it at 2am, and a second name for one outcome is a second thing to + -- forget in a WHERE clause. state TEXT NOT NULL CHECK (state IN ( 'discovered', 'postv2_written', 'verified', 'migrated', 'done', - 'fallback_left_legacy', - 'fallback_no_creds' + 'fallback_left_legacy' )), -- Who the postv2 is re-authored under: the legacy record's `author` field. @@ -50,6 +84,25 @@ CREATE TABLE post_rematerialization_ledger ( -- into — but it is the audit trail for which repo the tool wrote to. author_did TEXT, + -- The community repo the legacy record lives in. + -- + -- IT IS STORED RATHER THAN PARSED BACK OUT OF old_uri because the whole + -- destructive half of the tool is scoped by it: a staged `-community` run + -- resumes, counts and DELETES only rows carrying this value, and a scope that + -- depends on string-slicing a URI at every call site is a scope one call site + -- eventually gets wrong. + community_did TEXT, + + -- The legacy record's CID as read at the moment its postv2 was built. + -- + -- THIS IS A SAFETY INTERLOCK, NOT AUDIT TRIM. The delete is refused unless a + -- fresh read of the legacy record still reports this exact CID, and it is sent + -- to the PDS as the delete's swapRecord so the PDS refuses a stale delete + -- independently. Without it, an edit landing after the conversion — an + -- aggregator cron, a cached mobile session — is destroyed silently, and the + -- run still reports clean. + source_cid TEXT, + -- The postv2 coordinates, populated at the postv2_written transition and read -- back (never recomputed) on resume, so a resumed run converges on the record -- its first attempt wrote rather than deriving a fresh CID. NULL until then. @@ -64,9 +117,17 @@ CREATE TABLE post_rematerialization_ledger ( updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW() ); +-- The staged-rollout scope. Every scoped query (resume, census, reopen) filters +-- on community_did, and a sequential scan of the whole ledger per record is what +-- a run over a large instance would otherwise pay. +CREATE INDEX idx_post_rematerialization_ledger_community_state + ON post_rematerialization_ledger (community_did, state); + COMMENT ON TABLE post_rematerialization_ledger IS 'One row per legacy community.post: the cutover tool''s resumable, idempotent progress ledger (PRD_AUTHOR_OWNED_POSTS 11)'; COMMENT ON COLUMN post_rematerialization_ledger.old_uri IS 'The OLD community.post AT-URI; the primary key and the material the deterministic postv2 rkey is derived from'; COMMENT ON COLUMN post_rematerialization_ledger.state IS 'The state-machine cursor; CHECK-closed. migrated (safe to delete, old record present) is DISTINCT from done (old record deleted)'; +COMMENT ON COLUMN post_rematerialization_ledger.community_did IS 'The community repo the legacy record lives in; the scope a staged -community run resumes, counts and deletes within'; +COMMENT ON COLUMN post_rematerialization_ledger.source_cid IS 'The legacy record CID the postv2 was built from; re-checked against a fresh read and sent as the delete swapRecord so a concurrent edit cannot be destroyed'; COMMENT ON COLUMN post_rematerialization_ledger.new_uri IS 'The postv2 URI written at the postv2_written transition; read back on resume, never recomputed'; -- +goose Down diff --git a/internal/db/postgres/community_hosted.go b/internal/db/postgres/community_hosted.go new file mode 100644 index 0000000..bbb4d58 --- /dev/null +++ b/internal/db/postgres/community_hosted.go @@ -0,0 +1,86 @@ +package postgres + +import ( + "context" + "database/sql" + "fmt" +) + +// The hosted-community scope query, kept deliberately apart from the community +// repository (docs/PRD_AUTHOR_OWNED_POSTS.md §11). +// +// # WHY THIS IS ITS OWN QUERY AND NOT A FIELD ON `List` +// +// "Which communities does this AppView host?" is answered by ONE fact: whether +// it stores that community's PDS refresh token. Nothing else — not +// `hosted_by_did`, which is a claim in a record anyone can write — decides +// whether we can sign a write, and therefore whether the re-materialization tool +// may write into that repo and delete out of it. +// +// The obvious implementation, walking `Repository.List` and testing +// `PDSRefreshToken != ""`, is silently wrong in the most dangerous direction: +// `List`'s SELECT does not include the credential columns, so the field is empty +// on every listed row and the filter matches NOTHING. A tool built on it reports +// a clean, complete, exit-0 run having migrated nothing at all. +// +// The repair is NOT to teach `List` to decrypt credentials. `List` backs public +// discovery endpoints; putting a decrypted refresh token on that path so that +// one batch tool can test it for emptiness trades a leak for a convenience. This +// query instead answers the question directly, in the vocabulary the caller +// actually needs — DIDs — and never decrypts anything: presence of the +// ciphertext column IS the fact being asked about. +type hostedCommunityQuery struct { + db *sql.DB +} + +// HostedCommunitySource answers which communities this AppView can sign for. +// +// It is an interface so the tool's scope can be faked in tests without a +// database, and so the production wiring reads as the narrow capability it is +// rather than as "the community repository". +type HostedCommunitySource interface { + // HostedCommunityDIDs returns the DIDs of every community whose PDS refresh + // token this AppView stores, in a stable order. It returns identifiers only: + // no credential material crosses the boundary. + HostedCommunityDIDs(ctx context.Context) ([]string, error) +} + +// NewHostedCommunityQuery returns the hosted-community scope query. +func NewHostedCommunityQuery(db *sql.DB) HostedCommunitySource { + return &hostedCommunityQuery{db: db} +} + +// HostedCommunityDIDs returns the DIDs of the communities whose PDS refresh +// token is stored — the exact set whose repos this AppView can write to and +// delete from. +// +// The predicate is `pds_refresh_token_encrypted IS NOT NULL` because that column +// is written only by the provisioning path that also holds the account: a row +// carrying it is a repo we have credentials for, and a row without it is one we +// merely index. The ciphertext is never decrypted here — its presence is the +// whole answer. +func (q *hostedCommunityQuery) HostedCommunityDIDs(ctx context.Context) ([]string, error) { + rows, err := q.db.QueryContext(ctx, ` + SELECT did + FROM communities + WHERE pds_refresh_token_encrypted IS NOT NULL + ORDER BY did + `) + if err != nil { + return nil, fmt.Errorf("listing hosted community DIDs: %w", err) + } + defer func() { _ = rows.Close() }() + + var dids []string + for rows.Next() { + var did string + if err := rows.Scan(&did); err != nil { + return nil, fmt.Errorf("scanning a hosted community DID: %w", err) + } + dids = append(dids, did) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterating hosted community DIDs: %w", err) + } + return dids, nil +} diff --git a/internal/db/postgres/community_hosted_test.go b/internal/db/postgres/community_hosted_test.go new file mode 100644 index 0000000..6f6c3a5 --- /dev/null +++ b/internal/db/postgres/community_hosted_test.go @@ -0,0 +1,125 @@ +//go:build integration + +package postgres + +import ( + "context" + "fmt" + "testing" + "time" + + "Coves/internal/core/communities" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// HOSTING IS CREDENTIAL PRESENCE, AND IT HAS TO BE ASKED FOR DIRECTLY. +// +// The re-materialization tool can only re-materialize (and then DELETE) posts in +// a community whose repo it can sign for, so its whole scope is "the communities +// whose PDS refresh token this AppView stores". The obvious way to get that — +// walk `List` and test `community.PDSRefreshToken != ""` — is silently WRONG: +// List does not select the credential columns at all, so the field is the empty +// string on every listed row and the filter excludes EVERY community. A default +// all-communities production run then migrates nothing while reporting a clean, +// complete, exit-0 census. +// +// Widening List to carry decrypted secrets is the wrong repair: it would leak a +// refresh token into a general-purpose listing path that feeds public endpoints. +// The right one is this query — purpose-named, DIDs only, no credential material +// crossing the boundary at all. +func TestHostedCommunityDIDs_SelectsOnlyCommunitiesWithStoredCredentials(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ctx := context.Background() + + repo := NewCommunityRepository(db) + id := testkit.UniqueID(t) + + hostedDID := "did:plc:hosted" + id + remoteDID := "did:plc:remote" + id + + // A community this AppView hosts: it holds the refresh token, so it can sign + // writes and deletes in that repo. + _, err := repo.Create(ctx, &communities.Community{ + DID: hostedDID, + Handle: fmt.Sprintf("!hosted-%s@coves.local", id), + Name: "hosted-" + id, + OwnerDID: hostedDID, + CreatedByDID: "did:plc:user123", + HostedByDID: "did:web:coves.local", + Visibility: "public", + PDSEmail: "hosted@communities.coves.local", + PDSPassword: "cleartext", + PDSAccessToken: "access-token", + PDSRefreshToken: "refresh-token-" + id, + PDSURL: testkit.Endpoints().PDS.BaseURL, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }) + require.NoError(t, err) + + // A community indexed off the firehose from somewhere else: no credentials, + // so nothing this tool does may ever touch it. + _, err = repo.Create(ctx, &communities.Community{ + DID: remoteDID, + Handle: fmt.Sprintf("!remote-%s@elsewhere.example", id), + Name: "remote-" + id, + OwnerDID: remoteDID, + CreatedByDID: "did:plc:user456", + HostedByDID: "did:web:elsewhere.example", + Visibility: "public", + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }) + require.NoError(t, err) + + dids, err := NewHostedCommunityQuery(db).HostedCommunityDIDs(ctx) + require.NoError(t, err) + + assert.Containsf(t, dids, hostedDID, + "a community whose PDS refresh token is stored was NOT returned. This is the tool's entire scope: if the query returns nothing, "+ + "a default all-communities production run migrates ZERO posts and still reports a complete, exit-0 census") + assert.NotContainsf(t, dids, remoteDID, + "a community with no stored credentials was returned. The tool cannot sign for it, so listing it would fail the run at the first "+ + "listRecords — or worse, invite a delete it has no right to make") +} + +// The query must never carry credential material across the boundary. Returning +// DIDs is what makes it safe to call from a batch tool that logs its scope. +func TestHostedCommunityDIDs_ReturnsIdentifiersOnlyNeverSecrets(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ctx := context.Background() + + repo := NewCommunityRepository(db) + id := testkit.UniqueID(t) + did := "did:plc:secret" + id + secret := "refresh-token-that-must-not-escape-" + id + + _, err := repo.Create(ctx, &communities.Community{ + DID: did, + Handle: fmt.Sprintf("!secret-%s@coves.local", id), + Name: "secret-" + id, + OwnerDID: did, + CreatedByDID: "did:plc:user123", + HostedByDID: "did:web:coves.local", + Visibility: "public", + PDSEmail: "secret@communities.coves.local", + PDSRefreshToken: secret, + PDSURL: testkit.Endpoints().PDS.BaseURL, + CreatedAt: time.Now(), + UpdatedAt: time.Now(), + }) + require.NoError(t, err) + + dids, err := NewHostedCommunityQuery(db).HostedCommunityDIDs(ctx) + require.NoError(t, err) + + for _, got := range dids { + assert.NotContainsf(t, got, secret, + "the hosted-community query returned credential material; it must answer with DIDs alone so a batch tool can log its scope safely") + } +} diff --git a/internal/db/postgres/rematerialize_ledger.go b/internal/db/postgres/rematerialize_ledger.go index 5bdfa56..9e4056b 100644 --- a/internal/db/postgres/rematerialize_ledger.go +++ b/internal/db/postgres/rematerialize_ledger.go @@ -18,6 +18,12 @@ import ( // strict order, so a transition finding the row in an unexpected state means the // ledger and the tool have diverged, and that is a fault to surface, not to // swallow into a false success. +// +// THE COMMUNITY SCOPE IS PART OF THE SCHEMA, not a filter callers remember to +// apply. `ListResumable`, `CountByState` and `ReopenFallback` all take it +// explicitly, because the tool's staged rollout mode is only meaningful if the +// destructive half of the run cannot reach outside it — an unscoped resume for +// community A will happily drive, and delete, community B's rows. // rematerializeLedger is the migration-037-backed posts.RematerializeLedger. type rematerializeLedger struct { @@ -29,17 +35,21 @@ func NewRematerializeLedger(db *sql.DB) posts.RematerializeLedger { return &rematerializeLedger{db: db} } +// ledgerColumns is the one SELECT list every read uses, so a column added to the +// row struct cannot be scanned by one query and forgotten by another. +const ledgerColumns = `old_uri, state, author_did, community_did, source_cid, new_uri, new_cid, new_rkey, reason, created_at, updated_at` + // Discover upserts the row for oldURI in state discovered, idempotently: a // re-run finds the existing row (whatever state it stands in) rather than // resetting it, then reads it back so the caller resumes from where it stopped. -func (l *rematerializeLedger) Discover(ctx context.Context, oldURI, authorDID string) (posts.RematerializeLedgerRow, error) { +func (l *rematerializeLedger) Discover(ctx context.Context, oldURI, communityDID, authorDID string) (posts.RematerializeLedgerRow, error) { // ON CONFLICT DO NOTHING keeps a resumed row untouched; a plain INSERT would // reset an in-flight row back to discovered and re-do the whole migration. _, err := l.db.ExecContext(ctx, ` - INSERT INTO post_rematerialization_ledger (old_uri, state, author_did, created_at, updated_at) - VALUES ($1, $2, $3, NOW(), NOW()) + INSERT INTO post_rematerialization_ledger (old_uri, state, author_did, community_did, created_at, updated_at) + VALUES ($1, $2, $3, $4, NOW(), NOW()) ON CONFLICT (old_uri) DO NOTHING - `, oldURI, string(posts.RematerializeDiscovered), nullString(authorDID)) + `, oldURI, string(posts.RematerializeDiscovered), nullString(authorDID), nullString(communityDID)) if err != nil { return posts.RematerializeLedgerRow{}, fmt.Errorf("discovering %s: %w", oldURI, err) } @@ -56,47 +66,36 @@ func (l *rematerializeLedger) Discover(ctx context.Context, oldURI, authorDID st // Get reads one row. found is false when the URI has never been discovered. func (l *rematerializeLedger) Get(ctx context.Context, oldURI string) (posts.RematerializeLedgerRow, bool, error) { - var ( - row posts.RematerializeLedgerRow - state string - authorDID sql.NullString - newURI sql.NullString - newCID sql.NullString - newRkey sql.NullString - reason sql.NullString - ) - err := l.db.QueryRowContext(ctx, ` - SELECT old_uri, state, author_did, new_uri, new_cid, new_rkey, reason, created_at, updated_at + row, err := scanLedgerRow(l.db.QueryRowContext(ctx, ` + SELECT `+ledgerColumns+` FROM post_rematerialization_ledger WHERE old_uri = $1 - `, oldURI).Scan(&row.OldURI, &state, &authorDID, &newURI, &newCID, &newRkey, &reason, &row.CreatedAt, &row.UpdatedAt) + `, oldURI)) if err == sql.ErrNoRows { return posts.RematerializeLedgerRow{}, false, nil } if err != nil { return posts.RematerializeLedgerRow{}, false, fmt.Errorf("reading ledger row %s: %w", oldURI, err) } - - row.State = posts.RematerializeState(state) - row.AuthorDID = authorDID.String - row.NewURI = newURI.String - row.NewCID = newCID.String - row.NewRkey = newRkey.String - row.Reason = reason.String return row, true, nil } // ListResumable returns every row still in a non-terminal state — the ledger- -// driven resume set (whole-branch review, P7). A migrated row whose delete -// succeeded but whose MarkDone crashed is GONE from the community repo, so only -// this query — never the source's listRecords — can rediscover it. -func (l *rematerializeLedger) ListResumable(ctx context.Context) ([]posts.RematerializeLedgerRow, error) { +// driven resume set (whole-branch review, P7) — restricted to communityDID when +// it is non-empty. +// +// A migrated row whose delete succeeded but whose MarkDone crashed is GONE from +// the community repo, so only this query — never the source's listRecords — can +// rediscover it. And a staged run must not rediscover ANOTHER community's row, +// because the very next thing the tool does with one is delete its record. +func (l *rematerializeLedger) ListResumable(ctx context.Context, communityDID string) ([]posts.RematerializeLedgerRow, error) { rows, err := l.db.QueryContext(ctx, ` - SELECT old_uri, state, author_did, new_uri, new_cid, new_rkey, reason, created_at, updated_at + SELECT `+ledgerColumns+` FROM post_rematerialization_ledger - WHERE state NOT IN ('done', 'fallback_left_legacy', 'fallback_no_creds') + WHERE state NOT IN ('done', 'fallback_left_legacy') + AND ($1 = '' OR community_did = $1) ORDER BY created_at - `) + `, communityDID) if err != nil { return nil, fmt.Errorf("listing resumable ledger rows: %w", err) } @@ -104,38 +103,28 @@ func (l *rematerializeLedger) ListResumable(ctx context.Context) ([]posts.Remate var out []posts.RematerializeLedgerRow for rows.Next() { - var ( - row posts.RematerializeLedgerRow - state string - authorDID sql.NullString - newURI sql.NullString - newCID sql.NullString - newRkey sql.NullString - reason sql.NullString - ) - if err := rows.Scan(&row.OldURI, &state, &authorDID, &newURI, &newCID, &newRkey, &reason, &row.CreatedAt, &row.UpdatedAt); err != nil { + row, err := scanLedgerRow(rows) + if err != nil { return nil, fmt.Errorf("scanning a resumable ledger row: %w", err) } - row.State = posts.RematerializeState(state) - row.AuthorDID = authorDID.String - row.NewURI = newURI.String - row.NewCID = newCID.String - row.NewRkey = newRkey.String - row.Reason = reason.String out = append(out, row) } return out, rows.Err() } -// RecordPostV2Written moves discovered → postv2_written and records the postv2 -// coordinates the resume path reads back. -func (l *rematerializeLedger) RecordPostV2Written(ctx context.Context, oldURI, newURI, newCID, newRkey string) error { +// RecordPostV2Written moves discovered → postv2_written and records both the +// postv2 coordinates the resume path reads back and the SOURCE CID the postv2 +// was built from — the value every later pre-delete check is made against. +func (l *rematerializeLedger) RecordPostV2Written(ctx context.Context, oldURI, sourceCID, newURI, newCID, newRkey string) error { + if sourceCID == "" { + return fmt.Errorf("recording the postv2 of %s: no source CID was supplied, so no later delete could be guarded against a concurrent edit", oldURI) + } return l.guardedTransition(ctx, ` UPDATE post_rematerialization_ledger - SET state = $2, new_uri = $3, new_cid = $4, new_rkey = $5, updated_at = NOW() - WHERE old_uri = $1 AND state = $6 + SET state = $2, source_cid = $3, new_uri = $4, new_cid = $5, new_rkey = $6, updated_at = NOW() + WHERE old_uri = $1 AND state = $7 `, "postv2_written", oldURI, - string(posts.RematerializePostV2Written), newURI, newCID, newRkey, string(posts.RematerializeDiscovered)) + string(posts.RematerializePostV2Written), sourceCID, newURI, newCID, newRkey, string(posts.RematerializeDiscovered)) } // MarkVerified moves postv2_written → verified. @@ -168,9 +157,11 @@ func (l *rematerializeLedger) MarkDone(ctx context.Context, oldURI string) error string(posts.RematerializeDone), string(posts.RematerializeMigrated)) } -// MarkFallback moves a discovered row to a terminal fallback state with a reason. -// The from-state guard is intentionally broad — a fallback is only ever reached -// from discovered in cycle 1 — but the reason is always recorded for the census. +// MarkFallback moves a discovered row to a terminal fallback state with a +// reason. The from-state guard is discovered ONLY: a row that has already had a +// postv2 written for it is past the point where "leave it as legacy" is a +// coherent verdict, and re-marking a row that is already a fallback would +// overwrite the reason the operator is about to read. func (l *rematerializeLedger) MarkFallback(ctx context.Context, oldURI string, state posts.RematerializeState, reason string) error { if !posts.IsFallback(state) { return fmt.Errorf("marking %s as fallback: %q is not a fallback state", oldURI, state) @@ -183,12 +174,46 @@ func (l *rematerializeLedger) MarkFallback(ctx context.Context, oldURI string, s string(state), reason, string(posts.RematerializeDiscovered)) } -// CountByState is the census: how many rows sit in each state, so the run can -// refuse "complete" while any fallback survives. -func (l *rematerializeLedger) CountByState(ctx context.Context) (map[posts.RematerializeState]int, error) { +// ReopenFallback moves fallback rows back to discovered so a later run can retry +// them, restricted to communityDID when it is non-empty. +// +// THIS IS THE ONLY WAY BACK OUT OF A FALLBACK, and it exists because without it +// there is none. A missing author grant sentences a row terminally; the operator +// re-authorizes the author and every subsequent run is a permanent no-op over +// those posts, with the only remedy an UPDATE statement typed by hand against a +// production table. Zero rows moved is NOT an error here — "there was nothing to +// reopen" is a legitimate, common answer, and this is not one of the ordered +// transitions whose no-op means divergence. +// +// It moves rows only from a fallback state to discovered. It cannot resurrect a +// done row, it clears no postv2 coordinates, and it writes to no repo. +func (l *rematerializeLedger) ReopenFallback(ctx context.Context, communityDID string) (int, error) { + res, err := l.db.ExecContext(ctx, ` + UPDATE post_rematerialization_ledger + SET state = $1, reason = NULL, updated_at = NOW() + WHERE state = $2 + AND ($3 = '' OR community_did = $3) + `, string(posts.RematerializeDiscovered), string(posts.RematerializeFallbackLeftLegacy), communityDID) + if err != nil { + return 0, fmt.Errorf("reopening fallback rows: %w", err) + } + affected, err := res.RowsAffected() + if err != nil { + return 0, fmt.Errorf("reopening fallback rows: reading rows affected: %w", err) + } + return int(affected), nil +} + +// CountByState is the census: how many rows sit in each state, restricted to +// communityDID when it is non-empty, so a staged run can report on its own scope +// and on the whole migration as two separate facts. +func (l *rematerializeLedger) CountByState(ctx context.Context, communityDID string) (map[posts.RematerializeState]int, error) { rows, err := l.db.QueryContext(ctx, ` - SELECT state, COUNT(*) FROM post_rematerialization_ledger GROUP BY state - `) + SELECT state, COUNT(*) + FROM post_rematerialization_ledger + WHERE ($1 = '' OR community_did = $1) + GROUP BY state + `, communityDID) if err != nil { return nil, fmt.Errorf("counting ledger rows by state: %w", err) } @@ -209,6 +234,36 @@ func (l *rematerializeLedger) CountByState(ctx context.Context) (map[posts.Remat return counts, nil } +// scanLedgerRow reads one row in the ledgerColumns order. It takes the package's +// shared rowScanner (post_repo.go) so one scan body serves both the single-row +// Get and the batched ListResumable. +func scanLedgerRow(src rowScanner) (posts.RematerializeLedgerRow, error) { + var ( + row posts.RematerializeLedgerRow + state string + authorDID sql.NullString + communityDID sql.NullString + sourceCID sql.NullString + newURI sql.NullString + newCID sql.NullString + newRkey sql.NullString + reason sql.NullString + ) + if err := src.Scan(&row.OldURI, &state, &authorDID, &communityDID, &sourceCID, + &newURI, &newCID, &newRkey, &reason, &row.CreatedAt, &row.UpdatedAt); err != nil { + return posts.RematerializeLedgerRow{}, err + } + row.State = posts.RematerializeState(state) + row.AuthorDID = authorDID.String + row.CommunityDID = communityDID.String + row.SourceCID = sourceCID.String + row.NewURI = newURI.String + row.NewCID = newCID.String + row.NewRkey = newRkey.String + row.Reason = reason.String + return row, nil +} + // guardedTransition runs a from-state-guarded UPDATE and treats a no-op as the // error it is: the tool only ever fires a transition when the row stands in the // expected state, so matching no row means the ledger and the tool diverged. diff --git a/internal/db/postgres/rematerialize_ledger_schema_test.go b/internal/db/postgres/rematerialize_ledger_schema_test.go index bc84e7a..f45ea6d 100644 --- a/internal/db/postgres/rematerialize_ledger_schema_test.go +++ b/internal/db/postgres/rematerialize_ledger_schema_test.go @@ -30,10 +30,16 @@ import ( // delete, and a delete of an already-gone record is success — which is only // coherent if the checkpoint BEFORE the delete is its own persisted state. // -// The two fallback states are the credential census (§11 step 3): a record whose +// The fallback state is the credential census (§11 step 3): a record whose // author credentials cannot be restored is left as legacy, never re-authored // under a forged signature, and the run refuses to report "complete" while any // such row survives. +// +// There is exactly ONE fallback state, and the schema is where that is enforced. +// An earlier revision also admitted 'fallback_no_creds' and NO code path ever +// wrote it. A state the vocabulary permits but nothing produces is worse than +// missing: it is what an operator writes recovery SQL against at 2am, silently +// matching nothing, and it is a second name one WHERE clause eventually forgets. const rematerializeLedgerTable = "post_rematerialization_ledger" @@ -70,6 +76,18 @@ func TestRematerializeLedgerTable_Columns(t *testing.T) { "new_cid": {"text", true}, "new_rkey": {"text", true}, + // The community repo the legacy record lives in. STORED rather than + // parsed back out of old_uri, because the destructive half of the tool is + // scoped by it: a staged run resumes, counts and DELETES only rows + // carrying this value. + "community_did": {"text", true}, + + // The legacy record's CID as of the read the postv2 was built from. It is + // a safety interlock, not audit trim: the delete is refused unless a fresh + // read still reports it, and it is the swapRecord the delete is sent + // under, so the PDS refuses a stale delete independently. + "source_cid": {"text", true}, + // The human-readable note on a fallback row. "reason": {"text", true}, @@ -125,11 +143,17 @@ func TestRematerializeLedgerTable_StateVocabularyIsClosed(t *testing.T) { "migrated", "done", "fallback_left_legacy", - "fallback_no_creds", } { assert.Containsf(t, all, state, "the state CHECK constraint must admit %q; a missing state is one the tool can never persist", state) } + + // The inverse, asserted rather than merely omitted: a state nothing produces + // must not be in the vocabulary at all. Leaving it admitted is how recovery + // SQL comes to be written against a state that cannot exist. + assert.NotContainsf(t, all, "fallback_no_creds", + "the CHECK constraint still admits 'fallback_no_creds', which no code path writes. One cause — the author-repo factory reporting no restorable "+ + "credentials — has exactly one state, fallback_left_legacy; a second admitted spelling is a trap for whoever writes the recovery UPDATE") } func TestRematerializeLedgerTable_ValidStatesInsertAndBogusIsRejected(t *testing.T) { @@ -142,7 +166,7 @@ func TestRematerializeLedgerTable_ValidStatesInsertAndBogusIsRejected(t *testing // migrated and done are BOTH valid and DISTINCT — the checkpoint-before-delete // property depends on it. A bogus state must be refused at the schema, where // every writer meets the constraint, not left to repository discipline. - valid := []string{"discovered", "postv2_written", "verified", "migrated", "done", "fallback_left_legacy", "fallback_no_creds"} + valid := []string{"discovered", "postv2_written", "verified", "migrated", "done", "fallback_left_legacy"} for i, state := range valid { oldURI := "at://did:plc:community2222222222222222/social.coves.community.post/valid" + string(rune('a'+i)) _, err := db.ExecContext(ctx, ` @@ -152,12 +176,14 @@ func TestRematerializeLedgerTable_ValidStatesInsertAndBogusIsRejected(t *testing require.NoErrorf(t, err, "state %q must be a permitted ledger state", state) } - _, err := db.ExecContext(ctx, ` - INSERT INTO post_rematerialization_ledger (old_uri, state, created_at, updated_at) - VALUES ($1, $2, NOW(), NOW()) - `, "at://did:plc:community2222222222222222/social.coves.community.post/bogus", "half_migrated") - require.Errorf(t, err, - "an unknown state 'half_migrated' was accepted; the vocabulary must be closed by a CHECK, or a typo lands as a row nothing resumes") + for _, bogus := range []string{"half_migrated", "fallback_no_creds"} { + _, err := db.ExecContext(ctx, ` + INSERT INTO post_rematerialization_ledger (old_uri, state, created_at, updated_at) + VALUES ($1, $2, NOW(), NOW()) + `, "at://did:plc:community2222222222222222/social.coves.community.post/bogus-"+bogus, bogus) + require.Errorf(t, err, + "the state %q was accepted; the vocabulary must be closed by a CHECK, or a typo — or a state no code path writes — lands as a row nothing resumes", bogus) + } } func TestRematerializeLedgerMigration_RollsBack(t *testing.T) { diff --git a/internal/db/postgres/rematerialize_ledger_test.go b/internal/db/postgres/rematerialize_ledger_test.go new file mode 100644 index 0000000..a0a9f26 --- /dev/null +++ b/internal/db/postgres/rematerialize_ledger_test.go @@ -0,0 +1,307 @@ +//go:build integration + +package postgres + +import ( + "context" + "testing" + + "Coves/internal/core/posts" + "Coves/tests/testkit" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// THE LEDGER'S FROM-STATE GUARDS, and the divergence error they exist to raise. +// +// Every state-advancing UPDATE carries `AND state = $N`. That clause is the only +// thing standing between the cutover tool and a ledger that says a post is safe +// to delete when it is not: without it, MarkMigrated on a row still at +// `discovered` succeeds, the row reads as "verified, safe to delete", and the +// next pass deletes a legacy record whose replacement was never written. +// +// It is also the only thing that makes two concurrent runs SAY SO rather than +// interleave silently. Zero rows affected is an error on purpose. +// +// Stripping `AND state = $N` from all five guarded UPDATEs used to keep the +// whole suite green. This file is what makes that mutation fail. + +// ledgerFixture stages one row at a chosen state and hands back the ledger. +func ledgerFixture(t *testing.T, at posts.RematerializeState) (posts.RematerializeLedger, string) { + t.Helper() + db := testkit.DB(t) + ledger := NewRematerializeLedger(db) + ctx := context.Background() + + oldURI := "at://did:plc:community2222222222222222/social.coves.community.post/" + testkit.TID() + communityDID := "did:plc:community2222222222222222" + authorDID := "did:plc:author11111111111111111" + + _, err := ledger.Discover(ctx, oldURI, communityDID, authorDID) + require.NoError(t, err) + + // Walk the machine forward to the requested state through its own transitions, + // so the fixture cannot stage a state the machine could not reach. + newRkey := "3kremat" + testkit.TID() + newURI := "at://" + authorDID + "/social.coves.community.postv2/" + newRkey + steps := []struct { + reaches posts.RematerializeState + do func() error + }{ + {posts.RematerializePostV2Written, func() error { + return ledger.RecordPostV2Written(ctx, oldURI, "bafysource", newURI, "bafynew", newRkey) + }}, + {posts.RematerializeVerified, func() error { return ledger.MarkVerified(ctx, oldURI) }}, + {posts.RematerializeMigrated, func() error { return ledger.MarkMigrated(ctx, oldURI) }}, + {posts.RematerializeDone, func() error { return ledger.MarkDone(ctx, oldURI) }}, + } + if at == posts.RematerializeFallbackLeftLegacy { + require.NoError(t, ledger.MarkFallback(ctx, oldURI, posts.RematerializeFallbackLeftLegacy, "staged")) + return ledger, oldURI + } + for _, step := range steps { + if at == posts.RematerializeDiscovered { + break + } + require.NoError(t, step.do()) + if step.reaches == at { + break + } + } + + row, found, err := ledger.Get(ctx, oldURI) + require.NoError(t, err) + require.True(t, found) + require.Equalf(t, at, row.State, "the fixture failed to stage the row at %s", at) + return ledger, oldURI +} + +func TestRematerializeLedger_EveryTransitionIsGuardedOnItsPriorState(t *testing.T) { + t.Parallel() + + allStates := []posts.RematerializeState{ + posts.RematerializeDiscovered, + posts.RematerializePostV2Written, + posts.RematerializeVerified, + posts.RematerializeMigrated, + posts.RematerializeDone, + posts.RematerializeFallbackLeftLegacy, + } + + transitions := []struct { + name string + from posts.RematerializeState + fire func(ledger posts.RematerializeLedger, oldURI string) error + // why says what a missing guard would cost. + why string + }{ + { + name: "RecordPostV2Written", from: posts.RematerializeDiscovered, + fire: func(l posts.RematerializeLedger, uri string) error { + return l.RecordPostV2Written(context.Background(), uri, "bafysource2", "at://x/y/z", "bafynew2", "3kagain") + }, + why: "unguarded, it would overwrite the postv2 coordinates of a row that is already past the write — pointing a later delete at a record " + + "this run never verified", + }, + { + name: "MarkVerified", from: posts.RematerializePostV2Written, + fire: func(l posts.RematerializeLedger, uri string) error { return l.MarkVerified(context.Background(), uri) }, + why: "unguarded, it would mark a row `verified` that has no postv2 at all", + }, + { + name: "MarkMigrated", from: posts.RematerializeVerified, + fire: func(l posts.RematerializeLedger, uri string) error { return l.MarkMigrated(context.Background(), uri) }, + why: "unguarded, it would write the CHECKPOINT BEFORE DELETE onto a row nothing has been verified for — and `migrated` is exactly what a " + + "resumed run reads as 'the delete is safe, just retry it'", + }, + { + name: "MarkDone", from: posts.RematerializeMigrated, + fire: func(l posts.RematerializeLedger, uri string) error { return l.MarkDone(context.Background(), uri) }, + why: "unguarded, it would mark a row done — meaning 'the legacy record has been deleted' — for a record that is still standing, so the " + + "census reports a drain that never happened", + }, + { + name: "MarkFallback", from: posts.RematerializeDiscovered, + fire: func(l posts.RematerializeLedger, uri string) error { + return l.MarkFallback(context.Background(), uri, posts.RematerializeFallbackLeftLegacy, "second thoughts") + }, + why: "unguarded, it would sentence a row that already has a postv2 written for it — abandoning work that succeeded, and overwriting the " + + "reason an operator is about to read", + }, + } + + for _, tr := range transitions { + for _, priorState := range allStates { + if priorState == tr.from { + continue + } + t.Run(tr.name+"_from_"+string(priorState), func(t *testing.T) { + t.Parallel() + ledger, oldURI := ledgerFixture(t, priorState) + + err := tr.fire(ledger, oldURI) + + require.Errorf(t, err, + "%s succeeded against a row standing at %s (it is guarded on %s). %s.\nA transition that finds no row in its expected prior state "+ + "means the ledger and the tool have diverged, and that is a fault to surface, not to swallow into a false success", + tr.name, priorState, tr.from, tr.why) + assert.Containsf(t, err.Error(), "diverged", + "the error must name the divergence: a 3am operator reading it needs to know the ledger disagrees with the tool, not that some SQL failed") + + row, found, err := ledger.Get(context.Background(), oldURI) + require.NoError(t, err) + require.True(t, found) + assert.Equalf(t, priorState, row.State, + "the refused transition CHANGED the row anyway: it now stands at %s. A guard that updates and then complains is not a guard", row.State) + }) + } + } +} + +// The happy path still has to work — a guard that refuses everything would pass +// every test above. +func TestRematerializeLedger_TheOrderedTransitionsSucceedFromTheirOwnPriorState(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := NewRematerializeLedger(db) + ctx := context.Background() + + oldURI := "at://did:plc:community2222222222222222/social.coves.community.post/" + testkit.TID() + _, err := ledger.Discover(ctx, oldURI, "did:plc:community2222222222222222", "did:plc:author11111111111111111") + require.NoError(t, err) + + require.NoError(t, ledger.RecordPostV2Written(ctx, oldURI, "bafysource", "at://a/b/c", "bafynew", "3krkey")) + require.NoError(t, ledger.MarkVerified(ctx, oldURI)) + require.NoError(t, ledger.MarkMigrated(ctx, oldURI)) + require.NoError(t, ledger.MarkDone(ctx, oldURI)) + + row, found, err := ledger.Get(ctx, oldURI) + require.NoError(t, err) + require.True(t, found) + assert.Equal(t, posts.RematerializeDone, row.State) + assert.Equalf(t, "bafysource", row.SourceCID, + "the source CID must round-trip: it is what the pre-delete re-read is compared against and what the delete's swap guard is made of") + assert.Equal(t, "at://a/b/c", row.NewURI) + assert.Equal(t, "bafynew", row.NewCID) + assert.Equal(t, "3krkey", row.NewRkey) + assert.Equalf(t, "did:plc:community2222222222222222", row.CommunityDID, + "the community DID must round-trip: it is the scope a staged run resumes, counts and deletes within") +} + +// A postv2 recorded with no source CID leaves nothing to guard the eventual +// delete on, so it is refused at the ledger rather than discovered later. +func TestRematerializeLedger_RecordPostV2Written_RequiresASourceCID(t *testing.T) { + t.Parallel() + ledger, oldURI := ledgerFixture(t, posts.RematerializeDiscovered) + + err := ledger.RecordPostV2Written(context.Background(), oldURI, "", "at://a/b/c", "bafynew", "3krkey") + require.Errorf(t, err, + "a postv2 was recorded with no source CID. The source CID is the ONLY thing a later pass can check the legacy record against before deleting it; "+ + "a row without one either blocks forever or deletes blind") + + row, _, err := ledger.Get(context.Background(), oldURI) + require.NoError(t, err) + assert.Equal(t, posts.RematerializeDiscovered, row.State) +} + +// ---- scope ------------------------------------------------------------------ + +// ListResumable, CountByState and ReopenFallback are all scoped, because a +// staged run that resumes another community's rows will DELETE that community's +// records. +func TestRematerializeLedger_ScopedQueriesDoNotReachAnotherCommunity(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := NewRematerializeLedger(db) + ctx := context.Background() + + mine := "did:plc:mycommunity222222222222" + theirs := "did:plc:theircommunity33333333" + mineURI := "at://" + mine + "/social.coves.community.post/" + testkit.TID() + theirsURI := "at://" + theirs + "/social.coves.community.post/" + testkit.TID() + + _, err := ledger.Discover(ctx, mineURI, mine, "did:plc:author11111111111111111") + require.NoError(t, err) + _, err = ledger.Discover(ctx, theirsURI, theirs, "did:plc:author11111111111111111") + require.NoError(t, err) + + resumable, err := ledger.ListResumable(ctx, mine) + require.NoError(t, err) + for _, row := range resumable { + assert.Equalf(t, mine, row.CommunityDID, + "a run scoped to %s was handed %s's row to resume. The very next thing the tool does with a resumable row is drive it toward a DELETE", + mine, row.CommunityDID) + } + assert.Lenf(t, resumable, 1, "the scoped resume set must contain exactly this community's row") + + counts, err := ledger.CountByState(ctx, mine) + require.NoError(t, err) + assert.Equalf(t, 1, counts[posts.RematerializeDiscovered], + "the scoped census counted %d discovered rows; a census that reaches outside its scope makes every staged run report itself incomplete", + counts[posts.RematerializeDiscovered]) + + global, err := ledger.CountByState(ctx, "") + require.NoError(t, err) + assert.GreaterOrEqualf(t, global[posts.RematerializeDiscovered], 2, + "the UNSCOPED census must see both communities: it is what gates the irreversible legacy-removal step, which is global") +} + +func TestRematerializeLedger_ReopenFallback_IsScopedAndOnlyTouchesFallbackRows(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := NewRematerializeLedger(db) + ctx := context.Background() + + mine := "did:plc:mycommunity222222222222" + theirs := "did:plc:theircommunity33333333" + author := "did:plc:author11111111111111111" + + mineFallback := "at://" + mine + "/social.coves.community.post/" + testkit.TID() + theirsFallback := "at://" + theirs + "/social.coves.community.post/" + testkit.TID() + mineInFlight := "at://" + mine + "/social.coves.community.post/" + testkit.TID() + + for _, uri := range []string{mineFallback, theirsFallback, mineInFlight} { + communityDID := mine + if uri == theirsFallback { + communityDID = theirs + } + _, err := ledger.Discover(ctx, uri, communityDID, author) + require.NoError(t, err) + } + require.NoError(t, ledger.MarkFallback(ctx, mineFallback, posts.RematerializeFallbackLeftLegacy, "no creds")) + require.NoError(t, ledger.MarkFallback(ctx, theirsFallback, posts.RematerializeFallbackLeftLegacy, "no creds")) + require.NoError(t, ledger.RecordPostV2Written(ctx, mineInFlight, "bafysource", "at://a/b/c", "bafynew", "3krkey")) + + moved, err := ledger.ReopenFallback(ctx, mine) + require.NoError(t, err) + assert.Equalf(t, 1, moved, "ReopenFallback moved %d row(s); it must move only this community's fallback rows", moved) + + reopened, _, err := ledger.Get(ctx, mineFallback) + require.NoError(t, err) + assert.Equal(t, posts.RematerializeDiscovered, reopened.State) + assert.Emptyf(t, reopened.Reason, "a reopened row's stale fallback reason must be cleared, or the operator reads a verdict that no longer applies") + + untouched, _, err := ledger.Get(ctx, theirsFallback) + require.NoError(t, err) + assert.Equalf(t, posts.RematerializeFallbackLeftLegacy, untouched.State, + "ReopenFallback reached outside its scope and reopened another community's row") + + inFlight, _, err := ledger.Get(ctx, mineInFlight) + require.NoError(t, err) + assert.Equalf(t, posts.RematerializePostV2Written, inFlight.State, + "ReopenFallback moved a row that was not a fallback, discarding work that had already succeeded") +} + +// Zero rows reopened is NOT an error: "there was nothing to reopen" is a normal, +// common answer, and this is not one of the ordered transitions whose no-op +// means divergence. +func TestRematerializeLedger_ReopenFallback_ZeroRowsIsNotAnError(t *testing.T) { + t.Parallel() + db := testkit.DB(t) + ledger := NewRematerializeLedger(db) + + moved, err := ledger.ReopenFallback(context.Background(), "did:plc:nothingtoreopen1111111") + require.NoErrorf(t, err, "reopening a scope with no fallback rows must be a quiet success, not a divergence error") + assert.Equal(t, 0, moved) +}