diff --git a/bench/README.md b/bench/README.md index 2f9624d..93cf003 100644 --- a/bench/README.md +++ b/bench/README.md @@ -106,6 +106,7 @@ blob, then measures these paths as distinct units of work: - `getRecord` by `(space, repo, collection, rkey)` - `listRecords`, limit 50, with values included by default - full-state `getRepo` CAR construction, including signed commit and record index +- full-state permissioned repo CAR verification for migration and backfill - `getBlob` exact-space reference authorization plus storage readback - `listRepoOps` catch-up reads diff --git a/bench/main.zig b/bench/main.zig index 6ff6f66..34ff41b 100644 --- a/bench/main.zig +++ b/bench/main.zig @@ -835,6 +835,7 @@ fn benchSpace(allocator: std.mem.Allocator, options: Options) !void { (try benchSpaceListSpaces(allocator, state.account, space.uri)).print(); (try benchSimpleSpaceListMembers(allocator, space.uri)).print(); (try benchSpaceRepoCar(allocator, state.account, space.uri)).print(); + (try benchSpaceRepoVerify(allocator, state.account, space.uri)).print(); (try benchSpaceWrite(allocator, state.account, space.uri, records)).print(); (try benchSpaceGetRecord(allocator, state.account, space.uri, records)).print(); (try benchSpaceListRecords(allocator, state.account, space.uri, records)).print(); @@ -869,6 +870,35 @@ fn benchSpaceRepoCar(allocator: std.mem.Allocator, account: zds.auth.tokens.Acco return .{ .name = "space getRepo", .ops = iterations, .bytes = total_bytes, .elapsed_ns = nowNs() - start }; } +fn benchSpaceRepoVerify(allocator: std.mem.Allocator, account: zds.auth.tokens.Account, space: []const u8) !BenchResult { + var fixture_arena = std.heap.ArenaAllocator.init(allocator); + defer fixture_arena.deinit(); + const a = fixture_arena.allocator(); + const state = try zds.storage.store.getSpaceRepoState(a, space, account.did); + var keypair = try zds.storage.store.signingKeypair(account.did); + const commit = try zds.internal.permissioned_data.createCommit(a, std.Options.debug_io, state.set_hash orelse return error.MissingRecords, .{ + .space = space, + .author = account.did, + .rev = state.rev orelse return error.MissingRecords, + }, &keypair); + const records = try zds.storage.store.loadSpaceRepoBlocks(a, space, account.did); + const car = try zds.internal.permissioned_data.serializeRepoCar(a, commit, records); + const did_key = try keypair.did(a); + + const iterations: usize = 20; + const start = nowNs(); + for (0..iterations) |_| { + var verified = try zds.internal.permissioned_data.verifyRepoCarFull( + allocator, + car, + .{ .space = space, .author = account.did }, + did_key, + ); + verified.deinit(); + } + return .{ .name = "space verifyRepo", .ops = iterations, .bytes = car.len * iterations, .elapsed_ns = nowNs() - start }; +} + fn seedSpaceRecords( allocator: std.mem.Allocator, account: zds.auth.tokens.Account, diff --git a/docs/space-host-migration.md b/docs/space-host-migration.md new file mode 100644 index 0000000..5c24af7 --- /dev/null +++ b/docs/space-host-migration.md @@ -0,0 +1,114 @@ +# space-host migration notes + +Status: exploratory. These notes describe the current proposal and local +experiments; they do not define a ZDS extension or promise a migration API. + +## Separate the two moves + +Permissioned data has two independently hosted kinds of state: + +- A **permissioned repo** is one author's records in one space. It lives on + that author's repo host, usually their PDS. +- A **space host** serves the authority's configuration, mints space + credentials, maintains the writer set, and routes write notifications. + +Proposal 0016 explicitly says moving a permissioned repo works like moving a +public repo, except an account can have many permissioned repos and their blob +sets must be enumerated. It does not define a space-host export, import, or +takeover procedure. + +## What is portable today + +The stable space coordinate is rooted in the authority DID. That DID may +publish a dedicated `#atproto_space_host` endpoint and `#atproto_space` signing +key, with fallbacks to `#atproto_pds` and `#atproto`. + +Changing the host endpoint does not change the space URI. A space credential +also contains the authority DID, space URI, holder DPoP thumbprint, and optional +client ID, but not the host URL. Existing credentials therefore remain valid +across an endpoint move while the authority key remains valid. Changing key +custody is a separate identity operation. + +The permissioned repo artifact is also portable. `com.atproto.space.getRepo` +returns a two-root CAR containing the signed deniable commit, authenticated +record index, and record blocks. Blobs are enumerated and fetched separately. +The oplog is explicitly disposable and starts fresh on a new repo host. + +Local two-way conformance evidence: + +- ZDS verifies the committed pds.js fixture generated by the official + `@atproto/space` implementation at proposal PR 5187 commit + `14e3ed0ec5219c58e33eb66a3efe049a3bf2b78f`. +- The official verifier on the current proposal branch accepted a CAR emitted + by a fresh local ZDS `smoke-permissioned` instance. +- The checks cover the commit signature and MAC, LtHash, canonical index, + record CIDs and values, and binding to the space and author. + +The local permissioned benchmark verifies a 1,001-record export in about +`2.29 ms` and a 10,001-record export in about `19.64 ms` on the development +machine. The larger run also caught and removed an internal mismatch where the +generic commit-CAR reader's 10,000-block default rejected a valid full-state +export containing 10,000 records plus its commit and index roots. Verification +now remains bounded by the supplied CAR's byte length rather than that smaller +commit-transport ceiling. + +A separate two-host ZDS experiment copied only a throwaway authority account +and its signing key from host A to a fresh host B. An A-issued credential +received `404 SpaceNotFound` from B before the space existed. After the same +simple-space coordinate and configuration were recreated on B, that unchanged +credential received `200` from B. B still had no writer-registry entries or +records. This demonstrates both halves of the boundary: credential continuity +comes from the authority identity and key, while control state does not follow +the DID endpoint automatically. + +## What a space host owns + +In ZDS, the authority-side state is currently spread across these concepts: + +- space type and key +- simple-space policy, managing app, and app-access configuration +- simple-space member list +- writer registry with each repo's latest revision and hash +- notification registrations +- authority signing-key custody through the resident account + +Writer record blocks and blobs are not space-host state and must not be copied +as part of a host-only move. + +The proposal intentionally permits custom host policy. Consequently, a generic +destination cannot promise to reproduce an arbitrary source host's policy. +Simple-space configuration is structurally transferable; application-specific +policy is transferable only when the destination implements a compatible +policy model or the application can reconstruct it. + +## Current ZDS gap + +ZDS can resolve a dedicated `#atproto_space_host` endpoint and verify a +dedicated `#atproto_space` key, but it does not yet host a space under a +standalone authority key. Space creation and credential minting are tied to a +resident account and its signing key. ZDS also has no host-state export/import +operation. + +This means a normal resident account migration can move the fallback PDS and +space-host roles together in principle, but independent space-host migration +is not yet a complete ZDS workflow. Copying SQLite rows would demonstrate +storage mechanics, not protocol interoperability. + +## Next local experiment + +Use throwaway identities and no resident production data: + +1. Create a simple space on host A and add two writers on distinct repo hosts. +2. Sync each writer through `listRepos`, `getRepo`, `listBlobs`, and `getBlob`. +3. Repoint the authority DID's `#atproto_space_host` service to host B while + preserving the authority key. +4. Reconstruct the simple-space config and members on B through an explicit, + inspectable transfer artifact. +5. Rebuild or import the writer registry, renew notification registrations, + and verify that old credentials still read while new credentials mint at B. +6. Confirm that no writer records or blobs were copied to the space host and + that a dropped notification is repaired by comparing `listRepos` revisions. + +Before implementing that experiment over HTTP, ZDS needs a clean authority-key +boundary and a deliberately scoped host-state artifact. Neither should be +smuggled into the public XRPC surface before cross-implementation discussion. diff --git a/src/internal/permissioned_data.zig b/src/internal/permissioned_data.zig index da80f0b..223b8d0 100644 --- a/src/internal/permissioned_data.zig +++ b/src/internal/permissioned_data.zig @@ -83,6 +83,16 @@ pub const RepoRecordBlock = struct { data: []const u8, }; +pub const VerifiedRepo = struct { + arena: std.heap.ArenaAllocator, + commit: SignedCommit, + records: []const RepoRecordBlock, + + pub fn deinit(self: *VerifiedRepo) void { + self.arena.deinit(); + } +}; + pub const DelegationToken = struct { requester_did: []const u8, authority_did: []const u8, @@ -183,6 +193,95 @@ pub fn serializeRepoCar( }); } +/// Verify and materialize a full-state permissioned repo CAR. The returned +/// value owns all of its data and must be deinitialized by the caller. +pub fn verifyRepoCarFull( + allocator: std.mem.Allocator, + car_bytes: []const u8, + ctx: struct { space: []const u8, author: []const u8 }, + author_did_key: []const u8, +) !VerifiedRepo { + var arena = std.heap.ArenaAllocator.init(allocator); + errdefer arena.deinit(); + const a = arena.allocator(); + const owned_car = try a.dupe(u8, car_bytes); + // Full-state exports routinely exceed the generic commit-CAR block cap. + // The input is already resident in memory, so derive a structural ceiling + // from its byte length: every CAR block consumes at least a length byte and + // a minimally encoded CID. + const max_blocks = owned_car.len / 5 + 1; + const car = try zat.car.readWithOptions(a, owned_car, .{ + .max_size = owned_car.len, + .max_blocks = max_blocks, + }); + if (car.roots.len != 2 or car.blocks.len < 2) return error.InvalidRepoCar; + if (!std.mem.eql(u8, car.roots[0].raw, car.blocks[0].cid_raw)) return error.InvalidRepoCar; + if (!std.mem.eql(u8, car.roots[1].raw, car.blocks[1].cid_raw)) return error.InvalidRepoCar; + + const commit_value = try zat.cbor.decodeAll(a, car.blocks[0].data); + if (commit_value.getUint("ver") != 1) return error.InvalidRepoCommit; + const hash = commit_value.getBytes("hash") orelse return error.InvalidRepoCommit; + const mac = commit_value.getBytes("mac") orelse return error.InvalidRepoCommit; + const ikm = commit_value.getBytes("ikm") orelse return error.InvalidRepoCommit; + const sig = commit_value.getBytes("sig") orelse return error.InvalidRepoCommit; + const rev = commit_value.getString("rev") orelse return error.InvalidRepoCommit; + if (hash.len != 32 or mac.len != 32 or ikm.len != 32 or sig.len != 64) return error.InvalidRepoCommit; + + var commit: SignedCommit = .{ + .hash = undefined, + .mac = undefined, + .ikm = undefined, + .sig = undefined, + .rev = try a.dupe(u8, rev), + }; + @memcpy(&commit.hash, hash); + @memcpy(&commit.mac, mac); + @memcpy(&commit.ikm, ikm); + @memcpy(&commit.sig, sig); + + const context = try commitContext(a, .{ + .space = ctx.space, + .author = ctx.author, + .rev = commit.rev, + }, &commit.ikm); + try zat.multicodec.verifyDidKeySignature(a, author_did_key, context, &commit.sig); + + const index = try zat.cbor.decodeAll(a, car.blocks[1].data); + const entries = switch (index) { + .map => |map| map, + else => return error.InvalidRepoIndex, + }; + if (car.blocks.len != entries.len + 2) return error.InvalidRepoCar; + + const records = try a.alloc(RepoRecordBlock, entries.len); + var set_hash: LtHash = .{}; + for (entries, 0..) |entry, idx| { + const cid = switch (entry.value) { + .cid => |value| value, + else => return error.InvalidRepoIndex, + }; + const slash = std.mem.indexOfScalar(u8, entry.key, '/') orelse return error.InvalidRepoPath; + if (std.mem.indexOfScalar(u8, entry.key[slash + 1 ..], '/') != null) return error.InvalidRepoPath; + const collection = entry.key[0..slash]; + const rkey = entry.key[slash + 1 ..]; + if (zat.Nsid.parse(collection) == null or zat.Rkey.parse(rkey) == null) return error.InvalidRepoPath; + + const block = car.blocks[idx + 2]; + if (!std.mem.eql(u8, cid.raw, block.cid_raw)) return error.InvalidRepoCar; + const record = try zat.cbor.decodeAll(a, block.data); + if (record != .map) return error.InvalidRepoRecord; + + const cid_text = try cid.toString(a); + set_hash.add(try recordElement(a, collection, rkey, cid_text)); + records[idx] = .{ .path = entry.key, .cid = cid, .data = block.data }; + } + if (!std.mem.eql(u8, &set_hash.digest(), &commit.hash)) return error.InvalidRepoIndex; + const expected_mac = commitMac(&commit.ikm, context, &commit.hash); + if (!std.crypto.timing_safe.eql([32]u8, expected_mac, commit.mac)) return error.InvalidRepoCommit; + + return .{ .arena = arena, .commit = commit, .records = records }; +} + fn base64Bytes(allocator: std.mem.Allocator, bytes: []const u8) ![]const u8 { const encoded = try allocator.alloc(u8, std.base64.standard.Encoder.calcSize(bytes.len)); _ = std.base64.standard.Encoder.encode(encoded, bytes); @@ -493,6 +592,54 @@ test "permissioned repo CAR has commit and index roots followed by sorted record try std.testing.expectEqualSlices(u8, second_cid.raw, index.get("fm.example.note/two").?.cid.raw); } +test "verifies a permissioned repo CAR produced by the reference implementation" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + // Generated by @atproto/space at bluesky-social/atproto PR 5187 commit + // 14e3ed0ec5219c58e33eb66a3efe049a3bf2b78f, via pds.js' two-way + // space-reference harness. + const encoded = std.mem.trim(u8, @embedFile("testdata/space-reference-repo.car.base64"), "\r\n"); + const car_bytes = try allocator.alloc(u8, try std.base64.standard.Decoder.calcSizeForSlice(encoded)); + try std.base64.standard.Decoder.decode(car_bytes, encoded); + + var verified = try verifyRepoCarFull( + allocator, + car_bytes, + .{ + .space = "at://did:plc:refauthority/space/com.example.forum/general", + .author = "did:plc:refuser", + }, + "did:key:zDnaefkZms3Qkc8zrfsujwKN9dBm9gVPJNpm342aJKkZxUbru", + ); + defer verified.deinit(); + + try std.testing.expectEqualStrings("3mrxrefrevaaa", verified.commit.rev); + try std.testing.expectEqual(@as(usize, 4), verified.records.len); + try std.testing.expectEqualStrings("com.e.p/x", verified.records[0].path); + try std.testing.expectEqualStrings("com.example.post/aaa", verified.records[1].path); + try std.testing.expectEqualStrings("com.example.post/bbb", verified.records[2].path); + try std.testing.expectEqualStrings("com.example.reply/zz", verified.records[3].path); + + var rejected_wrong_space = false; + if (verifyRepoCarFull( + allocator, + car_bytes, + .{ + .space = "at://did:plc:refauthority/space/com.example.forum/other", + .author = "did:plc:refuser", + }, + "did:key:zDnaefkZms3Qkc8zrfsujwKN9dBm9gVPJNpm342aJKkZxUbru", + )) |bad| { + var unexpected = bad; + unexpected.deinit(); + } else |_| { + rejected_wrong_space = true; + } + try std.testing.expect(rejected_wrong_space); +} + test "permissioned commit MAC uses HKDF expand without extract" { var ikm: [32]u8 = undefined; var hash: [32]u8 = undefined; @@ -583,3 +730,38 @@ test "delegation token and space credential round trip" { try std.testing.expectError(error.InvalidJwt, verifySpaceCredential(allocator, delegation_token, public_key_multibase)); try std.testing.expectError(error.InvalidJwt, verifyDelegationToken(allocator, credential_token, public_key_multibase)); } + +test "space credential continuity depends on authority identity and key" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + var authority_key = try zat.Keypair.fromSecretKey(.p256, .{0x31} ** 32); + const authority_did_key = try authority_key.did(allocator); + const credential_token = try createSpaceCredential( + allocator, + std.Options.debug_io, + "did:plc:portableauthority", + "at://did:plc:portableauthority/space/fm.example.group/main", + "holder-dpop-thumbprint", + "https://reader.example", + &authority_key, + ); + + // A credential names the stable authority DID and space coordinate, not + // the authority's current service endpoint. Moving #atproto_space_host + // therefore does not invalidate an already-issued credential. + const verified = try verifySpaceCredential( + allocator, + credential_token, + authority_did_key["did:key:".len..], + ); + try std.testing.expectEqualStrings("did:plc:portableauthority", verified.authority_did); + + var replacement_key = try zat.Keypair.fromSecretKey(.p256, .{0x32} ** 32); + const replacement_did_key = try replacement_key.did(allocator); + try std.testing.expectError( + error.SignatureVerificationFailed, + verifySpaceCredential(allocator, credential_token, replacement_did_key["did:key:".len..]), + ); +} diff --git a/src/internal/testdata/space-reference-repo.car.base64 b/src/internal/testdata/space-reference-repo.car.base64 new file mode 100644 index 0000000..e24f94d --- /dev/null +++ b/src/internal/testdata/space-reference-repo.car.base64 @@ -0,0 +1 @@ +Y6Jlcm9vdHOC2CpYJQABcRIg1cLvHIW5ZMRA4YbpSZ1aGRr8yy9QZeQceYIxK16PYB3YKlglAAFxEiBaDlrIyhFPEmkJ5IAtLkRinEK5TXLtG+1eKTbQ0MjX+2d2ZXJzaW9uAfUBAXESINXC7xyFuWTEQOGG6UmdWhka/MsvUGXkHHmCMStej2AdpmNpa21YIPtosBWWNHl+GFv6BTx7cfO4OF0xkioqddpu/nCkVEpZY21hY1ggt9P0w/TDFZRfPdrcC+pAIezf+QV0aPz+Mm4XjYqLRJ5jcmV2bTNtcnhyZWZyZXZhYWFjc2lnWEB/wEUvp03KKw1AruqmPQFYMa2IF14k4cWnvIsiviveexGhK/ns/WNFRGzVrIbeRHr+pFD3rPYmdd2Ux2kvwB7pY3ZlcgFkaGFzaFggQoafFA61ADR853irO/0VruFsSgae8M+4X5q7fXeLLl+SAgFxEiBaDlrIyhFPEmkJ5IAtLkRinEK5TXLtG+1eKTbQ0MjX+6RpY29tLmUucC942CpYJQABcRIgtuuN0yX4P/B6chCI+Q/4jVwRLYIHK4DduIUPKgU6k8Z0Y29tLmV4YW1wbGUucG9zdC9hYWHYKlglAAFxEiDzVyp/1LR7VaAPsNYEQj5QLMSO1QkG5yjOlDA/I5u+JXRjb20uZXhhbXBsZS5wb3N0L2JiYtgqWCUAAXESIIxjBzeSUGQMcxUc/itSfo4TvsrdOSt9JDEPsVvuG5/YdGNvbS5leGFtcGxlLnJlcGx5L3p62CpYJQABcRIgsOPl66IUQjAfFvu+8+bbWPsRtFeTkH1u+UZcdYaZo803AXESILbrjdMl+D/wenIQiPkP+I1cES2CByuA3biFDyoFOpPGomFuGCplJHR5cGVnY29tLmUucEcBcRIg81cqf9S0e1WgD7DWBEI+UCzEjtUJBucozpQwPyObviWiZHRleHRlZmlyc3RlJHR5cGVwY29tLmV4YW1wbGUucG9zdEgBcRIgjGMHN5JQZAxzFRz+K1J+jhO+yt05K30kMQ+xW+4bn9iiZHRleHRmc2Vjb25kZSR0eXBlcGNvbS5leGFtcGxlLnBvc3ROAXESILDj5euiFEIwHxb7vvPm21j7EbRXk5B9bvlGXHWGmaPNomRkZWVwoWZuZXN0ZWSDAQIDZSR0eXBlcWNvbS5leGFtcGxlLnJlcGx5 diff --git a/src/storage/store.zig b/src/storage/store.zig index 83dbfe2..01133a3 100644 --- a/src/storage/store.zig +++ b/src/storage/store.zig @@ -7909,6 +7909,70 @@ test "blob gc follows permissioned record references" { try std.testing.expect(getBlob(allocator, account.did, cid) == null); } +test "blob visibility follows public and permissioned references independently" { + var arena = std.heap.ArenaAllocator.init(std.testing.allocator); + defer arena.deinit(); + const allocator = arena.allocator(); + + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + var path_buf: [std.fs.max_path_bytes]u8 = undefined; + const path_len = try tmp.dir.realPath(std.testing.io, &path_buf); + blobstore.init(std.Options.debug_io, path_buf[0..path_len]); + + try init(std.Options.debug_io, ":memory:"); + defer close(); + const account = try createAccount(allocator, "blob-access.test", "blob-access@test.com", "password", "did:plc:blobaccess", true); + const cid = try putBlob(allocator, std.Options.debug_io, account, "shared", "text/plain"); + + const space = try createSpace(allocator, .{ + .actor_did = account.did, + .authority_did = account.did, + .space_type = "fm.example.private", + .skey = "self", + .is_authority = true, + .managing_app = null, + .policy = "member-list", + .app_access_json = "{\"type\":\"open\"}", + }); + const json = try std.fmt.allocPrint( + allocator, + "{{\"$type\":\"fm.example.blob\",\"media\":{{\"$type\":\"blob\",\"ref\":{{\"$link\":\"{s}\"}},\"mimeType\":\"text/plain\",\"size\":6}}}}", + .{cid}, + ); + const parsed = try std.json.parseFromSlice(std.json.Value, allocator, json, .{}); + defer parsed.deinit(); + const prepared = try prepareRecordValue(allocator, "fm.example.blob", "private", parsed.value); + + try std.testing.expect(getPublicBlob(allocator, account.did, cid) == null); + try std.testing.expect(!try permissionedBlobReferenced(space.uri, account.did, cid)); + try std.testing.expectEqual(@as(usize, 0), (try collectBlobGarbage(allocator, 60, 10)).deleted); + + _ = try putSpaceRecord(allocator, space.uri, account.did, "fm.example.blob", "private", prepared); + try std.testing.expect(getPublicBlob(allocator, account.did, cid) == null); + try std.testing.expect(try permissionedBlobReferenced(space.uri, account.did, cid)); + try std.testing.expectEqual(@as(usize, 0), (try collectBlobGarbage(allocator, 0, 10)).deleted); + + _ = try create(allocator, account, "fm.example.blob", "public", parsed.value); + try std.testing.expect(getPublicBlob(allocator, account.did, cid) != null); + try std.testing.expect(try permissionedBlobReferenced(space.uri, account.did, cid)); + + try deleteSpaceRecord(allocator, space.uri, account.did, "fm.example.blob", "private"); + try std.testing.expect(getPublicBlob(allocator, account.did, cid) != null); + try std.testing.expect(!try permissionedBlobReferenced(space.uri, account.did, cid)); + try std.testing.expectEqual(@as(usize, 0), (try collectBlobGarbage(allocator, 0, 10)).deleted); + + _ = try putSpaceRecord(allocator, space.uri, account.did, "fm.example.blob", "private", prepared); + _ = try delete(allocator, account, "fm.example.blob", "public"); + try std.testing.expect(getPublicBlob(allocator, account.did, cid) == null); + try std.testing.expect(try permissionedBlobReferenced(space.uri, account.did, cid)); + try std.testing.expectEqual(@as(usize, 0), (try collectBlobGarbage(allocator, 0, 10)).deleted); + + try deleteSpaceRecord(allocator, space.uri, account.did, "fm.example.blob", "private"); + try std.testing.expectEqual(@as(usize, 1), (try collectBlobGarbage(allocator, 0, 10)).deleted); + try std.testing.expect(getBlob(allocator, account.did, cid) == null); +} + test "re-upload refreshes the untethered grace period" { var arena = std.heap.ArenaAllocator.init(std.testing.allocator); defer arena.deinit();