diff --git a/bench/README.md b/bench/README.md index 4574735..aaa92a2 100644 --- a/bench/README.md +++ b/bench/README.md @@ -84,7 +84,7 @@ blob, then measures these paths as distinct units of work: - `listSpaces` - `createRecord` into a space writer repo - `getRecord` by `(space, repo, collection, rkey)` -- `listRecords`, limit 50, index-only response shape +- `listRecords`, limit 50, with values included by default - `getBlob` storage readback - `listRepoOps` catch-up reads diff --git a/bench/main.zig b/bench/main.zig index 6a2e9ff..fa2f804 100644 --- a/bench/main.zig +++ b/bench/main.zig @@ -737,8 +737,9 @@ fn benchSpaceListRecords(allocator: std.mem.Allocator, account: zds.auth.tokens. const start = nowNs(); for (0..iterations) |_| { _ = arena.reset(.retain_capacity); - const found = try zds.storage.store.listSpaceRecords(arena.allocator(), space, account.did, space_collection, null, false, 50); + const found = try zds.storage.store.listSpaceRecords(arena.allocator(), space, account.did, space_collection, null, false, 50, true); if (records > 0 and found.len == 0) return error.MissingRecords; + if (records > 0 and found[0].value_json == null) return error.MissingRecord; } return .{ .name = "space listRecords", .ops = iterations, .elapsed_ns = nowNs() - start }; } diff --git a/docs/permissioned-data.md b/docs/permissioned-data.md index 04a7bc1..2cfd5eb 100644 --- a/docs/permissioned-data.md +++ b/docs/permissioned-data.md @@ -124,6 +124,11 @@ explicit space-scoped tables instead of the public repo tables: Permissioned records and blobs are not public repo records. They must not be squeezed into `records`, `repo_blocks`, `commits`, or `seq_events`. +`com.atproto.space.listRecords` returns current record values by default, in +line with the permissioned-data proposal. Consumers that only need the +collection, rkey, and CID listing can pass `excludeValues=true` to avoid +materializing record JSON. + ## Zat first Before implementing local primitives, check Zat's current public API. ZDS uses diff --git a/src/atproto/space.zig b/src/atproto/space.zig index bd0df06..efbd9b6 100644 --- a/src/atproto/space.zig +++ b/src/atproto/space.zig @@ -555,6 +555,11 @@ fn listRecords(request: *http_api.Request) !void { std.mem.eql(u8, value, "true") else false; + var exclude_values_buf: [8]u8 = undefined; + const exclude_values = if (http_api.queryParam(request.url.raw, "excludeValues", &exclude_values_buf)) |value| + std.mem.eql(u8, value, "true") + else + false; const records = try store.listSpaceRecords( allocator, @@ -564,6 +569,7 @@ fn listRecords(request: *http_api.Request) !void { maybe_cursor, reverse, http_api.queryLimit(request.url.raw, 50), + !exclude_values, ); var out: std.Io.Writer.Allocating = .init(allocator); defer out.deinit(); @@ -571,9 +577,13 @@ fn listRecords(request: *http_api.Request) !void { for (records, 0..) |record, idx| { if (idx != 0) try out.writer.writeByte(','); try out.writer.print( - "{{\"collection\":{f},\"rkey\":{f},\"cid\":{f}}}", + "{{\"collection\":{f},\"rkey\":{f},\"cid\":{f}", .{ std.json.fmt(record.collection, .{}), std.json.fmt(record.rkey, .{}), std.json.fmt(record.cid, .{}) }, ); + if (record.value_json) |value_json| { + try out.writer.print(",\"value\":{s}", .{value_json}); + } + try out.writer.writeByte('}'); } try out.writer.writeByte(']'); if (records.len > 0) { diff --git a/src/http/router.zig b/src/http/router.zig index 793f10b..a86ffd6 100644 --- a/src/http/router.zig +++ b/src/http/router.zig @@ -224,7 +224,7 @@ pub const endpoints = [_]Endpoint{ .{ .route = .permissioned_data, .method = "POST", .path = "/xrpc/com.atproto.space.deleteRecord", .group = "space", .auth = "experimental bearer", .summary = "Delete a record inside a permissioned data space.", .body = &.{ "space", "repo", "collection", "rkey" }, .notes = permissioned_data_note }, .{ .route = .permissioned_data, .method = "POST", .path = "/xrpc/com.atproto.space.applyWrites", .group = "space", .auth = "experimental bearer", .summary = "Apply a batch of writes inside a permissioned data space.", .body = &.{ "space", "repo", "validate", "writes" }, .notes = permissioned_data_note }, .{ .route = .permissioned_data, .method = "GET", .path = "/xrpc/com.atproto.space.getRecord", .group = "space", .auth = "experimental bearer or space credential", .summary = "Read a record from a permissioned data space.", .params = &.{ "space", "repo", "collection", "rkey" }, .notes = permissioned_data_note }, - .{ .route = .permissioned_data, .method = "GET", .path = "/xrpc/com.atproto.space.listRecords", .group = "space", .auth = "experimental bearer or space credential", .summary = "List record keys and CIDs in a permissioned data space.", .params = &.{ "space", "repo", "collection", "limit", "cursor", "reverse" }, .notes = permissioned_data_note }, + .{ .route = .permissioned_data, .method = "GET", .path = "/xrpc/com.atproto.space.listRecords", .group = "space", .auth = "experimental bearer or space credential", .summary = "List records in a permissioned data space. Values are included unless excludeValues=true.", .params = &.{ "space", "repo", "collection", "limit", "cursor", "reverse", "excludeValues" }, .notes = permissioned_data_note }, .{ .route = .permissioned_data, .method = "GET", .path = "/xrpc/com.atproto.space.getBlob", .group = "space", .auth = "experimental bearer or space credential", .summary = "Read a blob referenced from a permissioned data record.", .params = &.{ "space", "repo", "cid" }, .notes = permissioned_data_note }, .{ .route = .permissioned_data, .method = "GET", .path = "/xrpc/com.atproto.space.getRepoState", .group = "space", .auth = "experimental bearer or space credential", .summary = "Read current record-set commitment state for a writer repo in a space.", .params = &.{ "space", "repo" }, .notes = permissioned_data_note }, .{ .route = .permissioned_data, .method = "GET", .path = "/xrpc/com.atproto.space.listRepoOps", .group = "space", .auth = "experimental bearer or space credential", .summary = "Read incremental record operations for a writer repo in a space.", .params = &.{ "space", "repo", "since", "limit" }, .notes = permissioned_data_note }, diff --git a/src/internal/account.zig b/src/internal/account.zig index 2117a2c..a76ee16 100644 --- a/src/internal/account.zig +++ b/src/internal/account.zig @@ -160,7 +160,7 @@ const html = \\function renderRecords(records,append=false){const box=$('records');if(!append)box.innerHTML='';if(!records.length&&!append)box.innerHTML='
No records for that repo.
';for(const rec of records){const btn=document.createElement('button');btn.type='button';btn.className='item';btn.innerHTML=''+esc(rec.collection)+' / '+esc(rec.rkey)+''+esc(rec.cid)+'';btn.onclick=()=>openRecord(rec);box.appendChild(btn)}} \\async function loadRecords(append=false){if(!selectedSpace)throw new Error('choose a space first');const repo=$('repo').value.trim(),collection=$('collection').value.trim();if(!repo)throw new Error('enter a writer repo DID');setStatus('loading records...');const params={space:selectedSpace,repo,limit:'50'};if(collection)params.collection=collection;if(append&&cursor)params.cursor=cursor;const out=await xrpc('/xrpc/com.atproto.space.listRecords',params);cursor=out.cursor||null;$('next-records').disabled=!cursor;renderRecords(out.records||[],append);setStatus('records loaded','ok')} \\function blobChips(value){const out=[],seen=new Set();function walk(v){if(!v||typeof v!=='object')return;if(v.ref&&v.ref.$link&&!seen.has(v.ref.$link)){seen.add(v.ref.$link);out.push(v.ref.$link)}for(const child of Array.isArray(v)?v:Object.values(v))walk(child)}walk(value);return out.map(cid=>'blob '+esc(cid)+'').join(' ')} - \\async function openRecord(rec){const detail=$('record-detail');detail.className='';detail.innerHTML='

loading record...

';const out=await xrpc('/xrpc/com.atproto.space.getRecord',{space:selectedSpace,repo:$('repo').value.trim(),collection:rec.collection,rkey:rec.rkey});detail.innerHTML='
'+esc(rec.collection)+' / '+esc(rec.rkey)+''+esc(out.cid)+'
'+blobChips(out.value)+'
'+esc(JSON.stringify(out.value,null,2))+'
';$('copy-json').onclick=()=>navigator.clipboard.writeText(JSON.stringify(out.value,null,2))} + \\async function openRecord(rec){const detail=$('record-detail');detail.className='';detail.innerHTML='

loading record...

';const out=rec.value?rec:await xrpc('/xrpc/com.atproto.space.getRecord',{space:selectedSpace,repo:$('repo').value.trim(),collection:rec.collection,rkey:rec.rkey});detail.innerHTML='
'+esc(rec.collection)+' / '+esc(rec.rkey)+''+esc(out.cid)+'
'+blobChips(out.value)+'
'+esc(JSON.stringify(out.value,null,2))+'
';$('copy-json').onclick=()=>navigator.clipboard.writeText(JSON.stringify(out.value,null,2))} \\async function loadAccountData(){setStatus('loading account...');const [session,passkeys,passwords,sessions,spaceOut]=await Promise.all([xrpc('/xrpc/com.atproto.server.getSession'),xrpc('/xrpc/com.atproto.server.listPasskeys'),xrpc('/xrpc/com.atproto.server.listAppPasswords'),xrpc('/xrpc/dev.zat.account.listSessions',{active:'false',limit:'200'}),xrpc('/xrpc/com.atproto.space.listSpaces',{limit:'100'}).catch(()=>({spaces:[]}))]);account=session;spaces=spaceOut.spaces||[];updateHeader();showPasskeys(passkeys.passkeys||[]);showAppPasswords(passwords.passwords||[]);showSessions(sessions);renderSpaces();quickChecks(sessions);setStatus('signed in','ok')} \\loginForm.addEventListener('submit',async(e)=>{e.preventDefault();try{setStatus('signing in...');const f=e.currentTarget;const session=await fail(await fetch('/xrpc/com.atproto.server.createSession',{method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({identifier:f.identifier.value,password:f.password.value})}),'sign in failed');accessJwt=session.accessJwt;account=session;loginForm.classList.add('off');resident.classList.add('on');showView();await loadAccountData()}catch(err){setStatus(err.message||String(err),'error')}}) \\$('add-passkey').onclick=async()=>{try{setStatus('opening browser passkey prompt...');const friendlyName=$('friendly').value;const start=await xrpc('/xrpc/com.atproto.server.startPasskeyRegistration',{}, {method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({friendlyName})});const credential=await navigator.credentials.create(prepCreate(start.options));await xrpc('/xrpc/com.atproto.server.finishPasskeyRegistration',{}, {method:'POST',headers:{'content-type':'application/json'},body:JSON.stringify({friendlyName,credential:serialize(credential)})});setStatus('passkey saved','ok');await loadAccountData()}catch(err){setStatus((err.name?err.name+': ':'')+(err.message||String(err)),'error')}} diff --git a/src/storage/store.zig b/src/storage/store.zig index cd31f1c..ce055b2 100644 --- a/src/storage/store.zig +++ b/src/storage/store.zig @@ -333,6 +333,7 @@ pub const SpaceRecordRef = struct { collection: []const u8, rkey: []const u8, cid: []const u8, + value_json: ?[]const u8 = null, }; pub const SpaceRecordOplogEntry = struct { @@ -3511,6 +3512,7 @@ pub fn listSpaceRecords( maybe_cursor: ?[]const u8, reverse: bool, limit: usize, + include_values: bool, ) ![]SpaceRecordRef { db_mutex.lockUncancelable(store_io); defer db_mutex.unlock(store_io); @@ -3527,8 +3529,8 @@ pub fn listSpaceRecords( var query = std.Io.Writer.Allocating.init(allocator); defer query.deinit(); try query.writer.print( - "SELECT collection, rkey, cid FROM permissioned_space_records WHERE space = ? AND repo_did = ?", - .{}, + "SELECT collection, rkey, cid, {s} FROM permissioned_space_records WHERE space = ? AND repo_did = ?", + .{if (include_values) "value_json" else "NULL"}, ); if (maybe_collection != null) try query.writer.writeAll(" AND collection = ?"); if (maybe_cursor != null and cursor_collection.len > 0 and cursor_rkey.len > 0) { @@ -3553,6 +3555,7 @@ pub fn listSpaceRecords( .collection = try allocator.dupe(u8, row.text(0)), .rkey = try allocator.dupe(u8, row.text(1)), .cid = try allocator.dupe(u8, row.text(2)), + .value_json = if (row.nullableText(3)) |value| try allocator.dupe(u8, value) else null, }); } if (rows.err) |err| return err; @@ -6308,10 +6311,16 @@ test "permissioned spaces store self-owned records outside public repo" { try std.testing.expectEqualStrings(stored.cid, found.cid); try std.testing.expect(std.mem.indexOf(u8, found.value_json, "secret track") != null); - const listed = try listSpaceRecords(allocator, space.uri, account.did, "fm.plyr.track", null, false, 50); + const listed = try listSpaceRecords(allocator, space.uri, account.did, "fm.plyr.track", null, false, 50, true); try std.testing.expectEqual(@as(usize, 1), listed.len); try std.testing.expectEqualStrings("fm.plyr.track", listed[0].collection); try std.testing.expectEqualStrings("track-one", listed[0].rkey); + try std.testing.expect(listed[0].value_json != null); + try std.testing.expect(std.mem.indexOf(u8, listed[0].value_json.?, "secret track") != null); + + const listed_refs = try listSpaceRecords(allocator, space.uri, account.did, "fm.plyr.track", null, false, 50, false); + try std.testing.expectEqual(@as(usize, 1), listed_refs.len); + try std.testing.expect(listed_refs[0].value_json == null); const updated_parsed = try std.json.parseFromSlice( std.json.Value, diff --git a/tools/smoke-permissioned.sh b/tools/smoke-permissioned.sh index 3d7b482..c85cdac 100755 --- a/tools/smoke-permissioned.sh +++ b/tools/smoke-permissioned.sh @@ -111,8 +111,14 @@ test "$public_blob_status" = "404" space_records=$(curl -fsS -H "authorization: Bearer $token" "$base/xrpc/com.atproto.space.listRecords?space=$encoded_space&repo=did:plc:permissionsmoke&collection=fm.plyr.track&limit=1") printf '%s' "$space_records" | grep -q '"collection":"fm.plyr.track"' +printf '%s' "$space_records" | grep -q '"value":' +printf '%s' "$space_records" | grep -q '"title":"private smoke"' printf '%s' "$space_records" | grep -q '"cursor":"fm.plyr.track/track-one"' +space_record_refs=$(curl -fsS -H "authorization: Bearer $token" "$base/xrpc/com.atproto.space.listRecords?space=$encoded_space&repo=did:plc:permissionsmoke&collection=fm.plyr.track&limit=1&excludeValues=true") +printf '%s' "$space_record_refs" | grep -q '"collection":"fm.plyr.track"' +! printf '%s' "$space_record_refs" | grep -q '"value":' + space_blob_status=$(curl -sS -o /tmp/zds-space-blob-range.bin -w '%{http_code}' \ -H "authorization: Bearer $token" \ -H 'range: bytes=0-2' \