diff --git a/packages/atproto/src/client.ts b/packages/atproto/src/client.ts index e90846b..f3c4e26 100644 --- a/packages/atproto/src/client.ts +++ b/packages/atproto/src/client.ts @@ -766,6 +766,7 @@ export interface ListedRecord { export interface RepoReadTransport { resolvePds(did: string, signal?: AbortSignal): Promise getLatestCommit(did: string, signal?: AbortSignal): Promise + describeRepo?(did: string, signal?: AbortSignal): Promise<{ collections: string[] }> listRecords(input: { did: string collection: string @@ -831,6 +832,21 @@ export class FetchRepoTransport implements RepoReadTransport { return { rev: requiredString(value, 'rev'), commitCid: requiredString(value, 'cid') } } + async describeRepo(did: string, signal?: AbortSignal): Promise<{ collections: string[] }> { + const service = await this.resolvePds(did, signal) + const query = new URLSearchParams({ repo: did }) + const value = await jsonResponse( + await this.fetcher( + new URL(`/xrpc/com.atproto.repo.describeRepo?${query}`, service), + signal ? { signal } : undefined, + ), + ) + if (!Array.isArray(value.collections) || !value.collections.every((entry) => typeof entry === 'string')) { + throw new TypeError('describeRepo response is missing collections') + } + return { collections: value.collections } + } + async listRecords(input: { did: string collection: string diff --git a/packages/atproto/test/client.test.mjs b/packages/atproto/test/client.test.mjs index 38e8d38..b36a5d4 100644 --- a/packages/atproto/test/client.test.mjs +++ b/packages/atproto/test/client.test.mjs @@ -223,6 +223,22 @@ describe('atproto credentials and writes', () => { }) describe('atproto repository reads', () => { + it('describes repository collections defensively', async () => { + const transport = new FetchRepoTransport( + async () => 'https://pds.test', + async () => json({ collections: [COLLECTIONS.space, COLLECTIONS.goal], handle: 'ignored' }), + ) + assert.deepEqual(await transport.describeRepo('did:plc:alice'), { + collections: [COLLECTIONS.space, COLLECTIONS.goal], + }) + + const malformed = new FetchRepoTransport( + async () => 'https://pds.test', + async () => json({ collections: [COLLECTIONS.space, 42] }), + ) + await assert.rejects(malformed.describeRepo('did:plc:alice'), /missing collections/) + }) + it('accepts a getRecord response for the exact requested URI', async () => { const did = 'did:plc:alice' const uri = `at://${did}/${COLLECTIONS.space}/wanted` diff --git a/packages/atproto/test/local-pds.mjs b/packages/atproto/test/local-pds.mjs index e7e2a6c..f0c2d41 100644 --- a/packages/atproto/test/local-pds.mjs +++ b/packages/atproto/test/local-pds.mjs @@ -66,6 +66,11 @@ class LocalPds { if (method === 'com.atproto.sync.getLatestCommit') { return this.json({ rev: this.rev(), cid: this.head() }) } + if (method === 'com.atproto.repo.describeRepo') { + return this.json({ + collections: [...new Set([...this.records.values()].map((record) => record.value.$type))], + }) + } if (method === 'com.atproto.repo.listRecords') { const collection = url.searchParams.get('collection') const offset = Number(url.searchParams.get('cursor') ?? 0) diff --git a/packages/core/src/materializer.ts b/packages/core/src/materializer.ts index 4496fae..15f1841 100644 --- a/packages/core/src/materializer.ts +++ b/packages/core/src/materializer.ts @@ -29,7 +29,7 @@ import { type SpaceRecord, type StrongRef, } from './generated/records.js' -import { claimContractViolation, claimLive } from './claim-lease.js' +import { claimContractViolation, claimDeadline, claimLive } from './claim-lease.js' import { deviceIgnoreReason, deviceViews, @@ -233,6 +233,8 @@ export interface MaterializedIndex { * already written stay blessed: a member vouched for those bytes, and a toggle is not a retraction. */ guestCommentsEnabled: boolean + /** Earliest instant the same records may fold differently because a live claim expires. */ + staleAt?: string edits: EditAnnotation[] ignored: IgnoredRecord[] } @@ -1067,6 +1069,11 @@ export function materialize(store: RecordStore, options: MaterializeOptions): Ma const firstSeenAt = firstSeenByUri.get(claim.uri) return firstSeenAt === undefined ? claim : { ...claim, firstSeenAt } }) + const staleAt = claims + .filter((claim) => claimLive(claim.value, asOf, claim.firstSeenAt)) + .map((claim) => claimDeadline(claim.value, claim.firstSeenAt) ?? claim.value.expiresAt) + .filter((deadline) => Number.isFinite(Date.parse(deadline)) && Date.parse(asOf) < Date.parse(deadline)) + .sort((left, right) => Date.parse(left) - Date.parse(right))[0] const targetUriForRequest = (request: IndexedRecord): string => (request.value.goal ?? request.value.project)!.uri const targetUriForReview = (review: IndexedRecord): string | undefined => @@ -1249,6 +1256,7 @@ export function materialize(store: RecordStore, options: MaterializeOptions): Ma projects: projectViews, images, guestCommentsEnabled: !isPrivate && guestCommentsSetting?.value.enabled === true, + ...(staleAt ? { staleAt } : {}), edits, ignored: [...trust.ignored, ...outOfContract, ...refusedInPrivate, ...unassociated].sort( (a, b) => compareCodePoints(a.uri, b.uri) || compareCodePoints(a.reason, b.reason), diff --git a/packages/core/test/claim-lease.test.mjs b/packages/core/test/claim-lease.test.mjs index d6bfed3..d4e158a 100644 --- a/packages/core/test/claim-lease.test.mjs +++ b/packages/core/test/claim-lease.test.mjs @@ -386,13 +386,15 @@ describe('the fold and an out-of-contract claim', () => { store.put(record.uri === base.claimWinnerUri ? claimVersion : record) } const index = materialize(store, { spaceUri: base.spaceUri, asOf: AS_OF }) - return index.goals.find((view) => view.target.uri === base.goalUri) + return { index, goal: index.goals.find((view) => view.target.uri === base.goalUri) } } - assert.equal(foldWith(seenJustNow).winningClaims[request.uri].uri, base.claimWinnerUri) - assert.notEqual(foldWith(seenLongAgo).winningClaims[request.uri].uri, base.claimWinnerUri) + const fresh = foldWith(seenJustNow) + assert.equal(fresh.goal.winningClaims[request.uri].uri, base.claimWinnerUri) + assert.notEqual(foldWith(seenLongAgo).goal.winningClaims[request.uri].uri, base.claimWinnerUri) + assert.equal(Date.parse(fresh.index.staleAt), Date.parse(claimDeadline(winner.value, seenJustNow.firstSeenAt))) // The stamp rides along on the indexed claim — the one observer-local field the index carries — // so a client can show the deadline the fold actually applied. - assert.equal(foldWith(seenJustNow).winningClaims[request.uri].firstSeenAt, '2026-02-01T00:00:00.000Z') + assert.equal(fresh.goal.winningClaims[request.uri].firstSeenAt, '2026-02-01T00:00:00.000Z') }) it('is invariant under arrival order when every record was seen at the same moment', () => { diff --git a/packages/ingest/src/poller.ts b/packages/ingest/src/poller.ts index d616c94..e420ab2 100644 --- a/packages/ingest/src/poller.ts +++ b/packages/ingest/src/poller.ts @@ -18,6 +18,7 @@ export interface PollResult { export interface RepoPollerOptions { pageSize?: number maxAttempts?: number + collectionConcurrency?: number now?: () => string /** * Which collections to list. Defaults to every Radial collection, which is what a public space @@ -46,6 +47,7 @@ export class RepoPoller { readonly #running = new Map>() readonly pageSize: number readonly maxAttempts: number + readonly collectionConcurrency: number readonly now: () => string readonly collections: readonly string[] @@ -57,6 +59,7 @@ export class RepoPoller { ) { this.pageSize = options.pageSize ?? 100 this.maxAttempts = options.maxAttempts ?? 3 + this.collectionConcurrency = Math.max(1, options.collectionConcurrency ?? 6) this.now = options.now ?? (() => new Date().toISOString()) this.collections = options.collections ?? radialCollections } @@ -89,12 +92,28 @@ export class RepoPoller { } } - const staged: StoredRecord[] = [] - const rejected: PollResult['rejected'] = [] + let collections = this.collections + if (this.transport.describeRepo) { + try { + const described = await this.transport.describeRepo(did, signal) + const held = new Set(described.collections) + collections = this.collections.filter((collection) => held.has(collection)) + } catch { + // describeRepo is an optimization only. Older or incomplete PDSes get the full sweep. + } + } + const stagedByCollection: StoredRecord[][] = collections.map(() => []) + const rejectedByCollection: PollResult['rejected'][] = collections.map(() => []) // One stamp for the whole scan: every record of this repo head arrived in the same pass, and // a per-record clock read would make the fold depend on how long a page took to come back. const observedAt = this.now() - for (const collection of this.collections) { + let nextCollection = 0 + const scanCollection = async (): Promise => { + const index = nextCollection++ + if (index >= collections.length) return + const collection = collections[index]! + const staged = stagedByCollection[index]! + const rejected = rejectedByCollection[index]! let cursor: string | undefined const seen = new Set() do { @@ -139,7 +158,14 @@ export class RepoPoller { if (page.cursor) seen.add(page.cursor) cursor = page.cursor } while (cursor) + await scanCollection() } + await Promise.all( + Array.from({ length: Math.min(this.collectionConcurrency, collections.length) }, () => scanCollection()), + ) + + const staged = stagedByCollection.flat() + const rejected = rejectedByCollection.flat() const end = await this.transport.getLatestCommit(did, signal) if (!sameHead(start, end)) { diff --git a/packages/ingest/src/space.ts b/packages/ingest/src/space.ts index e97f076..b409f12 100644 --- a/packages/ingest/src/space.ts +++ b/packages/ingest/src/space.ts @@ -5,21 +5,32 @@ import { RepoPoller } from './poller.js' export class SpaceIngestor { readonly spaceDid: string readonly knownDids = new Set() + #lastIndex: MaterializedIndex | undefined constructor( readonly spaceUri: string, readonly poller: RepoPoller, readonly records: RecordStore, + readonly now: () => string = () => new Date().toISOString(), ) { this.spaceDid = parseAtUri(spaceUri).did this.knownDids.add(this.spaceDid) } async sync(signal?: AbortSignal): Promise { - await this.poller.pollDid(this.spaceDid, signal) - const polled = new Set([this.spaceDid]) + const initialDids = [...this.knownDids].sort(compareCodePoints) + const initial = await Promise.all(initialDids.map((did) => this.poller.pollDid(did, signal))) + const polled = new Set(initialDids) + const now = this.now() + if ( + this.#lastIndex && + initial.every((result) => !result.changed) && + (!this.#lastIndex.staleAt || Date.parse(now) < Date.parse(this.#lastIndex.staleAt)) + ) { + return this.#lastIndex + } while (true) { - const index = materialize(this.records, { spaceUri: this.spaceUri }) + const index = materialize(this.records, { spaceUri: this.spaceUri, asOf: now }) const pending = index.members .filter((member) => member.active && !polled.has(member.did)) .sort((left, right) => { @@ -27,12 +38,18 @@ export class SpaceIngestor { const rightAdmin = right.role === 'admin' ? 0 : 1 return leftAdmin - rightAdmin || compareCodePoints(left.did, right.did) }) - if (pending.length === 0) return index + if (pending.length === 0) { + this.knownDids.clear() + this.knownDids.add(this.spaceDid) + for (const member of index.members) if (member.active) this.knownDids.add(member.did) + this.#lastIndex = index + return index + } for (const member of pending) { polled.add(member.did) this.knownDids.add(member.did) - await this.poller.pollDid(member.did, signal) } + await Promise.all(pending.map((member) => this.poller.pollDid(member.did, signal))) } } } diff --git a/packages/ingest/test/poller.test.mjs b/packages/ingest/test/poller.test.mjs index 6567251..f640e8e 100644 --- a/packages/ingest/test/poller.test.mjs +++ b/packages/ingest/test/poller.test.mjs @@ -26,6 +26,12 @@ function transport(options = {}) { calls, async resolvePds() { return new URL('https://pds.test') }, async getLatestCommit() { return heads[Math.min(headIndex++, heads.length - 1)] }, + ...(options.describe === false ? {} : { + async describeRepo() { + if (options.describeError) throw new Error('unsupported') + return { collections: options.collections ?? [COLLECTIONS.space] } + }, + }), async listRecords(input) { calls.push(input) if (options.empty) return { records: [] } @@ -38,6 +44,16 @@ function transport(options = {}) { } describe('stable repo polling', () => { + it('prunes absent collections and falls back when describeRepo is unavailable', async () => { + const pruned = transport() + await new RepoPoller(pruned, new MemoryRecordStore()).pollDid(did) + assert.deepEqual([...new Set(pruned.calls.map((call) => call.collection))], [COLLECTIONS.space]) + + const fallback = transport({ describeError: true, empty: true }) + await new RepoPoller(fallback, new MemoryRecordStore()).pollDid(did) + assert.equal(new Set(fallback.calls.map((call) => call.collection)).size, 26) + }) + it('paginates, stores validated records, and skips an unchanged head', async () => { const remote = transport({ paginate: true }) const store = new MemoryRecordStore() diff --git a/packages/ingest/test/space.test.mjs b/packages/ingest/test/space.test.mjs index e625af5..669fc69 100644 --- a/packages/ingest/test/space.test.mjs +++ b/packages/ingest/test/space.test.mjs @@ -119,3 +119,51 @@ it('polls a re-added member and discovers their transitive member', async () => assert.deepEqual(calls, [root, admin, child]) assert.equal(index.members.find((member) => member.did === child).active, true) }) + +it('returns the cached index when every known repository is unchanged', async () => { + const root = 'did:plc:root' + const store = new MemoryRecordStore() + const space = { + did: root, collection: COLLECTIONS.space, rkey: 'space', + uri: `at://${root}/${COLLECTIONS.space}/space`, cid: 'cid-space', rev: '0001', + value: { $type: COLLECTIONS.space, name: 'Space', description: 'test', createdAt: '2026-07-18T00:00:00Z' }, + } + store.put(space) + let changed = true + const poller = { + async pollDid(did) { + const result = { did, head: { rev: '0001', commitCid: 'head' }, changed, records: 0, rejected: [] } + changed = false + return result + }, + } + const ingestor = new SpaceIngestor(space.uri, poller, store) + const first = await ingestor.sync() + const second = await ingestor.sync() + assert.equal(second, first) +}) + +it('refolds unchanged records after the cached claim deadline', async () => { + const root = 'did:plc:root' + const store = new MemoryRecordStore() + const space = { + did: root, collection: COLLECTIONS.space, rkey: 'space', + uri: `at://${root}/${COLLECTIONS.space}/space`, cid: 'cid-space', rev: '0001', + value: { $type: COLLECTIONS.space, name: 'Space', description: 'test', createdAt: '2026-07-18T00:00:00Z' }, + } + store.put(space) + let changed = true + let now = '2026-07-18T00:00:00Z' + const poller = { + async pollDid(did) { + const result = { did, head: { rev: '0001', commitCid: 'head' }, changed, records: 0, rejected: [] } + changed = false + return result + }, + } + const ingestor = new SpaceIngestor(space.uri, poller, store, () => now) + const first = await ingestor.sync() + first.staleAt = '2026-07-18T00:01:00Z' + now = '2026-07-18T00:02:00Z' + assert.notEqual(await ingestor.sync(), first) +}) diff --git a/packages/ui/src/lib/fake-pds.ts b/packages/ui/src/lib/fake-pds.ts index c3bd6b3..0f5a2f6 100644 --- a/packages/ui/src/lib/fake-pds.ts +++ b/packages/ui/src/lib/fake-pds.ts @@ -143,6 +143,14 @@ export class FakePds { return json({ rev: String(head).padStart(13, '0'), cid: `commit-${did}-${head}` }) } + if (method === 'com.atproto.repo.describeRepo') { + const did = url.searchParams.get('repo') ?? '' + record('describeRepo', did) + return json({ + collections: [...new Set((this.repos.get(did) ?? []).map((entry) => entry.collection))], + }) + } + if (method === 'com.atproto.repo.listRecords') { const did = url.searchParams.get('repo') ?? '' const collection = url.searchParams.get('collection') ?? '' diff --git a/packages/ui/src/lib/session.svelte.ts b/packages/ui/src/lib/session.svelte.ts index a966921..1b0dc9e 100644 --- a/packages/ui/src/lib/session.svelte.ts +++ b/packages/ui/src/lib/session.svelte.ts @@ -752,6 +752,13 @@ export async function sync(): Promise { if (current.waiting) return await catchUpWaiting(current, undialable) const index = await current.ingestor.sync() if (current.stopped) return + if (session.space?.index === index) { + const now = new Date().toISOString() + session.space.asOf = now + session.space.syncedAt = now + session.error = '' + return + } await current.identities.resolve(index.members.map((member) => member.did)) if (current.stopped) return publish(index) diff --git a/packages/ui/src/lib/session.test.ts b/packages/ui/src/lib/session.test.ts index c53e45b..9e4f269 100644 --- a/packages/ui/src/lib/session.test.ts +++ b/packages/ui/src/lib/session.test.ts @@ -72,6 +72,14 @@ afterEach(() => { }) describe('opening a live space', () => { + it('prunes the cold collection sweep to collections each repository actually holds', async () => { + const live = harness() + await open(live) + const members = session.space?.index.members.filter((member) => member.active).length ?? 0 + expect(live.network.counts('describeRepo')).toBe(members) + expect(live.network.counts('listRecords')).toBeLessThan(members * 26) + }) + it('publishes and remembers the latest distinct space', async () => { const uriB = `${fixture.spaceUri}-b` const spaceA = harness() diff --git a/packages/ui/src/lib/write.test.ts b/packages/ui/src/lib/write.test.ts index 81863bd..ab1b169 100644 --- a/packages/ui/src/lib/write.test.ts +++ b/packages/ui/src/lib/write.test.ts @@ -1325,10 +1325,12 @@ describe('adding an image to a body', () => { image.markdown, ]) await point(harness, fixture.spaceUri) - // Everything except the one observer-local stamp on the index (`firstSeenAt`, which is when THIS - // reader saw a claim and is different on every cold read by construction). + // Everything except observer-local claim timing: firstSeenAt is when THIS reader saw the claim, + // and staleAt is the deadline derived from that stamp. const settled = (): string => - JSON.stringify(space().index, (key, value: unknown) => (key === 'firstSeenAt' ? undefined : value)) + JSON.stringify(space().index, (key, value: unknown) => + key === 'firstSeenAt' || key === 'staleAt' ? undefined : value, + ) const forwards = settled() // Same records, the image record ingested last. An image is presentation infrastructure and