// The Node-only half of @radial/core, imported as `@radial/core/node`. // // The package's main entry is the fold itself — records in, index out — and must stay isomorphic so // a browser tab can run the identical `materialize()` the daemon runs. The SQLite-backed store is // one *implementation* of `RecordStore`, not part of that contract, so it lives here where a // bundler following the main entry never sees `node:sqlite`. A browser supplies its own // implementation (IndexedDB) against the same interface. import { chmod, lstat, mkdir, readFile, rename, writeFile } from 'node:fs/promises' import { dirname, join } from 'node:path' import { DatabaseSync } from 'node:sqlite' import type { RadialRecord } from './generated/records.js' import { deriveDeviceKeyId, sortDeviceKeys, type DeviceKeyRecord, type DeviceKeyStore, } from './private/device-keys.js' import { exportPublicKey, exportSigningKey, generateDeviceKey, importSigningKey, } from './private/envelope.js' import type { EnvelopeSigner } from './private/writer.js' import { collapse, select, type EditAnnotation, type RecordStore, type RepoSnapshot, type StoredRecord, } from './store.js' import { DEFAULT_BLOB_LIMITS, verifyBlobBytes, type BlobLimits, type PrivateBlobStore, } from './private/blobs.js' import type { SpacePin } from './private/ticket.js' import type { EnvelopeStore } from './private/envelope-store.js' import { envelopeKey, projectEnvelope, type PrivateEnvelope } from './private/envelope.js' import type { WantEntry } from './private/admission.js' import { wantKey } from './private/admission.js' import type { WantList } from './private/quarantine.js' export class SqliteRecordStore implements RecordStore { readonly #database: DatabaseSync #revision = 0 #dataVersion: number get revision(): number { // `revision` is a cache-invalidation token, not a count of this object's method calls. Several // public spaces in one daemon deliberately open the same records.db through separate handles; // whichever space polls a changed repo first writes the shared corpus. SQLite's data_version // changes only when ANOTHER connection commits, which is exactly the event the old in-memory // counter could not see. Collapse any number of unseen commits into one bump: callers need only // know that their cached fold is stale. const dataVersion = this.#readDataVersion() if (dataVersion !== this.#dataVersion) { this.#dataVersion = dataVersion this.#revision += 1 } return this.#revision } /** * This store's connection, for the one other class that must write in the SAME transaction as a * record put: `SqliteEnvelopeStore`. Two `DatabaseSync` handles on one file cannot share a * transaction — SQLite refuses the second writer with `database is locked` — so the envelope store * borrows this one rather than opening its own. */ get database(): DatabaseSync { return this.#database } constructor(path = ':memory:') { this.#database = new DatabaseSync(path) this.#database.exec(` CREATE TABLE IF NOT EXISTS record_versions ( did TEXT NOT NULL, collection TEXT NOT NULL, rkey TEXT NOT NULL, uri TEXT NOT NULL, cid TEXT NOT NULL, rev TEXT NOT NULL, value_json TEXT NOT NULL, first_seen_at TEXT, -- Whether this version was ever seen on the public path. Provenance only: it must NEVER -- gate the signers recorded below. It used to, and that was a way to un-sign a record by -- putting a copy of it in a PDS -- see unionSigners() in store.ts, which this column is -- kept in step with. Retained rather than dropped so an existing database needs no -- migration, and because "this version was also in a repo" is a fact worth keeping. public_copy INTEGER NOT NULL DEFAULT 0, PRIMARY KEY (did, collection, rkey, cid) ) STRICT; -- Which devices signed one stored version, on the private path. Kept as a table for schema -- stability and to preserve imported archives written by earlier experimental builds. CREATE TABLE IF NOT EXISTS record_signers ( did TEXT NOT NULL, collection TEXT NOT NULL, rkey TEXT NOT NULL, cid TEXT NOT NULL, device_key_id TEXT NOT NULL, PRIMARY KEY (did, collection, rkey, cid, device_key_id) ) STRICT; CREATE TABLE IF NOT EXISTS repo_snapshots ( did TEXT PRIMARY KEY, rev TEXT NOT NULL, commit_cid TEXT NOT NULL, scanner_generation INTEGER NOT NULL DEFAULT 0 ) STRICT; `) // Nullable and added in place: an existing database keeps working with no migration script, and // a NULL simply means "this version predates the column" — the fold falls back to the claim's // own `expiresAt` for those, and the next poll of a changed repo fills them in. const versionColumns = this.#database .prepare('PRAGMA table_info(record_versions)') .all() as Array> if (!versionColumns.some((column) => column.name === 'first_seen_at')) { this.#database.exec('ALTER TABLE record_versions ADD COLUMN first_seen_at TEXT') } if (!versionColumns.some((column) => column.name === 'public_copy')) { // Defaulting to 0 is right for an existing database: every row in one predates private mode, // so none of them has signers, so the flag changes nothing until an envelope arrives. this.#database.exec( 'ALTER TABLE record_versions ADD COLUMN public_copy INTEGER NOT NULL DEFAULT 0', ) } const snapshotColumns = this.#database .prepare('PRAGMA table_info(repo_snapshots)') .all() as Array> if (!snapshotColumns.some((column) => column.name === 'scanner_generation')) { this.#database.exec( 'ALTER TABLE repo_snapshots ADD COLUMN scanner_generation INTEGER NOT NULL DEFAULT 0', ) } this.#dataVersion = this.#readDataVersion() } #readDataVersion(): number { const row = this.#database.prepare('PRAGMA data_version').get() as Record const value = row.data_version if (typeof value !== 'number') throw new Error('SQLite did not report a numeric data_version') return value } put(record: StoredRecord): void { this.#database .prepare(` INSERT INTO record_versions (did, collection, rkey, uri, cid, rev, value_json, first_seen_at, public_copy) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (did, collection, rkey, cid) DO UPDATE SET rev = min(record_versions.rev, excluded.rev), public_copy = max(record_versions.public_copy, excluded.public_copy), -- Both observer-local stamps keep the EARLIEST sighting across provenances. coalesce -- first because SQLite's min() is NULL-propagating, and a row stored before the column -- existed must be able to LEARN a first-sighting rather than stay unknown forever. first_seen_at = min( coalesce(record_versions.first_seen_at, excluded.first_seen_at), coalesce(excluded.first_seen_at, record_versions.first_seen_at) ) `) .run( record.did, record.collection, record.rkey, record.uri, record.cid, record.rev, JSON.stringify(record.value), record.firstSeenAt ?? null, record.deviceKeyIds ? 0 : 1, ) for (const deviceKeyId of record.deviceKeyIds ?? []) { this.#database .prepare(` INSERT INTO record_signers (did, collection, rkey, cid, device_key_id) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING `) .run(record.did, record.collection, record.rkey, record.cid, deviceKeyId) } // SQLite may have inserted a version or reconciled earlier observer-local provenance. Bumping // conservatively on a duplicate only costs a fold; failing to bump could serve a stale one. this.#revision += 1 } #versions(): StoredRecord[] { const rows = this.#database .prepare( 'SELECT did, collection, rkey, uri, cid, rev, value_json, first_seen_at, public_copy FROM record_versions', ) .all() as Array> const signers = new Map() for (const row of this.#database .prepare('SELECT did, collection, rkey, cid, device_key_id FROM record_signers') .all() as Array>) { const key = `${row.did} ${row.collection} ${row.rkey} ${row.cid}` signers.set(key, [...(signers.get(key) ?? []), row.device_key_id as string]) } return rows.map((row) => { // Signers accumulate and are never taken away, exactly as `mergeVersion` unions them: the // `public_copy` flag is provenance and deliberately does not appear here. const signed = signers.get(`${row.did} ${row.collection} ${row.rkey} ${row.cid}`) return { did: row.did as string, collection: row.collection as string, rkey: row.rkey as string, uri: row.uri as string, cid: row.cid as string, rev: row.rev as string, ...(row.first_seen_at !== null ? { firstSeenAt: row.first_seen_at as string } : {}), ...(signed ? { deviceKeyIds: [...signed].sort() } : {}), value: JSON.parse(row.value_json as string) as RadialRecord, } }) } records(): StoredRecord[] { return collapse(this.#versions()).records } get(did: string, collection: string, rkey: string): StoredRecord | undefined { const versions = this.#versions().filter( (record) => record.did === did && record.collection === collection && record.rkey === rkey, ) return versions.length > 0 ? select(versions).record : undefined } getSnapshot(did: string): RepoSnapshot | undefined { const row = this.#database .prepare( 'SELECT did, rev, commit_cid, scanner_generation FROM repo_snapshots WHERE did = ?', ) .get(did) as Record | undefined return row ? { did: row.did as string, rev: row.rev as string, commitCid: row.commit_cid as string, scannerGeneration: row.scanner_generation as number, } : undefined } setSnapshot(snapshot: RepoSnapshot): void { this.#database .prepare(` INSERT INTO repo_snapshots (did, rev, commit_cid, scanner_generation) VALUES (?, ?, ?, ?) ON CONFLICT (did) DO UPDATE SET rev = excluded.rev, commit_cid = excluded.commit_cid, scanner_generation = excluded.scanner_generation `) .run(snapshot.did, snapshot.rev, snapshot.commitCid, snapshot.scannerGeneration) } edits(): EditAnnotation[] { return collapse(this.#versions()).edits } close(): void { this.#database.close() } } /** * The private path's durable half: envelopes and their projections, in ONE database file. * * Both writes in `put` happen inside one transaction, because a projection without its envelope is a * record this replica believes but cannot prove, and an envelope without its projection is one it * can prove but never reads. Crashing between them would leave exactly one of those, and there is no * PDS to reconcile against — `envelope-store.ts` states the same rule for the memory implementation. * * That guarantee is why this takes a `SqliteRecordStore` and writes through ITS connection instead * of opening a second one. The earlier shape — two `DatabaseSync` handles, `BEGIN`/`COMMIT` on only * one of them — delivered neither half of the promise: pointed at two files it was two independent * transactions, and pointed at one file the record write inside the envelope transaction failed with * `database is locked`, so every `put()` threw. One connection is the only shape that makes the * sentence above true. */ export class SqliteEnvelopeStore implements EnvelopeStore { readonly #database: DatabaseSync readonly #records: SqliteRecordStore constructor(records: SqliteRecordStore) { this.#records = records this.#database = records.database this.#database.exec(` CREATE TABLE IF NOT EXISTS envelopes ( did TEXT NOT NULL, collection TEXT NOT NULL, rkey TEXT NOT NULL, record_cid TEXT NOT NULL, device_key_id TEXT NOT NULL, envelope_json TEXT NOT NULL, PRIMARY KEY (did, collection, rkey, record_cid, device_key_id) ) STRICT; `) } put(envelope: PrivateEnvelope, firstSeenAt?: string): void { this.#database.exec('BEGIN') try { this.#database .prepare(` INSERT INTO envelopes (did, collection, rkey, record_cid, device_key_id, envelope_json) VALUES (?, ?, ?, ?, ?, ?) ON CONFLICT DO NOTHING `) .run( envelope.did, envelope.collection, envelope.rkey, envelope.recordCid, envelope.deviceKeyId, JSON.stringify(envelope), ) this.#records.put(projectEnvelope(envelope, firstSeenAt)) this.#database.exec('COMMIT') } catch (error) { this.#database.exec('ROLLBACK') throw error } } has(envelope: PrivateEnvelope): boolean { const key = envelopeKey(envelope) return this.all().some((held) => envelopeKey(held) === key) } all(): PrivateEnvelope[] { return ( this.#database .prepare( 'SELECT envelope_json FROM envelopes ORDER BY did, collection, rkey, record_cid, device_key_id', ) .all() as Array> ).map((row) => JSON.parse(row.envelope_json as string) as PrivateEnvelope) } forRecord(did: string, collection: string, rkey: string): PrivateEnvelope[] { return ( this.#database .prepare( 'SELECT envelope_json FROM envelopes WHERE did = ? AND collection = ? AND rkey = ? ORDER BY record_cid, device_key_id', ) .all(did, collection, rkey) as Array> ).map((row) => JSON.parse(row.envelope_json as string) as PrivateEnvelope) } /** * A no-op: the record store owns the connection and closing it here would shut the store's tables * too. Present so a caller can treat every store it holds the same way. */ close(): void {} } /** * A want list that survives a restart — which is the only kind worth having, since the restart most * often follows the burst that caused the eviction (`quarantine.ts`). */ export class SqliteWantList implements WantList { readonly #database: DatabaseSync constructor(path = ':memory:') { this.#database = new DatabaseSync(path) this.#database.exec(` CREATE TABLE IF NOT EXISTS wants ( key TEXT PRIMARY KEY, did TEXT NOT NULL, collection TEXT NOT NULL, rkey TEXT NOT NULL, record_cid TEXT NOT NULL ) STRICT; `) } add(want: WantEntry): void { this.#database .prepare( 'INSERT INTO wants (key, did, collection, rkey, record_cid) VALUES (?, ?, ?, ?, ?) ON CONFLICT DO NOTHING', ) .run(wantKey(want), want.did, want.collection, want.rkey, want.recordCid) } clear(want: WantEntry): void { this.#database.prepare('DELETE FROM wants WHERE key = ?').run(wantKey(want)) } all(): WantEntry[] { return ( this.#database .prepare('SELECT did, collection, rkey, record_cid FROM wants ORDER BY key') .all() as Array> ).map((row) => ({ did: row.did as string, collection: row.collection as string, rkey: row.rkey as string, recordCid: row.record_cid as string, })) } close(): void { this.#database.close() } } /** * Blobs on disk, in their own file. * * Deliberately NOT in the record store's transaction, unlike envelopes. An envelope and its * projection are two halves of one fact and either both land or neither does; a blob and the record * that names it are independent — content addressing means the bytes are the same bytes whenever * they arrive, and a replica routinely holds the record for a while before any peer answers with the * bytes. Coupling them would only mean a picture could not be back-filled after the fact. */ export class SqliteBlobStore implements PrivateBlobStore { readonly #database: DatabaseSync readonly #limits: BlobLimits constructor(path = ':memory:', limits: BlobLimits = DEFAULT_BLOB_LIMITS) { this.#limits = limits this.#database = new DatabaseSync(path) this.#database.exec(` CREATE TABLE IF NOT EXISTS blobs ( cid TEXT PRIMARY KEY, bytes BLOB NOT NULL ) STRICT; `) } async put(bytes: Uint8Array, expectedCid?: string): Promise { const cid = await verifyBlobBytes(bytes, expectedCid, this.#limits) this.#database .prepare('INSERT INTO blobs (cid, bytes) VALUES (?, ?) ON CONFLICT DO NOTHING') .run(cid, bytes) return cid } get(cid: string): Uint8Array | undefined { const row = this.#database.prepare('SELECT bytes FROM blobs WHERE cid = ?').get(cid) as | Record | undefined return row ? new Uint8Array(row.bytes as Uint8Array) : undefined } has(cid: string): boolean { return this.#database.prepare('SELECT 1 FROM blobs WHERE cid = ?').get(cid) !== undefined } list(): string[] { return ( this.#database.prepare('SELECT cid FROM blobs ORDER BY cid').all() as Array< Record > ).map((row) => row.cid as string) } close(): void { this.#database.close() } } /** * A device's signing keys in a file under the instance data directory (design §18). * * It sits beside `sessions.json` and is handled with the same care, for a stronger reason: a session * token expires and can be re-minted from a password, while this file is the only copy of a key * whose signatures are the sole proof that this DID wrote a private space's records. Losing it costs * a rotation; leaking it compromises the DID in every private space where it remains a member. * * So: 0600, refused if it is a symlink or anything but a regular file, written through a temporary * file and renamed so a crash mid-write cannot leave a half-parsed key store, and serialized per * path so two commands in one process cannot interleave a read-modify-write. */ export class FileDeviceKeyStore implements DeviceKeyStore { readonly path: string #queue: Promise = Promise.resolve() constructor(directory: string) { this.path = join(directory, 'devices.json') } async #read(): Promise { try { const info = await lstat(this.path) if (info.isSymbolicLink() || !info.isFile()) { throw new Error(`Device key path is not a regular non-symlink file: ${this.path}`) } if ((info.mode & 0o077) !== 0) await chmod(this.path, 0o600) const value = JSON.parse(await readFile(this.path, 'utf8')) as unknown if ( typeof value !== 'object' || value === null || (value as { version?: unknown }).version !== 1 || !Array.isArray((value as { keys?: unknown }).keys) ) { throw new TypeError(`Invalid Radial device key file: ${this.path}`) } return (value as { keys: DeviceKeyRecord[] }).keys } catch (error) { if (typeof error === 'object' && error !== null && (error as { code?: string }).code === 'ENOENT') { return [] } throw error } } async #write(keys: DeviceKeyRecord[]): Promise { const directory = dirname(this.path) await mkdir(directory, { recursive: true, mode: 0o700 }) await chmod(directory, 0o700) const temporary = `${this.path}.tmp` await writeFile(temporary, `${JSON.stringify({ version: 1, keys }, null, 2)}\n`, { mode: 0o600 }) await chmod(temporary, 0o600) await rename(temporary, this.path) } /** Read-modify-write, serialized: two `device rotate`s in one process must not lose one another. */ async #update(change: (keys: DeviceKeyRecord[]) => Promise | T): Promise { const run = this.#queue.then(async () => { const keys = await this.#read() const result = await change(keys) await this.#write(keys) return result }) this.#queue = run.catch(() => {}) return run } async list(did?: string): Promise { const keys = await this.#read() return sortDeviceKeys(did ? keys.filter((key) => key.did === did) : keys) } async active(did: string): Promise { return (await this.list(did)).find((key) => !key.retiredAt) } async create( did: string, options: { kind?: 'node' | 'browser'; now?: () => string } = {}, ): Promise { const kind = options.kind ?? 'node' // Extractable, because a daemon has to survive a restart with no human present. A browser // generates a non-extractable key instead and keeps the handle — see `device-keys.ts`. const pair = await generateDeviceKey(true) const publicKey = await exportPublicKey(pair.publicKey) const record: DeviceKeyRecord = { did, deviceKeyId: await deriveDeviceKeyId(kind, publicKey), publicKey, privateKey: await exportSigningKey(pair.privateKey), kind, createdAt: options.now?.() ?? new Date().toISOString(), } return this.#update((keys) => { keys.push(record) return record }) } async retire(did: string, deviceKeyId: string, at?: string): Promise { return this.#update((keys) => { const key = keys.find((entry) => entry.did === did && entry.deviceKeyId === deviceKeyId) // Retirement is fold-neutral, so the key material stays and old signatures remain verifiable. if (key && !key.retiredAt) key.retiredAt = at ?? new Date().toISOString() return key }) } async signer(key: DeviceKeyRecord): Promise { if (!key.privateKey) { throw new Error(`${key.deviceKeyId} holds no key material in ${this.path}`) } return { did: key.did, deviceKeyId: key.deviceKeyId, privateKey: await importSigningKey(key.privateKey), } } } // ── where a private replica lives on a filesystem ────────────────────────────────────────────── // // A private space's records exist in exactly one place per machine, and TWO programs hold that place // open: the `radial` CLI (`sidecar/src/private-space.ts`) and `radiald` (`daemon/src/private-run.ts`). // The layout is therefore defined once, here, beside the device key store it sits next to. Two // definitions of it would not be a duplicate constant — they would be two replicas of one space on // one machine, each holding records the other has never seen, with no PDS to reconcile them. /** What `space.json` records about a private space this machine holds a replica of. */ export interface PrivateSpaceMeta { spaceUri: string /** * The cid of the space record this replica was bootstrapped against — the ticket's pin, kept. * * Kept rather than checked once and discarded, because the record it pins arrives *later*: an * invitee accepts a ticket long before any peer hands them a space record, and a pin that was not * written down would have nothing to compare when one finally shows up. Optional only so a replica * created before this field existed still opens; such a replica simply checks nothing. */ spaceCid?: string /** The founder's DID — the first repo whose device directory is polled, before membership exists. */ founder: string protocol: number /** DIDs expected to be reachable, from the ticket or the member's own `join` bookmark (§18). */ peerHints?: string[] relays?: string[] /** A local-only label. Publishing one in `join` is opt-in and separate. */ label?: string createdAt: string } /** * A private space's directory name: the founder's DID and the space's rkey, with the characters a * filesystem dislikes replaced. * * Readable on purpose. An operator looking for the space they are about to lose the only copy of * should be able to find it with `ls`, and a hashed name would make the backup instructions in §18 * ("your SQLite is the backup") impossible to follow. */ export function spaceDirectoryName(spaceUri: string): string { const match = /^at:\/\/([^/]+)\/[^/]+\/([^/]+)$/.exec(spaceUri) if (!match) throw new TypeError(`Not a space AT URI: ${spaceUri}`) return `${(match[1] as string).replace(/[^A-Za-z0-9._-]/g, '_')}~${(match[2] as string).replace(/[^A-Za-z0-9._-]/g, '_')}` } export const privateRoot = (dataDir: string): string => join(dataDir, 'private') export const spaceDirectory = (dataDir: string, spaceUri: string): string => join(privateRoot(dataDir), spaceDirectoryName(spaceUri)) const metaPath = (directory: string): string => join(directory, 'space.json') export async function readSpaceMeta(directory: string): Promise { try { const value = JSON.parse(await readFile(metaPath(directory), 'utf8')) as PrivateSpaceMeta return typeof value?.spaceUri === 'string' ? value : undefined } catch (error) { if ((error as { code?: string }).code === 'ENOENT') return undefined throw error } } export async function writeSpaceMeta(directory: string, meta: PrivateSpaceMeta): Promise { await mkdir(directory, { recursive: true, mode: 0o700 }) await writeSecretFile(metaPath(directory), `${JSON.stringify(meta, null, 2)}\n`) } /** * Write a file that holds private-space material: 0600, and complete or absent. * * Used for `space.json` and for backup archives, which are the whole corpus in the clear — as * sensitive as the store they came from and far easier to mislay. The temporary-then-rename is not * ceremony either: an archive interrupted halfway through is a file that looks like a backup and * restores most of a space, and the whole point of the header's envelope count is that nobody should * ever have to find that out by reading it back. */ export async function writeSecretFile(path: string, contents: string): Promise { await mkdir(dirname(path), { recursive: true, mode: 0o700 }) const temporary = `${path}.tmp-${Math.random().toString(36).slice(2)}` await writeFile(temporary, contents, { mode: 0o600 }) await chmod(temporary, 0o600) await rename(temporary, path) await chmod(path, 0o600) } /** The pin this replica's `space.json` carries, when it has one. */ export const spacePinOf = (meta: PrivateSpaceMeta): SpacePin | undefined => meta.spaceCid === undefined ? undefined : { uri: meta.spaceUri, cid: meta.spaceCid }