diff --git a/apps/group-chat/scripts/generate.ts b/apps/group-chat/scripts/generate.ts index c29853a..e6cf38f 100644 --- a/apps/group-chat/scripts/generate.ts +++ b/apps/group-chat/scripts/generate.ts @@ -11,7 +11,10 @@ const ROOT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..'); // per-collection endpoints. Values are stubs — only the shape matters for codegen. const configForGen = { ...baseConfig, - spaces: { type: 'tools.atmo.chat.space', serviceDid: 'did:web:localhost' }, + spaces: { + authority: { type: 'tools.atmo.chat.space', serviceDid: 'did:web:localhost' }, + recordHost: {} + }, community: { masterKey: new Uint8Array(32), serviceDid: 'did:web:localhost' }, realtime: { ticketSecret: new Uint8Array(32) } }; diff --git a/apps/group-chat/src/lib/contrail/index.ts b/apps/group-chat/src/lib/contrail/index.ts index a75157a..85a6aa3 100644 --- a/apps/group-chat/src/lib/contrail/index.ts +++ b/apps/group-chat/src/lib/contrail/index.ts @@ -43,12 +43,16 @@ function build(env: Env): Bundle { const config: ContrailConfig = { ...baseConfig, spaces: { - type: 'tools.atmo.chat.space', - serviceDid: env.SERVICE_DID, - blobs: { - adapter: blobAdapter, - maxSize: 2 * 1024 * 1024, - accept: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'] + authority: { + type: 'tools.atmo.chat.space', + serviceDid: env.SERVICE_DID + }, + recordHost: { + blobs: { + adapter: blobAdapter, + maxSize: 2 * 1024 * 1024, + accept: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'] + } } }, community: { diff --git a/packages/contrail/src/core/community/router.ts b/packages/contrail/src/core/community/router.ts index 2897ec0..84e86d1 100644 --- a/packages/contrail/src/core/community/router.ts +++ b/packages/contrail/src/core/community/router.ts @@ -45,8 +45,8 @@ export function registerCommunityRoutes( ): void { const cfg = config.community; if (!cfg) return; - if (!config.spaces) { - throw new Error("community module requires spaces to be enabled in config"); + if (!config.spaces?.authority) { + throw new Error("community module requires spaces.authority to be enabled in config"); } const community = @@ -64,8 +64,8 @@ export function registerCommunityRoutes( })(); const NS = `${config.namespace}.community`; - const spaceType = config.spaces.type; - const spaceServiceDid = cfg.serviceDid ?? config.spaces.serviceDid; + const spaceType = config.spaces.authority.type; + const spaceServiceDid = cfg.serviceDid ?? config.spaces.authority.serviceDid; // ========================================================================== // Community lifecycle diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 3f3c676..5ef78e8 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -307,7 +307,7 @@ export async function initSchema( await db.batch(all.map((s) => db.prepare(s))); - if (config.spaces) { + if (config.spaces?.authority || config.spaces?.recordHost) { await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); } diff --git a/packages/contrail/src/core/invite/router.ts b/packages/contrail/src/core/invite/router.ts index 0b9a4fc..440bc16 100644 --- a/packages/contrail/src/core/invite/router.ts +++ b/packages/contrail/src/core/invite/router.ts @@ -78,7 +78,7 @@ export function registerInviteRoutes( community: CommunityAdapter | null, options: InviteRoutesOptions ): void { - if (!config.spaces) return; + if (!config.spaces?.authority) return; const NS = `${config.namespace}.invite`; const auth = options.authMiddleware; diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index f5e6808..8c10854 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -115,14 +115,17 @@ export function createApp( // Shared spaces context — verifier + adapter — reused by both the per-collection // routes (for `?spaceUri=...` dispatch) and the `.space.*` routes. + // Built when an authority is configured (spaces are gated on the authority, + // not the record host — a record-host-only deployment still needs an + // authority somewhere, just possibly external). const spacesDb = options.spacesDb ?? db; let spacesCtx: SpacesContext | null = options.spacesCtx !== undefined ? options.spacesCtx - : config.spaces + : config.spaces?.authority ? { adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), - verifier: buildVerifier(config.spaces), + verifier: buildVerifier(config.spaces.authority), } : null; @@ -179,7 +182,7 @@ export function createApp( ); } - if (config.spaces && spacesCtx) { + if (config.spaces?.authority && spacesCtx) { // Unified invite surface: one `.invite.*` family that dispatches on // space ownership (user-owned → addMember; community-owned → grant). const authMiddleware = diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 4ad723b..85ba814 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -7,16 +7,17 @@ import { type DidDocumentResolver, } from "@atcute/identity-resolver"; import type { Did, Nsid } from "@atcute/lexicons"; -import type { SpacesConfig } from "./types"; +import type { AuthorityConfig } from "./types"; import { readInProcess } from "./in-process"; export { ServiceJwtVerifier }; -/** Build a ServiceJwtVerifier from a SpacesConfig, using the configured - * resolver or a default PLC+Web composite. */ -export function buildVerifier(spaces: SpacesConfig): ServiceJwtVerifier { +/** Build a ServiceJwtVerifier from an AuthorityConfig, using the configured + * resolver or a default PLC+Web composite. The verifier checks that incoming + * JWTs target this authority's serviceDid (aud claim). */ +export function buildVerifier(authority: AuthorityConfig): ServiceJwtVerifier { const resolver = - spaces.resolver ?? + authority.resolver ?? new CompositeDidDocumentResolver({ methods: { plc: new PlcDidDocumentResolver(), @@ -24,7 +25,7 @@ export function buildVerifier(spaces: SpacesConfig): ServiceJwtVerifier { }, }); return new ServiceJwtVerifier({ - serviceDid: spaces.serviceDid as Did, + serviceDid: authority.serviceDid as Did, resolver, }); } diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index e3fa1c7..f3e3fd5 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -8,7 +8,6 @@ import { checkInviteReadGrant, createServiceAuthMiddleware, extractInviteToken, - verifyServiceAuthRequest, } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; @@ -16,22 +15,29 @@ import { resolveEffectiveLevel } from "../community/acl"; import { buildSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, + type AuthorityConfig, + type RecordHostConfig, + type RecordHost, + type SpaceAuthority, type SpaceRow, - type SpacesConfig, type StorageAdapter, } from "./types"; import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; import { create as createCid, toString as cidToString } from "@atcute/cid"; -import type { Did } from "@atcute/lexicons"; export interface SpacesRoutesOptions { - /** Provide a custom middleware (e.g. for tests). If omitted and spaces.resolver is set, a real one is built. */ + /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ authMiddleware?: MiddlewareHandler; /** Storage adapter override. Defaults to HostedAdapter(db). */ adapter?: StorageAdapter; } +/** Umbrella registration: wires both the authority and the record-host + * routes against the same adapter. Today's deployments enable both via + * `config.spaces.authority` and `config.spaces.recordHost`. Either may be + * omitted in future split deployments — phase 5 lifts the assumption that + * one process runs both. */ export function registerSpacesRoutes( app: Hono, db: Database, @@ -42,45 +48,30 @@ export function registerSpacesRoutes( ): void { const spacesConfig = config.spaces; if (!spacesConfig) return; + const authorityConfig = spacesConfig.authority; + if (!authorityConfig) return; const adapter = options.adapter ?? ctx?.adapter ?? new HostedAdapter(db, config); - const verifier = ctx?.verifier ?? buildVerifier(spacesConfig); + const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is - * present so anonymous bearer reads don't 401 before the route handler can - * validate the token. The route handler is responsible for actually checking - * the token (via `authorizeRead`). */ - const readAuth: MiddlewareHandler = async (c, next) => { - if (extractInviteToken(c.req.raw)) { - await next(); - return; - } - return auth(c, next); - }; + registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, community ?? null); - /** Authorize a read request: either a valid service-auth JWT (which also - * identifies the caller for member checks downstream) or a valid read-grant - * invite token bearer (`?inviteToken=...` or - * `Authorization: Bearer atmo-invite:`). */ - async function authorizeRead( - c: Context, - spaceUri: string - ): Promise<{ via: "token" } | { via: "jwt"; sa: ServiceAuth } | Response> { - const rawToken = extractInviteToken(c.req.raw); - if (rawToken) { - const ok = await checkInviteReadGrant(adapter, rawToken, spaceUri, hashInviteToken); - if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); - return { via: "token" }; - } - const sa = c.get("serviceAuth") as ServiceAuth | undefined; - if (sa) return { via: "jwt", sa }; - return c.json( - { error: "AuthRequired", message: "JWT or read-grant invite token required" }, - 401 - ); + if (spacesConfig.recordHost) { + registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth); } +} +/** Register the **space authority** XRPC surface — space lifecycle, member + * list, app policy, whoami. Does NOT touch records or blobs. */ +export function registerAuthorityRoutes( + app: Hono, + authority: SpaceAuthority, + authorityConfig: AuthorityConfig, + config: ContrailConfig, + auth: MiddlewareHandler, + community: import("../community/adapter").CommunityAdapter | null +): void { /** Space endpoints are emitted per-deployment under the configured namespace; * the deployment owns and publishes its own lexicons. The library ships * templates at `lexicon-templates/spaces/*` that the generator instantiates @@ -89,7 +80,8 @@ export function registerSpacesRoutes( const SPACE = `${config.namespace}.space`; const SPACE_EXT = `${config.namespace}.spaceExt`; - // Read endpoints + // ---- Read endpoints ---- + app.get(`/xrpc/${SPACE}.listSpaces`, auth, async (c) => { const sa = getAuth(c); const scope = c.req.query("scope") ?? "member"; // "member" | "owner" @@ -98,14 +90,14 @@ export function registerSpacesRoutes( const cursor = c.req.query("cursor") ?? undefined; const limit = c.req.query("limit") ? Number(c.req.query("limit")) : undefined; - const opts: Parameters[0] = { type, cursor, limit }; + const opts: Parameters[0] = { type, cursor, limit }; if (scope === "owner") opts.ownerDid = sa.issuer; else { opts.memberDid = sa.issuer; if (owner) opts.ownerDid = owner; // narrow to spaces owned by this DID } - const result = await adapter.listSpaces(opts); + const result = await authority.listSpaces(opts); return c.json({ spaces: result.spaces.map((s) => publicSpaceView(s, s.ownerDid === sa.issuer)), cursor: result.cursor, @@ -116,25 +108,36 @@ export function registerSpacesRoutes( const sa = getAuth(c); const spaceUri = c.req.query("spaceUri"); if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); const isOwner = space.ownerDid === sa.issuer; - const member = isOwner ? null : await adapter.getMember(spaceUri, sa.issuer); + const member = isOwner ? null : await authority.getMember(spaceUri, sa.issuer); if (!isOwner && !member) { return c.json({ error: "Forbidden", reason: "not-member" }, 403); } - const members = await adapter.listMembers(spaceUri); + const members = await authority.listMembers(spaceUri); return c.json({ members }); }); + /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is + * present so anonymous bearer reads don't 401 before the route handler can + * validate the token. */ + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return auth(c, next); + }; + app.get(`/xrpc/${SPACE}.getSpace`, readAuth, async (c) => { const uri = c.req.query("uri"); if (!uri) return c.json({ error: "InvalidRequest", message: "uri required" }, 400); - const space = await adapter.getSpace(uri); + const space = await authority.getSpace(uri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, uri); + const authz = await authorizeRead(c, authority, uri); if (authz instanceof Response) return authz; if (authz.via === "token") { @@ -144,28 +147,173 @@ export function registerSpacesRoutes( const sa = authz.sa; const isOwner = sa.issuer === space.ownerDid; - const member = isOwner ? null : await adapter.getMember(uri, sa.issuer); + const member = isOwner ? null : await authority.getMember(uri, sa.issuer); if (!isOwner && !member) { return c.json({ error: "Forbidden", reason: "not-member" }, 403); } return c.json({ space: publicSpaceView(space, isOwner) }); }); + // ---- Space management (owner-gated) ---- + + app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => ({}))) as { + type?: string; + key?: string; + appPolicy?: SpaceRow["appPolicy"]; + appPolicyRef?: string; + }; + + const type = body.type ?? authorityConfig.type; + const key = body.key ?? nextTid(); + const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); + + const existing = await authority.getSpace(uri); + if (existing) return c.json({ error: "AlreadyExists", uri }, 409); + + const space = await authority.createSpace({ + uri, + ownerDid: sa.issuer, + type, + key, + serviceDid: authorityConfig.serviceDid, + appPolicyRef: body.appPolicyRef ?? null, + appPolicy: body.appPolicy ?? authorityConfig.defaultAppPolicy ?? null, + }); + // Owner is implicit; we still write a row so membership queries are uniform. + await authority.addMember(uri, sa.issuer, sa.issuer); + + return c.json({ space: publicSpaceView(space, true) }); + }); + + app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + await authority.addMember(body.spaceUri, body.did, sa.issuer); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + if (body.did === space.ownerDid) { + return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); + } + await authority.removeMember(body.spaceUri, body.did); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid === sa.issuer) { + return c.json( + { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, + 400 + ); + } + await authority.removeMember(body.spaceUri, sa.issuer); + return c.json({ ok: true }); + }); + + // Unified whoami — `.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, + // accessLevel? }. `accessLevel` is present only when the target space is + // community-owned; for user-owned spaces membership is binary. + app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + + // Community-owned space: resolve through the access-level ladder. The + // reconciler keeps spaces_members in sync, so isMember derives from the + // effective level directly. + const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; + if (isCommunity) { + const level = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); + return c.json({ + isOwner, + isMember: isOwner || !!level, + accessLevel: level, + }); + } + + // User-owned space: binary membership. + if (isOwner) return c.json({ isOwner: true, isMember: true }); + const member = await authority.getMember(spaceUri, sa.issuer); + return c.json({ isOwner: false, isMember: !!member }); + }); +} + +/** Register the **record host** XRPC surface — record + blob CRUD. Today + * consults the authority for membership / app-policy checks at write time; + * phase 3 adds a credential verifier that replaces those calls in + * split-deployment configurations. */ +export function registerRecordHostRoutes( + app: Hono, + recordHost: RecordHost, + authority: SpaceAuthority, + recordHostConfig: RecordHostConfig, + config: ContrailConfig, + auth: MiddlewareHandler +): void { + const SPACE = `${config.namespace}.space`; + + /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is + * present so anonymous bearer reads don't 401 before the route handler can + * validate the token. */ + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return auth(c, next); + }; + app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { const spaceUri = c.req.query("spaceUri"); const collection = c.req.query("collection"); if (!spaceUri || !collection) { return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, spaceUri); + const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", space, @@ -178,7 +326,7 @@ export function registerSpacesRoutes( } } - const list = await adapter.listRecords(spaceUri, collection, { + const list = await recordHost.listRecords(spaceUri, collection, { byUser: c.req.query("byUser") ?? undefined, cursor: c.req.query("cursor") ?? undefined, limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, @@ -194,15 +342,15 @@ export function registerSpacesRoutes( if (!spaceUri || !collection || !author || !rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, spaceUri); + const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", space, @@ -214,7 +362,7 @@ export function registerSpacesRoutes( if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); } - const record = await adapter.getRecord(spaceUri, collection, author, rkey); + const record = await recordHost.getRecord(spaceUri, collection, author, rkey); if (!record) return c.json({ error: "NotFound" }, 404); return c.json({ record }); }); @@ -228,10 +376,10 @@ export function registerSpacesRoutes( if (!body?.spaceUri || !body.collection || !body.record) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); } - const space = await adapter.getSpace(body.spaceUri); + const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(body.spaceUri, sa.issuer); + const member = await authority.getMember(body.spaceUri, sa.issuer); const result = checkAccess({ op: "write", space, @@ -245,10 +393,10 @@ export function registerSpacesRoutes( // uploaded into this space. This mirrors how PDSes require uploadBlob // before putRecord, and prevents forging refs to blobs the caller never // actually claimed. - if (spacesConfig.blobs) { + if (recordHostConfig.blobs) { const cids = collectBlobCids(body.record); for (const cid of cids) { - const meta = await adapter.getBlobMeta(body.spaceUri, cid); + const meta = await recordHost.getBlobMeta(body.spaceUri, cid); if (!meta) { return c.json( { @@ -264,7 +412,7 @@ export function registerSpacesRoutes( const rkey = body.rkey ?? nextTid(); const now = Date.now(); - await adapter.putRecord({ + await recordHost.putRecord({ spaceUri: body.spaceUri, collection: body.collection, authorDid: sa.issuer, @@ -284,10 +432,10 @@ export function registerSpacesRoutes( if (!body?.spaceUri || !body.collection || !body.rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); } - const space = await adapter.getSpace(body.spaceUri); + const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(body.spaceUri, sa.issuer); + const member = await authority.getMember(body.spaceUri, sa.issuer); const result = checkAccess({ op: "delete", space, @@ -298,13 +446,13 @@ export function registerSpacesRoutes( }); if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - await adapter.deleteRecord(body.spaceUri, body.collection, sa.issuer, body.rkey); + await recordHost.deleteRecord(body.spaceUri, body.collection, sa.issuer, body.rkey); return c.json({ ok: true }); }); // Blobs (only registered when a blob adapter is configured) - if (spacesConfig.blobs) { - const blobsCfg = spacesConfig.blobs; + if (recordHostConfig.blobs) { + const blobsCfg = recordHostConfig.blobs; const blobAdapter = blobsCfg.adapter; const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; const accept = blobsCfg.accept; @@ -315,10 +463,10 @@ export function registerSpacesRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "write", space, @@ -360,7 +508,7 @@ export function registerSpacesRoutes( const key = await blobKey(spaceUri, cidString); await blobAdapter.put(key, bytes, { mimeType, size: bytes.byteLength }); - await adapter.putBlobMeta({ + await recordHost.putBlobMeta({ spaceUri, cid: cidString, mimeType, @@ -385,15 +533,15 @@ export function registerSpacesRoutes( if (!spaceUri || !cid) { return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, spaceUri); + const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "read", space, @@ -406,7 +554,7 @@ export function registerSpacesRoutes( } } - const meta = await adapter.getBlobMeta(spaceUri, cid); + const meta = await recordHost.getBlobMeta(spaceUri, cid); if (!meta) return c.json({ error: "NotFound" }, 404); const key = await blobKey(spaceUri, cid); const bytes = await blobAdapter.get(key); @@ -426,10 +574,10 @@ export function registerSpacesRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "read", space, @@ -441,7 +589,7 @@ export function registerSpacesRoutes( return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); } - const result = await adapter.listBlobMeta(spaceUri, { + const result = await recordHost.listBlobMeta(spaceUri, { byUser: c.req.query("byUser") ?? undefined, cursor: c.req.query("cursor") ?? undefined, limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, @@ -449,132 +597,28 @@ export function registerSpacesRoutes( return c.json(result); }); } - - // Space management (owner-gated) - app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => ({}))) as { - type?: string; - key?: string; - appPolicy?: SpaceRow["appPolicy"]; - appPolicyRef?: string; - }; - - const type = body.type ?? spacesConfig.type; - const key = body.key ?? nextTid(); - const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); - - const existing = await adapter.getSpace(uri); - if (existing) return c.json({ error: "AlreadyExists", uri }, 409); - - const space = await adapter.createSpace({ - uri, - ownerDid: sa.issuer, - type, - key, - serviceDid: spacesConfig.serviceDid, - appPolicyRef: body.appPolicyRef ?? null, - appPolicy: body.appPolicy ?? spacesConfig.defaultAppPolicy ?? null, - }); - // Owner is implicit; we still write a row so membership queries are uniform. - await adapter.addMember(uri, sa.issuer, sa.issuer); - - return c.json({ space: publicSpaceView(space, true) }); - }); - - // Invites live under `.invite.*` — see src/core/invite/router.ts. The - // unified surface dispatches on space ownership. - - app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await adapter.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - await adapter.addMember(body.spaceUri, body.did, sa.issuer); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await adapter.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - if (body.did === space.ownerDid) { - return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); - } - await adapter.removeMember(body.spaceUri, body.did); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const space = await adapter.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid === sa.issuer) { - return c.json( - { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, - 400 - ); - } - await adapter.removeMember(body.spaceUri, sa.issuer); - return c.json({ ok: true }); - }); - - // Unified whoami — `.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, - // accessLevel? }. `accessLevel` is present only when the target space is - // community-owned; for user-owned spaces membership is binary. - app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await adapter.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - - // Community-owned space: resolve through the access-level ladder. The - // reconciler keeps spaces_members in sync, so isMember derives from the - // effective level directly. - const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; - if (isCommunity) { - const level = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); - return c.json({ - isOwner, - isMember: isOwner || !!level, - accessLevel: level, - }); - } - - // User-owned space: binary membership. - if (isOwner) return c.json({ isOwner: true, isMember: true }); - const member = await adapter.getMember(spaceUri, sa.issuer); - return c.json({ isOwner: false, isMember: !!member }); - }); } -function buildAuthMiddleware(spaces: SpacesConfig): MiddlewareHandler { - const verifier = buildVerifier(spaces); - return createServiceAuthMiddleware(verifier); +/** Authorize a read request: either a valid service-auth JWT (which also + * identifies the caller for member checks downstream) or a valid read-grant + * invite token bearer. */ +async function authorizeRead( + c: Context, + authority: SpaceAuthority, + spaceUri: string +): Promise<{ via: "token" } | { via: "jwt"; sa: ServiceAuth } | Response> { + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); + if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); + return { via: "token" }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (sa) return { via: "jwt", sa }; + return c.json( + { error: "AuthRequired", message: "JWT or read-grant invite token required" }, + 401 + ); } function getAuth(c: Parameters[0]): ServiceAuth { @@ -583,7 +627,6 @@ function getAuth(c: Parameters[0]): ServiceAuth { return auth; } - function publicSpaceView(space: SpaceRow, forOwner: boolean) { return { uri: space.uri, @@ -596,4 +639,3 @@ function publicSpaceView(space: SpaceRow, forOwner: boolean) { ...(forOwner ? { appPolicy: space.appPolicy } : {}), }; } - diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 397e6a8..fb05d2a 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -25,8 +25,13 @@ export interface SpacesBlobsConfig { export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; -export interface SpacesConfig { - /** NSID that identifies the kind of space this service hosts, e.g. "tools.atmo.event.space". */ +/** Configuration for the **space authority** role: holds the member list, + * signs credentials (later phases), and gates space-management operations. + * In a fully-split deployment, the authority can run in a different process + * (or even a different operator) than the record host. */ +export interface AuthorityConfig { + /** NSID that identifies the kind of space this authority hosts, + * e.g. "tools.atmo.event.space". */ type: string; /** Service DID that service-auth tokens must target (aud claim). */ serviceDid: string; @@ -35,10 +40,29 @@ export interface SpacesConfig { /** DID document resolver for service-auth JWT verification. * Defaults to a composite PLC + did:web resolver if omitted. */ resolver?: DidDocumentResolver; +} + +/** Configuration for the **record host** role: stores per-space records and + * blobs and serves reads. Verifies space credentials (later phases) on + * incoming traffic. */ +export interface RecordHostConfig { /** Blob-upload backend. When omitted, blob XRPCs are not exposed. */ blobs?: SpacesBlobsConfig; } +/** Spaces config — host an authority, a record host, or both. + * Today both run in one process and most deployments will set both; the + * shape is split now so phase 5 can run them independently without churning + * every consumer's config. */ +export interface SpacesConfig { + /** Space-authority config — member list, credentials (later), space + * management. Required for any space to exist. */ + authority?: AuthorityConfig; + /** Record-host config — record + blob storage. Required for records to be + * written/read on this deployment. */ + recordHost?: RecordHostConfig; +} + export interface SpaceRow { uri: string; ownerDid: string; @@ -145,7 +169,13 @@ export interface ListBlobsResult { cursor?: string; } -export interface StorageAdapter { +/** **Space authority** interface — owner of the space's ACL state and + * (eventually) credential issuer. Holds the member list, manages invites, + * governs space lifecycle and app policy. Does NOT touch records or blobs. + * + * In a fully-split deployment this is a separate service; today the + * HostedAdapter implements both this and {@link RecordHost} against one DB. */ +export interface SpaceAuthority { // Space lifecycle createSpace(space: Omit): Promise; getSpace(spaceUri: string): Promise; @@ -167,7 +197,7 @@ export interface StorageAdapter { addedBy: string | null ): Promise; - // Invites + // Invites (token primitive — issued by the authority, scoped to a space) createInvite(input: CreateInviteInput): Promise; listInvites(spaceUri: string, options?: { includeRevoked?: boolean }): Promise; revokeInvite(tokenHash: string): Promise; @@ -176,7 +206,16 @@ export interface StorageAdapter { /** Atomically mark a join-capable invite as used. Returns the row if usable * (kind allows join, not expired/revoked/exhausted), null otherwise. */ redeemInvite(tokenHash: string, now: number): Promise; +} +/** **Record host** interface — stores records and blobs for a space. Trusts + * the authority's ACL: in later phases, validates incoming credentials + * rather than calling out for a member check. + * + * Read-side note: today the record host needs `getSpace` to know the space + * exists at all (and for app-policy checks before permissioned writes). When + * enrollment lands (phase 5), this becomes a local lookup instead. */ +export interface RecordHost { // Records putRecord(record: StoredRecord): Promise; getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise; @@ -194,6 +233,12 @@ export interface StorageAdapter { findOrphanBlobs(spaceUri: string, cutoff: number, limit: number): Promise; } +/** Combined adapter. Used internally where a single object satisfies both + * roles (today's HostedAdapter, the community reconciler, the realtime + * publishing wrapper). Phases 5+ replace consumers of this with two + * injected interfaces. */ +export type StorageAdapter = SpaceAuthority & RecordHost; + export interface AdapterContext { db: Database; } diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index 53239d0..b895141 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -390,9 +390,9 @@ export function validateConfig(config: ContrailConfig): void { } } - if (config.community && !config.spaces) { + if (config.community && !config.spaces?.authority) { throw new Error( - "Invalid config: `community` requires `spaces`. Community-owned spaces reuse the spaces storage adapter." + "Invalid config: `community` requires `spaces.authority`. Community-owned spaces reuse the spaces storage adapter." ); } } diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 1e0c0cf..8d678cb 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -39,6 +39,10 @@ export type { PersistentIngestOptions } from "./core/persistent"; // Spaces export type { SpacesConfig, + AuthorityConfig, + RecordHostConfig, + SpaceAuthority, + RecordHost, AppPolicy, AppPolicyMode, SpaceRow, diff --git a/packages/contrail/tests/community-delegation.test.ts b/packages/contrail/tests/community-delegation.test.ts index d602f86..1514964 100644 --- a/packages/contrail/tests/community-delegation.test.ts +++ b/packages/contrail/tests/community-delegation.test.ts @@ -20,8 +20,11 @@ const CONFIG: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -68,7 +71,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/community-e2e.test.ts b/packages/contrail/tests/community-e2e.test.ts index 79e0a64..db008e6 100644 --- a/packages/contrail/tests/community-e2e.test.ts +++ b/packages/contrail/tests/community-e2e.test.ts @@ -22,8 +22,11 @@ const CONFIG: ContrailConfig = { message: { collection: "app.event.message" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -77,7 +80,7 @@ function fakeAuth(): MiddlewareHandler { if (!did) return c.json({ error: "AuthRequired" }, 401); c.set("serviceAuth", { issuer: did, - audience: CONFIG.spaces!.serviceDid, + audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined, }); await next(); diff --git a/packages/contrail/tests/community-mint.test.ts b/packages/contrail/tests/community-mint.test.ts index 1f1b6a7..c83491f 100644 --- a/packages/contrail/tests/community-mint.test.ts +++ b/packages/contrail/tests/community-mint.test.ts @@ -27,8 +27,11 @@ const CONFIG: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -52,7 +55,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/community-publishing.test.ts b/packages/contrail/tests/community-publishing.test.ts index 5ab42c3..10a3d76 100644 --- a/packages/contrail/tests/community-publishing.test.ts +++ b/packages/contrail/tests/community-publishing.test.ts @@ -22,8 +22,11 @@ const CONFIG: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -82,7 +85,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/invite-unified.test.ts b/packages/contrail/tests/invite-unified.test.ts index a3892ef..b56d81d 100644 --- a/packages/contrail/tests/invite-unified.test.ts +++ b/packages/contrail/tests/invite-unified.test.ts @@ -23,7 +23,10 @@ const MASTER_KEY = new Uint8Array(32).fill(11); const CONFIG: ContrailConfig = { namespace: "test.inv", collections: { message: { collection: "app.event.message" } }, - spaces: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + recordHost: {}, + }, community: { masterKey: MASTER_KEY, fetch: mockFetch, @@ -55,7 +58,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/realtime-e2e.test.ts b/packages/contrail/tests/realtime-e2e.test.ts index 7a3d1ec..59e9338 100644 --- a/packages/contrail/tests/realtime-e2e.test.ts +++ b/packages/contrail/tests/realtime-e2e.test.ts @@ -19,8 +19,11 @@ const CONFIG: ContrailConfig = { namespace: "test.rt", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -61,7 +64,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/spaces-blobs.test.ts b/packages/contrail/tests/spaces-blobs.test.ts index 63af7a5..f4c4dee 100644 --- a/packages/contrail/tests/spaces-blobs.test.ts +++ b/packages/contrail/tests/spaces-blobs.test.ts @@ -21,9 +21,13 @@ function makeConfig(blobs: MemoryBlobAdapter, maxSize = 2 * 1024 * 1024): Contra photo: { collection: "app.event.photo" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", - blobs: { adapter: blobs, maxSize }, + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: { + blobs: { adapter: blobs, maxSize }, + }, }, }; } @@ -51,7 +55,7 @@ async function makeApp( const resolved = resolveConfig(cfg); await initSchema(db, resolved); const app = createApp(db, resolved, { - spaces: { authMiddleware: fakeAuth(cfg.spaces!.serviceDid) }, + spaces: { authMiddleware: fakeAuth(cfg.spaces!.authority!.serviceDid) }, }); return { app, db, config: resolved }; } diff --git a/packages/contrail/tests/spaces-e2e.test.ts b/packages/contrail/tests/spaces-e2e.test.ts index e69c29b..e70f9ec 100644 --- a/packages/contrail/tests/spaces-e2e.test.ts +++ b/packages/contrail/tests/spaces-e2e.test.ts @@ -19,8 +19,11 @@ const CONFIG: ContrailConfig = { ticket: { collection: "app.event.ticket" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, }; @@ -31,7 +34,7 @@ function fakeAuth(): MiddlewareHandler { if (!did) return c.json({ error: "AuthRequired" }, 401); c.set("serviceAuth", { issuer: did, - audience: CONFIG.spaces!.serviceDid, + audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined, clientId: c.req.header("X-Test-App") ?? undefined, }); diff --git a/packages/contrail/tests/spaces-invites.test.ts b/packages/contrail/tests/spaces-invites.test.ts index eb3fc1c..d4e0406 100644 --- a/packages/contrail/tests/spaces-invites.test.ts +++ b/packages/contrail/tests/spaces-invites.test.ts @@ -18,8 +18,11 @@ const CONFIG: ContrailConfig = { message: { collection: "app.event.message" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, }; @@ -29,7 +32,7 @@ function fakeAuth(): MiddlewareHandler { if (!did) return c.json({ error: "AuthRequired" }, 401); c.set("serviceAuth", { issuer: did, - audience: CONFIG.spaces!.serviceDid, + audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined, }); await next(); -- 2.51.2 From 44194b2497ac28a6a514fe0d074648cb813ceb37 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:03:18 +0200 Subject: [PATCH 02/25] phase two --- packages/contrail/src/core/community/index.ts | 2 + .../src/core/community/invite-handler.ts | 155 ++++++++++++++++ .../contrail/src/core/community/reconcile.ts | 11 +- .../contrail/src/core/community/whoami.ts | 26 +++ .../src/core/invite/community-handler.ts | 67 +++++++ packages/contrail/src/core/invite/router.ts | 175 ++++++------------ packages/contrail/src/core/router/index.ts | 44 +++-- packages/contrail/src/core/spaces/router.ts | 45 +++-- 8 files changed, 371 insertions(+), 154 deletions(-) create mode 100644 packages/contrail/src/core/community/invite-handler.ts create mode 100644 packages/contrail/src/core/community/whoami.ts create mode 100644 packages/contrail/src/core/invite/community-handler.ts diff --git a/packages/contrail/src/core/community/index.ts b/packages/contrail/src/core/community/index.ts index 3fbe1ad..fd0cc53 100644 --- a/packages/contrail/src/core/community/index.ts +++ b/packages/contrail/src/core/community/index.ts @@ -21,6 +21,8 @@ export { export { CredentialCipher } from "./credentials"; export { resolveEffectiveLevel, flattenEffectiveMembers, wouldCycle } from "./acl"; export { reconcile } from "./reconcile"; +export { createCommunityInviteHandler } from "./invite-handler"; +export { createCommunityWhoamiExtension } from "./whoami"; export { initCommunitySchema, buildCommunitySchema } from "./schema"; export { resolveIdentity, createPdsSession } from "./pds"; export { diff --git a/packages/contrail/src/core/community/invite-handler.ts b/packages/contrail/src/core/community/invite-handler.ts new file mode 100644 index 0000000..0b41f8a --- /dev/null +++ b/packages/contrail/src/core/community/invite-handler.ts @@ -0,0 +1,155 @@ +/** Implementation of {@link CommunityInviteHandler} for community-grant + * invites. Lives here (not in invite/) so the dependency edge points + * community → invite (downward), not the other way around. */ + +import type { + CommunityInviteHandler, + HandlerResponse, +} from "../invite/community-handler"; +import { mintInviteToken } from "../invite/token"; +import type { CommunityAdapter } from "./adapter"; +import type { SpaceAuthority } from "../spaces/types"; +import { resolveEffectiveLevel } from "./acl"; +import { reconcile } from "./reconcile"; +import type { AccessLevel, CommunityInviteRow } from "./types"; +import { isAccessLevel, rankOf } from "./types"; + +interface PublicInviteView { + tokenHash: string; + spaceUri: string; + accessLevel: AccessLevel; + createdBy: string; + createdAt: number; + expiresAt: number | null; + maxUses: number | null; + usedCount: number; + revokedAt: number | null; + note: string | null; +} + +function toView(row: CommunityInviteRow): PublicInviteView { + return { + tokenHash: row.tokenHash, + spaceUri: row.spaceUri, + accessLevel: row.accessLevel, + createdBy: row.createdBy, + createdAt: row.createdAt, + expiresAt: row.expiresAt, + maxUses: row.maxUses, + usedCount: row.usedCount, + revokedAt: row.revokedAt, + note: row.note, + }; +} + +const ok = (body: Record): HandlerResponse => ({ status: 200, body }); +const err = (status: number, body: Record): HandlerResponse => ({ status, body }); + +export function createCommunityInviteHandler(args: { + community: CommunityAdapter; + /** Space authority — used to look up space metadata after redemption (e.g. + * to return the community DID). */ + authority: SpaceAuthority; +}): CommunityInviteHandler { + const { community, authority } = args; + + return { + async isCommunityOwned(spaceUri) { + const space = await authority.getSpace(spaceUri); + if (!space) return false; + return !!(await community.getCommunity(space.ownerDid)); + }, + + async create(input) { + if (input.kind) { + return err(400, { + error: "InvalidRequest", + reason: "kind-on-community-space", + message: "community spaces take accessLevel, not kind", + }); + } + if (!input.accessLevel || !isAccessLevel(input.accessLevel)) { + return err(400, { error: "InvalidRequest", reason: "accessLevel-required" }); + } + const callerLevel = await resolveEffectiveLevel(community, input.spaceUri, input.callerDid); + if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { + return err(403, { error: "Forbidden", reason: "manager-required" }); + } + if (rankOf(input.accessLevel) > rankOf(callerLevel)) { + return err(403, { error: "Forbidden", reason: "cannot-grant-higher-than-self" }); + } + const { token, tokenHash } = await mintInviteToken(); + const row = await community.createInvite({ + spaceUri: input.spaceUri, + tokenHash, + accessLevel: input.accessLevel, + createdBy: input.callerDid, + expiresAt: input.expiresAt, + maxUses: input.maxUses, + note: input.note, + }); + return ok({ token, invite: toView(row) }); + }, + + async list(input) { + const callerLevel = await resolveEffectiveLevel(community, input.spaceUri, input.callerDid); + if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { + return err(403, { error: "Forbidden", reason: "manager-required" }); + } + const rows = await community.listInvites(input.spaceUri, { + includeRevoked: input.includeRevoked, + }); + return ok({ invites: rows.map(toView) }); + }, + + async revoke(input) { + const level = await resolveEffectiveLevel(community, input.spaceUri, input.callerDid); + const managerOrHigher = !!level && rankOf(level) >= rankOf("manager"); + if (!managerOrHigher) { + const crow = await community.getInvite(input.tokenHash); + if (!crow || crow.createdBy !== input.callerDid) { + return err(403, { error: "Forbidden", reason: "creator-or-manager-required" }); + } + } + const revoked = await community.revokeInvite(input.tokenHash); + return ok({ ok: revoked }); + }, + + async tryRevokeByToken(input) { + const crow = await community.getInvite(input.tokenHash); + if (!crow) return null; + let allowed = crow.createdBy === input.callerDid; + if (!allowed) { + const level = await resolveEffectiveLevel(community, crow.spaceUri, input.callerDid); + allowed = !!level && rankOf(level) >= rankOf("manager"); + } + if (!allowed) { + return err(403, { error: "Forbidden", reason: "creator-or-manager-required" }); + } + const revoked = await community.revokeInvite(input.tokenHash); + return ok({ ok: revoked }); + }, + + async tryRedeem(input) { + const cinvite = await community.redeemInvite(input.tokenHash, input.now); + if (!cinvite) return null; + const space = await authority.getSpace(cinvite.spaceUri); + if (!space) return err(404, { error: "NotFound", reason: "space-not-found" }); + // The token itself is the authorization: creator (manager+) pre-signed + // "anyone with this token gets level X". Grant directly, attributing + // to the creator so audit trails make sense. + await community.grant({ + spaceUri: cinvite.spaceUri, + subjectDid: input.callerDid, + accessLevel: cinvite.accessLevel, + grantedBy: cinvite.createdBy, + }); + await reconcile(community, authority, cinvite.spaceUri, cinvite.createdBy); + return ok({ + spaceUri: cinvite.spaceUri, + accessLevel: cinvite.accessLevel, + communityDid: space.ownerDid, + }); + }, + }; +} diff --git a/packages/contrail/src/core/community/reconcile.ts b/packages/contrail/src/core/community/reconcile.ts index b9d7e35..5fde1d3 100644 --- a/packages/contrail/src/core/community/reconcile.ts +++ b/packages/contrail/src/core/community/reconcile.ts @@ -1,13 +1,18 @@ -import type { StorageAdapter as SpacesAdapter } from "../spaces/types"; +import type { SpaceAuthority } from "../spaces/types"; import type { CommunityAdapter } from "./adapter"; import { flattenEffectiveMembers } from "./acl"; /** Reconcile `spaces_members` for `spaceUri` to match the flattened effective * member set derived from `community_access_levels`. Also re-reconciles any - * spaces that delegate to this one (reverse-graph). */ + * spaces that delegate to this one (reverse-graph). + * + * Takes a {@link SpaceAuthority} (not a full `StorageAdapter`) — the + * reconciler only needs member-level operations, so depending on the + * narrower interface keeps the dependency direction clean and proves we + * could swap in a non-Contrail authority. */ export async function reconcile( community: CommunityAdapter, - spaces: SpacesAdapter, + spaces: SpaceAuthority, spaceUri: string, byDid: string, opts: { depth?: number; maxReverseDepth?: number } = {} diff --git a/packages/contrail/src/core/community/whoami.ts b/packages/contrail/src/core/community/whoami.ts new file mode 100644 index 0000000..4a75a34 --- /dev/null +++ b/packages/contrail/src/core/community/whoami.ts @@ -0,0 +1,26 @@ +/** Whoami extension that adds `accessLevel` (and the corrected `isMember`) + * for community-owned spaces. Returns null for non-community spaces so the + * spaces module's default binary-membership logic runs. */ + +import type { WhoamiExtension } from "../spaces/router"; +import type { CommunityAdapter } from "./adapter"; +import { resolveEffectiveLevel } from "./acl"; + +export function createCommunityWhoamiExtension(args: { + community: CommunityAdapter; +}): WhoamiExtension { + const { community } = args; + return async ({ spaceUri, callerDid, isOwner, ownerDid }) => { + const isCommunity = !!(await community.getCommunity(ownerDid)); + if (!isCommunity) return null; + + // The reconciler keeps spaces_members in sync with the access-level + // ladder, so isMember derives from the effective level directly. + const level = await resolveEffectiveLevel(community, spaceUri, callerDid); + return { + isOwner, + isMember: isOwner || !!level, + accessLevel: level, + }; + }; +} diff --git a/packages/contrail/src/core/invite/community-handler.ts b/packages/contrail/src/core/invite/community-handler.ts new file mode 100644 index 0000000..1043525 --- /dev/null +++ b/packages/contrail/src/core/invite/community-handler.ts @@ -0,0 +1,67 @@ +/** Pluggable handler for community-grant invites within the unified invite + * surface. The invite router calls into this when the target space is + * community-owned, or "tries" it on the redeem / revoke-without-spaceUri + * paths. Community module provides the impl; invite/router doesn't import + * from community at all. + * + * Each method returns a `HandlerResponse`: a `{status, body}` envelope that + * the router relays as JSON, or `null` (only on the "try" methods) meaning + * "not applicable, fall through to the user-owned path." */ + +export type HandlerResponse = { + status: number; + body: Record; +}; + +export interface CommunityInviteHandler { + /** True iff this space is owned by a community (vs. a regular user DID). + * Used by the invite router to choose the dispatch path on + * create / list / revoke-with-spaceUri. */ + isCommunityOwned(spaceUri: string): Promise; + + /** Create a community-grant invite. Caller is validated upstream for + * having a JWT; this method handles the access-level checks. */ + create(input: { + spaceUri: string; + callerDid: string; + /** Raw caller-supplied access level — implementation validates. */ + accessLevel?: string; + /** Caller-supplied `kind` field — community spaces don't accept this; the + * handler returns an InvalidRequest if set. */ + kind?: string; + expiresAt: number | null; + maxUses: number | null; + note: string | null; + }): Promise; + + /** List invites for a community-owned space. */ + list(input: { + spaceUri: string; + callerDid: string; + includeRevoked: boolean; + }): Promise; + + /** Revoke a known community-owned invite (caller already passed spaceUri + * and the router classified it as community-owned). */ + revoke(input: { + spaceUri: string; + tokenHash: string; + callerDid: string; + }): Promise; + + /** Revoke without a spaceUri — try to find the invite in the community + * table; return null if not a community invite (router falls through). */ + tryRevokeByToken(input: { + tokenHash: string; + callerDid: string; + }): Promise; + + /** Try to redeem a token as a community invite. Returns null if the token + * is not a community invite, in which case the router falls through to + * the user-owned redeem path. */ + tryRedeem(input: { + tokenHash: string; + callerDid: string; + now: number; + }): Promise; +} diff --git a/packages/contrail/src/core/invite/router.ts b/packages/contrail/src/core/invite/router.ts index 440bc16..941af9b 100644 --- a/packages/contrail/src/core/invite/router.ts +++ b/packages/contrail/src/core/invite/router.ts @@ -2,36 +2,31 @@ * user-owned and community-owned spaces. Dispatches on space ownership. * * - User-owned space → `kind` in create, `addMember` on redeem, owner-only. - * - Community-owned → `accessLevel` in create, `grant` on redeem, - * manager+ with "cannot grant higher than self". + * - Community-owned → routed to a {@link CommunityInviteHandler} provided + * by the community module (or null when community is + * not configured). * * Storage stays separate (`spaces_invites` vs `community_invites` tables) — * schemas differ enough that unifying them would be net-negative. The token - * primitive and HTTP dance are shared. */ + * primitive and HTTP dance are shared. invite/router has zero imports from + * community/ — coupling is via the {@link CommunityInviteHandler} interface. */ import type { Context, Hono, MiddlewareHandler } from "hono"; import type { ContrailConfig } from "../types"; import type { ServiceAuth } from "../spaces/auth"; -import type { StorageAdapter } from "../spaces/types"; +import type { SpaceAuthority } from "../spaces/types"; import type { InviteKind, InviteRow } from "../spaces/types"; -import type { CommunityAdapter } from "../community/adapter"; -import type { CommunityInviteRow, AccessLevel } from "../community/types"; -import { isAccessLevel, rankOf } from "../community/types"; import { hashInviteToken, mintInviteToken } from "./token"; -import { resolveEffectiveLevel } from "../community/acl"; -import { reconcile } from "../community/reconcile"; +import type { CommunityInviteHandler, HandlerResponse } from "./community-handler"; export interface InviteRoutesOptions { authMiddleware: MiddlewareHandler; } -/** Shape returned to clients — `kind` (user-owned space) or `accessLevel` - * (community-owned space) is set, never both. */ interface PublicInviteView { tokenHash: string; spaceUri: string; kind?: InviteKind; - accessLevel?: AccessLevel; createdBy: string; createdAt: number; expiresAt: number | null; @@ -56,26 +51,11 @@ function toSpacesView(row: InviteRow): PublicInviteView { }; } -function toCommunityView(row: CommunityInviteRow): PublicInviteView { - return { - tokenHash: row.tokenHash, - spaceUri: row.spaceUri, - accessLevel: row.accessLevel, - createdBy: row.createdBy, - createdAt: row.createdAt, - expiresAt: row.expiresAt, - maxUses: row.maxUses, - usedCount: row.usedCount, - revokedAt: row.revokedAt, - note: row.note, - }; -} - export function registerInviteRoutes( app: Hono, config: ContrailConfig, - spaces: StorageAdapter, - community: CommunityAdapter | null, + authority: SpaceAuthority, + community: CommunityInviteHandler | null, options: InviteRoutesOptions ): void { if (!config.spaces?.authority) return; @@ -86,9 +66,9 @@ export function registerInviteRoutes( /** Resolve whether a space is community-owned. Returns null if the space * doesn't exist. */ const classifySpace = async (spaceUri: string) => { - const space = await spaces.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return null; - const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; + const isCommunity = community ? await community.isCommunityOwned(spaceUri) : false; return { space, isCommunity }; }; @@ -120,35 +100,15 @@ export function registerInviteRoutes( if (isCommunity) { if (!community) return c.json({ error: "InvalidState" }, 500); - if (body.kind) { - return c.json( - { error: "InvalidRequest", reason: "kind-on-community-space", message: "community spaces take accessLevel, not kind" }, - 400 - ); - } - if (!body.accessLevel || !isAccessLevel(body.accessLevel)) { - return c.json({ error: "InvalidRequest", reason: "accessLevel-required" }, 400); - } - // Caller must have manager+ on the target space and cannot create an - // invite that confers a higher level than their own. - const callerLevel = await resolveEffectiveLevel(community, body.spaceUri, sa.issuer); - if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { - return c.json({ error: "Forbidden", reason: "manager-required" }, 403); - } - if (rankOf(body.accessLevel) > rankOf(callerLevel)) { - return c.json({ error: "Forbidden", reason: "cannot-grant-higher-than-self" }, 403); - } - const { token, tokenHash } = await mintInviteToken(); - const row = await community.createInvite({ + return relay(c, await community.create({ spaceUri: body.spaceUri, - tokenHash, + callerDid: sa.issuer, accessLevel: body.accessLevel, - createdBy: sa.issuer, + kind: body.kind, expiresAt: body.expiresAt ?? null, maxUses: body.maxUses ?? null, note: body.note ?? null, - }); - return c.json({ token, invite: toCommunityView(row) }); + })); } // User-owned space. @@ -166,7 +126,7 @@ export function registerInviteRoutes( return c.json({ error: "InvalidRequest", message: "kind must be 'join', 'read', or 'read-join'" }, 400); } const { token, tokenHash } = await mintInviteToken(); - const invite = await spaces.createInvite({ + const invite = await authority.createInvite({ spaceUri: body.spaceUri, tokenHash, kind, @@ -189,18 +149,17 @@ export function registerInviteRoutes( const { space, isCommunity } = classified; if (isCommunity) { - const callerLevel = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); - if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { - return c.json({ error: "Forbidden", reason: "manager-required" }, 403); - } - const rows = await community!.listInvites(spaceUri, { includeRevoked }); - return c.json({ invites: rows.map(toCommunityView) }); + return relay(c, await community!.list({ + spaceUri, + callerDid: sa.issuer, + includeRevoked, + })); } if (space.ownerDid !== sa.issuer) { return c.json({ error: "Forbidden", reason: "not-owner" }, 403); } - const rows = await spaces.listInvites(spaceUri, { includeRevoked }); + const rows = await authority.listInvites(spaceUri, { includeRevoked }); return c.json({ invites: rows.map(toSpacesView) }); }); @@ -213,54 +172,39 @@ export function registerInviteRoutes( return c.json({ error: "InvalidRequest", message: "tokenHash required" }, 400); } - // When the caller passes spaceUri we do an auth check up front so the - // response doesn't leak token existence. Community revokers may also be - // the invite creator (even without manager+), which is resolved after. if (body.spaceUri) { const classified = await classifySpace(body.spaceUri); if (!classified) return c.json({ error: "NotFound" }, 404); if (classified.isCommunity) { - const level = await resolveEffectiveLevel(community!, body.spaceUri, sa.issuer); - const managerOrHigher = !!level && rankOf(level) >= rankOf("manager"); - if (!managerOrHigher) { - const crow = await community!.getInvite(body.tokenHash); - if (!crow || crow.createdBy !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "creator-or-manager-required" }, 403); - } - } - const ok = await community!.revokeInvite(body.tokenHash); - return c.json({ ok }); + return relay(c, await community!.revoke({ + spaceUri: body.spaceUri, + tokenHash: body.tokenHash, + callerDid: sa.issuer, + })); } if (classified.space.ownerDid !== sa.issuer) { return c.json({ error: "Forbidden", reason: "not-owner" }, 403); } - const ok = await spaces.revokeInvite(body.tokenHash); + const ok = await authority.revokeInvite(body.tokenHash); return c.json({ ok }); } - // No spaceUri provided — infer from the invite row. + // No spaceUri — try the community handler first (it returns null if the + // token isn't a community invite), then fall back to the user-owned path. if (community) { - const crow = await community.getInvite(body.tokenHash); - if (crow) { - let allowed = crow.createdBy === sa.issuer; - if (!allowed) { - const level = await resolveEffectiveLevel(community, crow.spaceUri, sa.issuer); - allowed = !!level && rankOf(level) >= rankOf("manager"); - } - if (!allowed) { - return c.json({ error: "Forbidden", reason: "creator-or-manager-required" }, 403); - } - const ok = await community.revokeInvite(body.tokenHash); - return c.json({ ok }); - } + const r = await community.tryRevokeByToken({ + tokenHash: body.tokenHash, + callerDid: sa.issuer, + }); + if (r) return relay(c, r); } - const srow = await spaces.getInvite(body.tokenHash); + const srow = await authority.getInvite(body.tokenHash); if (!srow) return c.json({ error: "NotFound" }, 404); - const space = await spaces.getSpace(srow.spaceUri); + const space = await authority.getSpace(srow.spaceUri); if (space && space.ownerDid !== sa.issuer) { return c.json({ error: "Forbidden", reason: "not-owner" }, 403); } - const ok = await spaces.revokeInvite(body.tokenHash); + const ok = await authority.revokeInvite(body.tokenHash); return c.json({ ok }); }); @@ -273,43 +217,32 @@ export function registerInviteRoutes( const tokenHash = await hashInviteToken(body.token); const now = Date.now(); - // Try community first (it's atomic — returns null if not consumable). + // Try community first (atomic — null if not a community invite). if (community) { - const cinvite = await community.redeemInvite(tokenHash, now); - if (cinvite) { - const space = await spaces.getSpace(cinvite.spaceUri); - if (!space) { - return c.json({ error: "NotFound", reason: "space-not-found" }, 404); - } - // The token itself is the authorization: creator (manager+) pre-signed - // "anyone with this token gets level X". Grant directly, attributing - // to the creator so audit trails make sense. - await community.grant({ - spaceUri: cinvite.spaceUri, - subjectDid: sa.issuer, - accessLevel: cinvite.accessLevel, - grantedBy: cinvite.createdBy, - }); - await reconcile(community, spaces, cinvite.spaceUri, cinvite.createdBy); - return c.json({ - spaceUri: cinvite.spaceUri, - accessLevel: cinvite.accessLevel, - communityDid: space.ownerDid, - }); - } + const r = await community.tryRedeem({ + tokenHash, + callerDid: sa.issuer, + now, + }); + if (r) return relay(c, r); } - // Fall back to the spaces (user-owned) path. The spaces redeem filter - // already restricts to `kind IN ('join','read-join')` at the SQL level. - const sinvite = await spaces.redeemInvite(tokenHash, now); + // Fall back to the user-owned spaces path. The redeem filter at the SQL + // level already restricts to `kind IN ('join','read-join')`. + const sinvite = await authority.redeemInvite(tokenHash, now); if (!sinvite) { return c.json({ error: "InvalidInvite", reason: "expired-revoked-or-exhausted" }, 400); } - await spaces.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); + await authority.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); return c.json({ spaceUri: sinvite.spaceUri, kind: sinvite.kind }); }); } +/** Forward a community-handler response to the wire. */ +function relay(c: Context, r: HandlerResponse) { + return c.json(r.body, r.status as Parameters[1]); +} + function getAuth(c: Context): ServiceAuth { const a = c.get("serviceAuth") as ServiceAuth | undefined; if (!a) throw new Error("service auth not set"); diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 8c10854..535c15f 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -15,6 +15,8 @@ import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; import { registerCommunityRoutes } from "../community/router"; import type { CommunityRoutesOptions } from "../community/router"; import { CommunityAdapter } from "../community/adapter"; +import { createCommunityInviteHandler } from "../community/invite-handler"; +import { createCommunityWhoamiExtension } from "../community/whoami"; import { registerRealtimeRoutes } from "../realtime/router"; import type { RealtimeRoutesOptions } from "../realtime/router"; import { registerInviteRoutes } from "../invite/router"; @@ -129,6 +131,12 @@ export function createApp( } : null; + // Community is wired up at this layer, not from inside spaces / invite — + // those modules consume injected hooks, not community internals. The + // adapter is shared across every call site that needs it (publishing + // wrapper, collection routes, whoami extension, invite handler, realtime). + const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; + // Realtime pubsub is built whenever realtime is configured — independent of // spaces. With spaces, the spaces adapter is wrapped so private record/member // events publish to space:/community: topics. Without spaces, only public @@ -141,7 +149,6 @@ export function createApp( queueBound: config.realtime.queueBound, }); if (spacesCtx) { - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; const isCommunityDid = communityAdapter ? cachedIsCommunityDid(communityAdapter) : undefined; @@ -153,19 +160,25 @@ export function createApp( } registerAdminRoutes(app, db, config); - const communityAdapterForCollection = config.community - ? new CommunityAdapter(spacesDb) - : null; + registerCollectionRoutes(app, db, config, spacesCtx, { pubsub: realtimePubsub, - community: communityAdapterForCollection, + community: communityAdapter, }); registerFeedRoutes(app, db, config); registerNotifyRoute(app, db, config); - const communityAdapterForSpaces = config.community && spacesCtx - ? new CommunityAdapter(spacesDb) - : null; - registerSpacesRoutes(app, spacesDb, config, options.spaces, spacesCtx, communityAdapterForSpaces); + + // Spaces routes — get a whoami extension when community is configured so + // community-owned spaces get an `accessLevel` field. + const spacesOptions = { + ...options.spaces, + whoamiExtension: + options.spaces?.whoamiExtension ?? + (communityAdapter + ? createCommunityWhoamiExtension({ community: communityAdapter }) + : undefined), + }; + registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); if (config.community && spacesCtx) { // Community routes reuse the spaces service-auth middleware (same JWT verifier). @@ -184,12 +197,18 @@ export function createApp( if (config.spaces?.authority && spacesCtx) { // Unified invite surface: one `.invite.*` family that dispatches on - // space ownership (user-owned → addMember; community-owned → grant). + // space ownership (user-owned → addMember; community-owned → grant via + // an injected community-invite handler). const authMiddleware = options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier); - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; - registerInviteRoutes(app, config, spacesCtx.adapter, communityAdapter, { authMiddleware }); + const inviteHandler = communityAdapter + ? createCommunityInviteHandler({ + community: communityAdapter, + authority: spacesCtx.adapter, + }) + : null; + registerInviteRoutes(app, config, spacesCtx.adapter, inviteHandler, { authMiddleware }); } if (config.realtime && realtimePubsub) { @@ -202,7 +221,6 @@ export function createApp( options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier) : null; - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; registerRealtimeRoutes(app, config, spacesCtx?.adapter ?? null, communityAdapter, { authMiddleware, pubsub: realtimePubsub, diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index f3e3fd5..4e71c05 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -11,7 +11,6 @@ import { } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; -import { resolveEffectiveLevel } from "../community/acl"; import { buildSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, @@ -26,11 +25,27 @@ import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; import { create as createCid, toString as cidToString } from "@atcute/cid"; +/** Optional hook to extend `.spaceExt.whoami` with extra fields when a + * module above spaces (e.g. community) wants to override the default + * binary-membership response. If the hook returns a non-null object, that + * object is the entire response body. If null, falls through to the + * default behavior (just `isOwner`/`isMember`). + * + * Spaces stays community-agnostic: any consumer can plug in here. */ +export type WhoamiExtension = (input: { + spaceUri: string; + callerDid: string; + isOwner: boolean; + ownerDid: string; +}) => Promise | null>; + export interface SpacesRoutesOptions { /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ authMiddleware?: MiddlewareHandler; /** Storage adapter override. Defaults to HostedAdapter(db). */ adapter?: StorageAdapter; + /** Optional whoami extension; see {@link WhoamiExtension}. */ + whoamiExtension?: WhoamiExtension; } /** Umbrella registration: wires both the authority and the record-host @@ -43,8 +58,7 @@ export function registerSpacesRoutes( db: Database, config: ContrailConfig, options: SpacesRoutesOptions = {}, - ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null, - community?: import("../community/adapter").CommunityAdapter | null + ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null ): void { const spacesConfig = config.spaces; if (!spacesConfig) return; @@ -55,7 +69,7 @@ export function registerSpacesRoutes( const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, community ?? null); + registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); if (spacesConfig.recordHost) { registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth); @@ -70,7 +84,7 @@ export function registerAuthorityRoutes( authorityConfig: AuthorityConfig, config: ContrailConfig, auth: MiddlewareHandler, - community: import("../community/adapter").CommunityAdapter | null + whoamiExtension?: WhoamiExtension ): void { /** Space endpoints are emitted per-deployment under the configured namespace; * the deployment owns and publishes its own lexicons. The library ships @@ -243,8 +257,8 @@ export function registerAuthorityRoutes( }); // Unified whoami — `.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, - // accessLevel? }. `accessLevel` is present only when the target space is - // community-owned; for user-owned spaces membership is binary. + // ... }. Extra fields (e.g. accessLevel for community-owned spaces) come + // from the optional whoamiExtension hook; without one, response is binary. app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { const sa = getAuth(c); const spaceUri = c.req.query("spaceUri"); @@ -254,20 +268,17 @@ export function registerAuthorityRoutes( const isOwner = space.ownerDid === sa.issuer; - // Community-owned space: resolve through the access-level ladder. The - // reconciler keeps spaces_members in sync, so isMember derives from the - // effective level directly. - const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; - if (isCommunity) { - const level = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); - return c.json({ + if (whoamiExtension) { + const ext = await whoamiExtension({ + spaceUri, + callerDid: sa.issuer, isOwner, - isMember: isOwner || !!level, - accessLevel: level, + ownerDid: space.ownerDid, }); + if (ext) return c.json(ext); } - // User-owned space: binary membership. + // Default: binary membership. if (isOwner) return c.json({ isOwner: true, isMember: true }); const member = await authority.getMember(spaceUri, sa.issuer); return c.json({ isOwner: false, isMember: !!member }); -- 2.51.2 From 4a47874caae0df4ca86b7b4d6dba00bbbeb13bca Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:31:53 +0200 Subject: [PATCH 03/25] phase tres --- packages/contrail/src/core/spaces/auth.ts | 7 + .../contrail/src/core/spaces/credentials.ts | 253 +++++++++++ packages/contrail/src/core/spaces/router.ts | 377 ++++++++++++++--- packages/contrail/src/core/spaces/types.ts | 20 +- packages/contrail/src/index.ts | 19 + .../contrail/tests/spaces-credentials.test.ts | 398 ++++++++++++++++++ .../spaces/getCredential.json | 42 ++ .../spaces/refreshCredential.json | 38 ++ 8 files changed, 1084 insertions(+), 70 deletions(-) create mode 100644 packages/contrail/src/core/spaces/credentials.ts create mode 100644 packages/contrail/tests/spaces-credentials.test.ts create mode 100644 packages/lexicons/lexicon-templates/spaces/getCredential.json create mode 100644 packages/lexicons/lexicon-templates/spaces/refreshCredential.json diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 85ba814..4c1e817 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -136,6 +136,13 @@ export async function verifyServiceAuthRequest( }; } +/** Pull a space credential off the request — `X-Space-Credential: ` + * header. Returns the raw token or null. */ +export function extractSpaceCredential(request: Request): string | null { + const header = request.headers.get("X-Space-Credential"); + return header ? header.trim() : null; +} + /** Pull a read-grant invite token off the request — query string `?inviteToken=` * or `Authorization: Bearer atmo-invite:`. Returns the raw token (not * hashed) or null. Routes hash + look up via the adapter. */ diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts new file mode 100644 index 0000000..8c4a843 --- /dev/null +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -0,0 +1,253 @@ +/** Space-credential primitives: ES256 (P-256) JWTs minted by the authority, + * verified by the record host (or any third party that can resolve the + * authority's DID document). + * + * Format is a compact JWS: + * header = { alg: "ES256", typ: "JWT", kid: "#" } + * payload = { iss, sub, space, scope, iat, exp } + * + * - `iss` is the authority DID (the signer; for phase 3 this is the local + * authority's serviceDid; phase 4 adds a binding-resolution layer that + * lets the issuer be a *different* DID from the space owner). + * - `sub` is the caller DID — the credential bearer. + * - `space` is the full `ats:////` URI. + * - `scope` is "rw" or "read". + * + * We don't use a JWT library — Web Crypto's subtle covers everything (P-256 + * generate, sign, verify, JWK import/export) and saves a runtime dep. */ + +const ALG = "ES256"; +const TYP = "JWT"; +const DEFAULT_KEY_ID = "atproto_space_authority"; + +export type CredentialScope = "rw" | "read"; + +export interface CredentialClaims { + iss: string; + sub: string; + space: string; + scope: CredentialScope; + iat: number; // seconds since epoch + exp: number; // seconds since epoch +} + +export interface CredentialKeyMaterial { + /** Private key in JWK form. P-256 / ES256. */ + privateKey: JsonWebKey; + /** Public key in JWK form. Must match privateKey. */ + publicKey: JsonWebKey; + /** DID-doc verification method id. The full JWT `kid` becomes + * `#`. Defaults to "atproto_space_authority". */ + keyId?: string; +} + +/** Generate a fresh P-256 keypair as JWKs. Useful for local dev / tests; in + * production the operator generates once and stores out-of-band. */ +export async function generateAuthoritySigningKey(): Promise { + const pair = (await crypto.subtle.generateKey( + { name: "ECDSA", namedCurve: "P-256" }, + true, + ["sign", "verify"] + )) as CryptoKeyPair; + const privateKey = (await crypto.subtle.exportKey("jwk", pair.privateKey)) as JsonWebKey; + const publicKey = (await crypto.subtle.exportKey("jwk", pair.publicKey)) as JsonWebKey; + return { privateKey, publicKey }; +} + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function base64urlEncode(bytes: Uint8Array): string { + let s = btoa(String.fromCharCode(...bytes)); + return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +function base64urlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function jsonEncode(value: unknown): string { + return base64urlEncode(enc.encode(JSON.stringify(value))); +} + +function jsonDecode(seg: string): T { + return JSON.parse(dec.decode(base64urlDecode(seg))) as T; +} + +async function importPrivate(jwk: JsonWebKey): Promise { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["sign"] + ); +} + +async function importPublic(jwk: JsonWebKey): Promise { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"] + ); +} + +/** Sign a credential payload with the authority's private key. + * `iat` and `exp` are filled in by the caller (so tests can mint expired + * tokens deterministically). */ +export async function signCredential( + payload: CredentialClaims, + key: CredentialKeyMaterial +): Promise { + const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; + const header = { alg: ALG, typ: TYP, kid }; + const head = jsonEncode(header); + const body = jsonEncode(payload); + const signingInput = `${head}.${body}`; + const privateKey = await importPrivate(key.privateKey); + const sig = await crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + privateKey, + enc.encode(signingInput) + ); + return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; +} + +/** Issue a credential using the current wall-clock for iat/exp. */ +export async function issueCredential( + args: Omit & { ttlMs: number }, + key: CredentialKeyMaterial +): Promise<{ credential: string; expiresAt: number }> { + const now = Math.floor(Date.now() / 1000); + const expSec = now + Math.floor(args.ttlMs / 1000); + const claims: CredentialClaims = { + iss: args.iss, + sub: args.sub, + space: args.space, + scope: args.scope, + iat: now, + exp: expSec, + }; + const credential = await signCredential(claims, key); + return { credential, expiresAt: expSec * 1000 }; +} + +export type VerifyOk = { ok: true; claims: CredentialClaims }; +export type VerifyErr = { + ok: false; + reason: + | "malformed" + | "bad-alg" + | "bad-signature" + | "expired" + | "not-yet-valid" + | "wrong-space" + | "wrong-scope" + | "unknown-issuer"; +}; + +export interface VerifyOptions { + /** Optional: when set, rejects credentials whose `space` claim differs. + * Omit when verifying in middleware where the target space isn't known + * yet — handlers can do the match themselves against the verified + * claims. */ + expectedSpace?: string; + /** Optional: required scope (e.g. "rw" rejects read-only credentials on writes). */ + requiredScope?: CredentialScope; + /** Resolve a verification key for `iss`. If null, verification fails with + * unknown-issuer. */ + resolveKey: (iss: string, kid: string | undefined) => Promise; + /** Time provider for tests. Returns ms since epoch. */ + now?: () => number; +} + +export async function verifyCredential( + jwt: string, + opts: VerifyOptions +): Promise { + const parts = jwt.split("."); + if (parts.length !== 3) return { ok: false, reason: "malformed" }; + const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; + + let header: { alg?: string; typ?: string; kid?: string }; + let claims: CredentialClaims; + try { + header = jsonDecode(headSeg); + claims = jsonDecode(bodySeg); + } catch { + return { ok: false, reason: "malformed" }; + } + if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; + if (opts.expectedSpace !== undefined && claims.space !== opts.expectedSpace) { + return { ok: false, reason: "wrong-space" }; + } + if (opts.requiredScope === "rw" && claims.scope !== "rw") { + return { ok: false, reason: "wrong-scope" }; + } + + const nowMs = (opts.now ?? Date.now)(); + const nowSec = Math.floor(nowMs / 1000); + if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; + if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; + + const jwk = await opts.resolveKey(claims.iss, header.kid); + if (!jwk) return { ok: false, reason: "unknown-issuer" }; + + const publicKey = await importPublic(jwk); + const sigBytes = base64urlDecode(sigSeg); + const signingInput = `${headSeg}.${bodySeg}`; + const valid = await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + sigBytes, + enc.encode(signingInput) + ); + if (!valid) return { ok: false, reason: "bad-signature" }; + return { ok: true, claims }; +} + +/** Header reader for handlers that want to peek at `iss` before resolving the + * key (e.g. to short-circuit DID-doc fetches for the local authority). */ +export function decodeUnverifiedClaims(jwt: string): CredentialClaims | null { + const parts = jwt.split("."); + if (parts.length !== 3) return null; + try { + return jsonDecode(parts[1]!); + } catch { + return null; + } +} + +/** Verifier interface consumed by the record host. The record host doesn't + * care HOW credentials get verified — it only cares whether a given JWT is + * valid. Phase 3 ships an in-process verifier that knows the local + * authority's public key; phase 4 adds a binding-resolving verifier that + * consults PDS records / DID docs. */ +export interface CredentialVerifier { + /** Verify a credential's signature, expiry, and `not-before` window. Does + * NOT enforce a space match — handlers do that against the request URI. */ + verify(jwt: string): Promise; +} + +/** In-process verifier for the simple deployment: the authority and record + * host run in one process and the record host has direct access to the + * authority's public key. Rejects any credential whose `iss` isn't the + * configured authority. Phase 4 generalizes to multi-authority. */ +export function createInProcessVerifier(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): CredentialVerifier { + return { + verify(jwt) { + return verifyCredential(jwt, { + resolveKey: async (iss) => (iss === args.authorityDid ? args.publicKey : null), + }); + }, + }; +} diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index 4e71c05..fa9aeb5 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -8,12 +8,14 @@ import { checkInviteReadGrant, createServiceAuthMiddleware, extractInviteToken, + extractSpaceCredential, } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; import { buildSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, + DEFAULT_CREDENTIAL_TTL_MS, type AuthorityConfig, type RecordHostConfig, type RecordHost, @@ -23,6 +25,15 @@ import { } from "./types"; import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; +import { + createInProcessVerifier, + decodeUnverifiedClaims, + issueCredential, + verifyCredential, + type CredentialClaims, + type CredentialScope, + type CredentialVerifier, +} from "./credentials"; import { create as createCid, toString as cidToString } from "@atcute/cid"; /** Optional hook to extend `.spaceExt.whoami` with extra fields when a @@ -72,7 +83,16 @@ export function registerSpacesRoutes( registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); if (spacesConfig.recordHost) { - registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth); + // In-process verifier: when authority and record host are colocated, + // the host has direct access to the authority's public key. Phase 4 + // adds a binding-resolving verifier for split deployments. + const verifier = authorityConfig.signing + ? createInProcessVerifier({ + authorityDid: authorityConfig.serviceDid, + publicKey: authorityConfig.signing.publicKey, + }) + : undefined; + registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, verifier); } } @@ -159,6 +179,12 @@ export function registerAuthorityRoutes( return c.json({ space: publicSpaceView(space, false) }); } + if (authz.via === "credential") { + // Credential proves membership; derive isOwner from sub vs ownerDid. + const isOwner = authz.claims.sub === space.ownerDid; + return c.json({ space: publicSpaceView(space, isOwner) }); + } + const sa = authz.sa; const isOwner = sa.issuer === space.ownerDid; const member = isOwner ? null : await authority.getMember(uri, sa.issuer); @@ -283,31 +309,189 @@ export function registerAuthorityRoutes( const member = await authority.getMember(spaceUri, sa.issuer); return c.json({ isOwner: false, isMember: !!member }); }); + + // ---- Credential endpoints ---- + + /** Mint a space credential for a member of `spaceUri`. Caller is identified + * by the JWT issuer; the credential's `sub` is set to that DID. */ + app.post(`/xrpc/${SPACE}.getCredential`, auth, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + const member = isOwner ? null : await authority.getMember(body.spaceUri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + // App policy is checked at credential-issuance time. Existing credentials + // remain valid until expiry — that's the spec contract (revocation + // bounded by TTL, not synchronous). + if (space.appPolicy) { + const allowed = checkClientId(space.appPolicy, sa.clientId); + if (!allowed) return c.json({ error: "Forbidden", reason: "app-not-allowed" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: sa.issuer, + space: body.spaceUri, + scope: "rw", + ttlMs: ttl, + }, + authorityConfig.signing + ); + return c.json({ credential, expiresAt }); + }); + + /** Refresh an unexpired credential. Used by long-running clients to extend + * their access without going back through the JWT mint dance. The current + * credential must verify; the bearer must still be a member. */ + app.post(`/xrpc/${SPACE}.refreshCredential`, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const body = (await c.req.json().catch(() => null)) as { credential?: string } | null; + if (!body?.credential) { + return c.json({ error: "InvalidRequest", message: "credential required" }, 400); + } + const signing = authorityConfig.signing; + const claims = await verifyAndAuthorizeRefresh(body.credential, authorityConfig); + if ("error" in claims) return c.json(claims, claims.status); + + const space = await authority.getSpace(claims.space); + if (!space) return c.json({ error: "NotFound" }, 404); + const isOwner = space.ownerDid === claims.sub; + const member = isOwner ? null : await authority.getMember(claims.space, claims.sub); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: claims.sub, + space: claims.space, + scope: claims.scope, + ttlMs: ttl, + }, + signing + ); + return c.json({ credential, expiresAt }); + }); } -/** Register the **record host** XRPC surface — record + blob CRUD. Today - * consults the authority for membership / app-policy checks at write time; - * phase 3 adds a credential verifier that replaces those calls in - * split-deployment configurations. */ +/** Verify a credential presented at refreshCredential. Returns the claims, or + * an error envelope ready to relay. Different from the record-host verifier + * in two ways: (a) we don't have the expectedSpace yet — we read it from the + * credential itself; (b) we don't enforce a scope. */ +async function verifyAndAuthorizeRefresh( + credential: string, + authorityConfig: AuthorityConfig +): Promise { + const peek = decodeUnverifiedClaims(credential); + if (!peek) return { error: "InvalidRequest", reason: "malformed", status: 400 }; + if (peek.iss !== authorityConfig.serviceDid) { + return { error: "Forbidden", reason: "wrong-issuer", status: 401 }; + } + if (!authorityConfig.signing) { + return { error: "InvalidState", status: 401 }; + } + const signing = authorityConfig.signing; + const result = await verifyCredential(credential, { + expectedSpace: peek.space, + resolveKey: async (iss) => (iss === authorityConfig.serviceDid ? signing.publicKey : null), + }); + if (!result.ok) { + return { error: "InvalidCredential", reason: result.reason, status: 401 }; + } + return result.claims; +} + +/** App-policy check using just `clientId`. Mirrors `acl.ts:checkAppPolicy` + * but inlined here so the credential-issuance path doesn't need to construct + * a full AclInput. */ +function checkClientId( + appPolicy: NonNullable, + clientId: string | undefined +): boolean { + const listed = clientId ? appPolicy.apps.includes(clientId) : false; + if (appPolicy.mode === "allow") return !listed; + return listed; +} + +/** Register the **record host** XRPC surface — record + blob CRUD. + * + * Auth precedence on every route: + * 1. `X-Space-Credential` header (if a verifier is wired and the credential + * is valid) — caller DID = credential `sub`, no clientId. + * 2. Read-route invite token (`?inviteToken=` or `Bearer atmo-invite:...`). + * 3. Service-auth JWT (existing behavior) — caller DID = JWT issuer. + * + * When a credential is presented, the record host trusts it: no member + * check, no app-policy check (those happen at issuance time on the + * authority side). Service-auth requests still consult the authority — that + * bridge is what phase 5 cuts when the host/authority split goes runtime. */ export function registerRecordHostRoutes( app: Hono, recordHost: RecordHost, authority: SpaceAuthority, recordHostConfig: RecordHostConfig, config: ContrailConfig, - auth: MiddlewareHandler + auth: MiddlewareHandler, + /** Optional credential verifier. When present, the record host accepts + * `X-Space-Credential` as an alternative to a service-auth JWT. */ + credentialVerifier?: CredentialVerifier ): void { const SPACE = `${config.namespace}.space`; - /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is - * present so anonymous bearer reads don't 401 before the route handler can - * validate the token. */ + /** Auth wrapper: tries credential first, then delegates to JWT auth. */ + const authWithCredential: MiddlewareHandler = async (c, next) => { + const credToken = extractSpaceCredential(c.req.raw); + if (credToken) { + if (!credentialVerifier) { + return c.json( + { error: "AuthRequired", reason: "credential-verifier-not-configured" }, + 401 + ); + } + const result = await credentialVerifier.verify(credToken); + if (!result.ok) { + return c.json({ error: "AuthRequired", reason: result.reason }, 401); + } + c.set("spaceCredential", result.claims); + await next(); + return; + } + return auth(c, next); + }; + + /** Read-route auth: like {@link authWithCredential} but also short-circuits + * on a read-grant invite token. Token presence skips both credential and + * JWT middlewares; the route handler validates the token via authorizeRead. */ const readAuth: MiddlewareHandler = async (c, next) => { if (extractInviteToken(c.req.raw)) { await next(); return; } - return auth(c, next); + return authWithCredential(c, next); }; app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { @@ -336,6 +520,8 @@ export function registerRecordHostRoutes( return c.json({ error: "Forbidden", reason: result.reason }, 403); } } + // Credential and token paths are pre-authorized — credential's signature + // proves the authority granted access; token validation already happened. const list = await recordHost.listRecords(spaceUri, collection, { byUser: c.req.query("byUser") ?? undefined, @@ -379,8 +565,7 @@ export function registerRecordHostRoutes( }); // Write endpoints - app.post(`/xrpc/${SPACE}.putRecord`, auth, async (c) => { - const sa = getAuth(c); + app.post(`/xrpc/${SPACE}.putRecord`, authWithCredential, async (c) => { const body = (await c.req.json().catch(() => null)) as | { spaceUri?: string; collection?: string; rkey?: string; record?: Record } | null; @@ -390,15 +575,20 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, sa.issuer); - const result = checkAccess({ - op: "write", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } // Validate that every blob referenced by this record has already been // uploaded into this space. This mirrors how PDSes require uploadBlob @@ -426,17 +616,16 @@ export function registerRecordHostRoutes( await recordHost.putRecord({ spaceUri: body.spaceUri, collection: body.collection, - authorDid: sa.issuer, + authorDid: caller.callerDid, rkey, cid: null, record: body.record, createdAt: now, }); - return c.json({ rkey, authorDid: sa.issuer, createdAt: now }); + return c.json({ rkey, authorDid: caller.callerDid, createdAt: now }); }); - app.post(`/xrpc/${SPACE}.deleteRecord`, auth, async (c) => { - const sa = getAuth(c); + app.post(`/xrpc/${SPACE}.deleteRecord`, authWithCredential, async (c) => { const body = (await c.req.json().catch(() => null)) as | { spaceUri?: string; collection?: string; rkey?: string } | null; @@ -446,18 +635,26 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, sa.issuer); - const result = checkAccess({ - op: "delete", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - targetAuthorDid: sa.issuer, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "delete", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + targetAuthorDid: caller.callerDid, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + // Credential path: scope=rw is checked in resolveCaller. Delete remains + // author-scoped — the credential's `sub` is the caller, and we only + // delete records authored by that DID. - await recordHost.deleteRecord(body.spaceUri, body.collection, sa.issuer, body.rkey); + await recordHost.deleteRecord(body.spaceUri, body.collection, caller.callerDid, body.rkey); return c.json({ ok: true }); }); @@ -468,8 +665,7 @@ export function registerRecordHostRoutes( const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; const accept = blobsCfg.accept; - app.post(`/xrpc/${SPACE}.uploadBlob`, auth, async (c) => { - const sa = getAuth(c); + app.post(`/xrpc/${SPACE}.uploadBlob`, authWithCredential, async (c) => { const spaceUri = c.req.query("spaceUri"); if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); @@ -477,16 +673,21 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const aclResult = checkAccess({ - op: "write", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + const caller = resolveCaller(c, spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } } const mimeType = c.req.header("content-type") ?? "application/octet-stream"; @@ -524,7 +725,7 @@ export function registerRecordHostRoutes( cid: cidString, mimeType, size: bytes.byteLength, - authorDid: sa.issuer, + authorDid: caller.callerDid, createdAt: Date.now(), }); @@ -579,8 +780,7 @@ export function registerRecordHostRoutes( }); }); - app.get(`/xrpc/${SPACE}.listBlobs`, auth, async (c) => { - const sa = getAuth(c); + app.get(`/xrpc/${SPACE}.listBlobs`, authWithCredential, async (c) => { const spaceUri = c.req.query("spaceUri"); if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); @@ -588,16 +788,21 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const aclResult = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + const caller = resolveCaller(c, spaceUri, "read"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "read", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } } const result = await recordHost.listBlobMeta(spaceUri, { @@ -610,14 +815,30 @@ export function registerRecordHostRoutes( } } -/** Authorize a read request: either a valid service-auth JWT (which also - * identifies the caller for member checks downstream) or a valid read-grant - * invite token bearer. */ +/** Authorize a read request — three valid paths: a verified space credential + * (set by the credential middleware), a read-grant invite token, or a + * service-auth JWT. + * + * Credential and token paths skip the membership check downstream — the + * credential or token IS the proof. The JWT path requires a member check + * in the route handler. */ async function authorizeRead( c: Context, authority: SpaceAuthority, spaceUri: string -): Promise<{ via: "token" } | { via: "jwt"; sa: ServiceAuth } | Response> { +): Promise< + | { via: "credential"; claims: CredentialClaims } + | { via: "token" } + | { via: "jwt"; sa: ServiceAuth } + | Response +> { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== spaceUri) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + return { via: "credential", claims: cred }; + } const rawToken = extractInviteToken(c.req.raw); if (rawToken) { const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); @@ -627,11 +848,35 @@ async function authorizeRead( const sa = c.get("serviceAuth") as ServiceAuth | undefined; if (sa) return { via: "jwt", sa }; return c.json( - { error: "AuthRequired", message: "JWT or read-grant invite token required" }, + { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, 401 ); } +/** Unified caller resolution for write/manage paths on the record host. + * Either a verified credential (set by middleware) or a service-auth JWT. + * When a credential is present, also enforces space-match and the requested + * scope. Returns either a caller envelope or a Response to relay. */ +function resolveCaller( + c: Context, + requestSpace: string, + requiredScope: CredentialScope +): { callerDid: string; clientId: string | undefined; viaCredential: boolean } | Response { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== requestSpace) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + if (requiredScope === "rw" && cred.scope !== "rw") { + return c.json({ error: "Forbidden", reason: "credential-wrong-scope" }, 403); + } + return { callerDid: cred.sub, clientId: undefined, viaCredential: true }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (!sa) return c.json({ error: "AuthRequired", reason: "no-auth" }, 401); + return { callerDid: sa.issuer, clientId: sa.clientId, viaCredential: false }; +} + function getAuth(c: Parameters[0]): ServiceAuth { const auth = c.get("serviceAuth") as ServiceAuth | undefined; if (!auth) throw new Error("service auth not set"); diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index fb05d2a..5429c96 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -1,6 +1,7 @@ import type { Database } from "../types"; import type { DidDocumentResolver } from "@atcute/identity-resolver"; import type { BlobAdapter } from "./blob-adapter"; +import type { CredentialKeyMaterial } from "./credentials"; export type AppPolicyMode = "allow" | "deny"; @@ -25,21 +26,32 @@ export interface SpacesBlobsConfig { export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; +/** Default credential lifetime. The rough spec calls for 2–4h; we pick the + * lower bound so revocation (kicked-from-space) is observable within 2h. */ +export const DEFAULT_CREDENTIAL_TTL_MS = 2 * 60 * 60 * 1000; + /** Configuration for the **space authority** role: holds the member list, - * signs credentials (later phases), and gates space-management operations. - * In a fully-split deployment, the authority can run in a different process - * (or even a different operator) than the record host. */ + * signs credentials, and gates space-management operations. In a fully-split + * deployment, the authority can run in a different process (or even a + * different operator) than the record host. */ export interface AuthorityConfig { /** NSID that identifies the kind of space this authority hosts, * e.g. "tools.atmo.event.space". */ type: string; - /** Service DID that service-auth tokens must target (aud claim). */ + /** Service DID that service-auth tokens must target (aud claim) AND that + * signs credentials it issues (`iss` claim on emitted JWTs). */ serviceDid: string; /** Default app policy applied to new spaces. */ defaultAppPolicy?: AppPolicy; /** DID document resolver for service-auth JWT verification. * Defaults to a composite PLC + did:web resolver if omitted. */ resolver?: DidDocumentResolver; + /** Signing key material for issuing space credentials. When omitted, + * `.space.getCredential` returns 501 NotImplemented and the record + * host's credential-verifying middleware can't be wired up. */ + signing?: CredentialKeyMaterial; + /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ + credentialTtlMs?: number; } /** Configuration for the **record host** role: stores per-space records and diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 8d678cb..54e22bf 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -68,6 +68,25 @@ export { export type { BlobAdapter, BlobUploadMeta, R2BucketLike } from "./core/spaces/blob-adapter"; export type { SpacesBlobsConfig, BlobMetaRow } from "./core/spaces/types"; +// Space credentials +export { + generateAuthoritySigningKey, + signCredential, + issueCredential, + verifyCredential, + decodeUnverifiedClaims, + createInProcessVerifier, +} from "./core/spaces/credentials"; +export type { + CredentialClaims, + CredentialKeyMaterial, + CredentialScope, + CredentialVerifier, + VerifyOk, + VerifyErr, + VerifyOptions, +} from "./core/spaces/credentials"; + // Realtime export type { PubSub, diff --git a/packages/contrail/tests/spaces-credentials.test.ts b/packages/contrail/tests/spaces-credentials.test.ts new file mode 100644 index 0000000..ec3c428 --- /dev/null +++ b/packages/contrail/tests/spaces-credentials.test.ts @@ -0,0 +1,398 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + generateAuthoritySigningKey, + signCredential, + verifyCredential, + issueCredential, +} from "../src/core/spaces/credentials"; +import type { CredentialKeyMaterial } from "../src/core/spaces/credentials"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const CHARLIE = "did:plc:charlie"; + +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function makeConfig(): ContrailConfig { + return { + namespace: "test.cred", + collections: { + message: { collection: "app.event.message" }, + }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + credentialTtlMs: 60_000, + }, + recordHost: {}, + }, + }; +} + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; +} + +async function makeApp(): Promise { + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); +} + +function call( + app: Hono, + method: string, + path: string, + did: string | null, + body?: any, + extraHeaders?: Record +) { + const headers: Record = { ...(extraHeaders ?? {}) }; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +async function createSpace(app: Hono, owner: string): Promise { + const res = await call(app, "POST", "/xrpc/test.cred.space.createSpace", owner, {}); + expect(res.status).toBe(200); + return ((await res.json()) as any).space.uri; +} + +async function getCredential(app: Hono, did: string, spaceUri: string): Promise { + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", did, { spaceUri }); + expect(res.status).toBe(200); + return ((await res.json()) as any).credential; +} + +describe("space credentials — sign/verify primitives", () => { + it("round-trips via verifyCredential", async () => { + const { credential } = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: "ats://did:plc:alice/test/main", + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://did:plc:alice/test/main", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.claims.iss).toBe(SERVICE_DID); + expect(result.claims.sub).toBe(ALICE); + expect(result.claims.scope).toBe("rw"); + } + }); + + it("rejects expired credentials", async () => { + const past = Math.floor(Date.now() / 1000) - 10; + const credential = await signCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: "ats://x/y/z", + scope: "rw", + iat: past - 60, + exp: past, + }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("expired"); + }); + + it("rejects wrong-space credentials", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://a/b/c", scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://different/space/here", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("wrong-space"); + }); + + it("rejects credentials signed by a different key", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://x/y/z", scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const otherKey = await generateAuthoritySigningKey(); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + resolveKey: async () => otherKey.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("bad-signature"); + }); + + it("rejects unknown issuer", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://x/y/z", scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + resolveKey: async () => null, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); + + it("scope=rw rejects read-only credentials when verifier requires rw", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://x/y/z", scope: "read", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + requiredScope: "rw", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("wrong-scope"); + }); +}); + +describe("space credentials — getCredential / refreshCredential endpoints", () => { + it("issues a credential for a space owner", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", ALICE, { + spaceUri: uri, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.credential).toBeTypeOf("string"); + expect(body.expiresAt).toBeTypeOf("number"); + expect(body.expiresAt).toBeGreaterThan(Date.now()); + }); + + it("issues a credential for a member who isn't the owner", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + await call(app, "POST", "/xrpc/test.cred.space.addMember", ALICE, { + spaceUri: uri, + did: BOB, + }); + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", BOB, { + spaceUri: uri, + }); + expect(res.status).toBe(200); + }); + + it("denies non-members", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", CHARLIE, { + spaceUri: uri, + }); + expect(res.status).toBe(403); + expect((await res.json()).reason).toBe("not-member"); + }); + + it("refreshes an unexpired credential", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const cred = await getCredential(app, ALICE, uri); + + const res = await call(app, "POST", "/xrpc/test.cred.space.refreshCredential", null, { + credential: cred, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.credential).toBeTypeOf("string"); + expect(body.credential).not.toBe(cred); // fresh iat/exp → different signature + }); + + it("refresh rejects when the holder is no longer a member", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + await call(app, "POST", "/xrpc/test.cred.space.addMember", ALICE, { + spaceUri: uri, + did: BOB, + }); + const cred = await getCredential(app, BOB, uri); + // Owner kicks Bob out. + const removeRes = await call(app, "POST", "/xrpc/test.cred.space.removeMember", ALICE, { + spaceUri: uri, + did: BOB, + }); + expect(removeRes.status).toBe(200); + + const res = await call(app, "POST", "/xrpc/test.cred.space.refreshCredential", null, { + credential: cred, + }); + expect(res.status).toBe(403); + }); +}); + +describe("space credentials — record host accepts X-Space-Credential", () => { + it("putRecord works with a credential and no service-auth JWT", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const cred = await getCredential(app, ALICE, uri); + + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, // no X-Test-Did header + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "hi from credential" }, + }, + { "X-Space-Credential": cred } + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.authorDid).toBe(ALICE); + expect(body.rkey).toBeTypeOf("string"); + }); + + it("listRecords works with a credential", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const cred = await getCredential(app, ALICE, uri); + // Plant a record via the JWT path so listRecords has something to return. + await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + ALICE, + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "one" }, + } + ); + + const res = await call( + app, + "GET", + `/xrpc/test.cred.space.listRecords?spaceUri=${encodeURIComponent(uri)}&collection=app.event.message`, + null, + undefined, + { "X-Space-Credential": cred } + ); + expect(res.status).toBe(200); + expect(((await res.json()) as any).records.length).toBe(1); + }); + + it("rejects credential issued for a different space", async () => { + const app = await makeApp(); + const uriA = await createSpace(app, ALICE); + // Alice can create a second one; key auto-generated, owner is implicit member. + const uriB = await createSpace(app, ALICE); + const credForA = await getCredential(app, ALICE, uriA); + + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, + { + spaceUri: uriB, + collection: "app.event.message", + record: { $type: "app.event.message", text: "wrong space" }, + }, + { "X-Space-Credential": credForA } + ); + expect(res.status).toBe(403); + expect((await res.json()).reason).toBe("credential-wrong-space"); + }); + + it("rejects malformed credentials", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "x" }, + }, + { "X-Space-Credential": "not-a-jwt" } + ); + expect(res.status).toBe(401); + expect((await res.json()).reason).toBe("malformed"); + }); + + it("rejects credentials forged with an unknown issuer", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const otherKey = await generateAuthoritySigningKey(); + const { credential } = await issueCredential( + { + iss: "did:web:attacker.example", + sub: ALICE, + space: uri, + scope: "rw", + ttlMs: 60_000, + }, + otherKey + ); + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "x" }, + }, + { "X-Space-Credential": credential } + ); + expect(res.status).toBe(401); + expect((await res.json()).reason).toBe("unknown-issuer"); + }); +}); diff --git a/packages/lexicons/lexicon-templates/spaces/getCredential.json b/packages/lexicons/lexicon-templates/spaces/getCredential.json new file mode 100644 index 0000000..e24e79b --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/getCredential.json @@ -0,0 +1,42 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.getCredential", + "defs": { + "main": { + "type": "procedure", + "description": "Mint a short-lived space credential for a member of `spaceUri`. The caller is identified by their service-auth JWT; the credential's `sub` is set to that DID, scoped 'rw'. Use the returned credential as the `X-Space-Credential` header on subsequent requests instead of minting a fresh service-auth JWT each time. Refresh via `refreshCredential` before expiry.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["spaceUri"], + "properties": { + "spaceUri": { "type": "string" } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["credential", "expiresAt"], + "properties": { + "credential": { + "type": "string", + "description": "Compact JWS (ES256) signed by the authority's key. Header `kid` references the verification method on the authority's DID document." + }, + "expiresAt": { + "type": "integer", + "description": "Expiry as ms since epoch." + } + } + } + }, + "errors": [ + { "name": "NotFound" }, + { "name": "Forbidden" }, + { "name": "NotImplemented", "description": "Authority is not configured to sign credentials." } + ] + } + } +} diff --git a/packages/lexicons/lexicon-templates/spaces/refreshCredential.json b/packages/lexicons/lexicon-templates/spaces/refreshCredential.json new file mode 100644 index 0000000..7e8bbfb --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/refreshCredential.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.refreshCredential", + "defs": { + "main": { + "type": "procedure", + "description": "Refresh an unexpired space credential. The current credential must verify and the bearer must still be a member. Returns a fresh credential (same scope, same sub, same space) with a new expiry. No service-auth JWT required — the credential itself is the authentication.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["credential"], + "properties": { + "credential": { "type": "string" } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["credential", "expiresAt"], + "properties": { + "credential": { "type": "string" }, + "expiresAt": { "type": "integer" } + } + } + }, + "errors": [ + { "name": "InvalidRequest" }, + { "name": "InvalidCredential" }, + { "name": "NotFound" }, + { "name": "Forbidden" }, + { "name": "NotImplemented" } + ] + } + } +} -- 2.51.2 From 774a833a9f832c6ddaca5a93b37d7c64a0db1cb2 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:40:08 +0200 Subject: [PATCH 04/25] phase 4 --- packages/contrail/src/core/spaces/binding.ts | 256 +++++++++++ .../contrail/src/core/spaces/credentials.ts | 33 +- packages/contrail/src/core/spaces/router.ts | 39 +- packages/contrail/src/index.ts | 14 + .../contrail/tests/spaces-binding.test.ts | 435 ++++++++++++++++++ .../lexicon-templates/spaces/declaration.json | 31 ++ 6 files changed, 797 insertions(+), 11 deletions(-) create mode 100644 packages/contrail/src/core/spaces/binding.ts create mode 100644 packages/contrail/tests/spaces-binding.test.ts create mode 100644 packages/lexicons/lexicon-templates/spaces/declaration.json diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts new file mode 100644 index 0000000..d349d55 --- /dev/null +++ b/packages/contrail/src/core/spaces/binding.ts @@ -0,0 +1,256 @@ +/** Binding resolution: given a space URI, which DID is authorized to sign + * credentials for it, and where do we find that DID's verification key? + * + * Two layers of pluggable resolvers compose into a credential verifier: + * + * BindingResolver — `ats:////` → authority DID + * KeyResolver — (DID, kid) → JsonWebKey + * + * The BindingResolver is what makes user-owned-DID-with-PDS-record work: + * given a space URI, we resolve the owner's PDS, fetch the declaration + * record, and read its `authority` field. For provisioned (no-PDS) DIDs we + * fall back to the owner DID's `#atproto_space_authority` service entry. + * And finally for the trivial case (HappyView-style "owner self-issues"), + * we return the owner DID itself. + * + * See conversation history (phase 4 design) for the rationale on why these + * three sources, in this order. */ + +import type { DidDocumentResolver } from "@atcute/identity-resolver"; +import type { Did } from "@atcute/lexicons"; +import { parseSpaceUri } from "./uri"; + +export interface BindingResolver { + /** Resolve the DID authorized to sign credentials for this space. Returns + * null if no binding could be found via this resolver — the composite + * walks down its list looking for a non-null. */ + resolveAuthority(spaceUri: string): Promise; +} + +export interface KeyResolver { + /** Resolve `did`'s verification key for credential signing. `kid` is the + * full header `kid` value (e.g. "did:web:x.com#atproto_space_authority"), + * used to disambiguate when a DID doc lists multiple methods. */ + resolveKey(did: string, kid: string | undefined): Promise; +} + +// --------------------------------------------------------------------------- +// Binding resolvers +// --------------------------------------------------------------------------- + +/** Always returns the configured authority DID. Used in-process when the + * authority and record host run in one deployment — no need to walk DID + * docs or PDSes; we know what we are. */ +export function createLocalBindingResolver(args: { + authorityDid: string; +}): BindingResolver { + const { authorityDid } = args; + return { + async resolveAuthority() { + return authorityDid; + }, + }; +} + +/** Returns the space owner DID as the authority. This is the implicit + * fallback ("HappyView path") — when no PDS record and no DID-doc service + * entry declare an issuer, the owner is taken to be its own. Whether the + * resulting credential actually verifies depends on whether the owner's DID + * doc publishes a usable signing key. */ +export function createOwnerSelfBindingResolver(): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + return parts ? parts.ownerDid : null; + }, + }; +} + +/** Walks the resolver list in order, returns the first non-null. Use this + * to compose [pdsRecord, didDocService, ownerSelf] etc. */ +export function createCompositeBindingResolver( + resolvers: BindingResolver[] +): BindingResolver { + return { + async resolveAuthority(spaceUri) { + for (const r of resolvers) { + const did = await r.resolveAuthority(spaceUri); + if (did) return did; + } + return null; + }, + }; +} + +/** Reads a space-declaration record from the owner's PDS at + * `at:////` and returns its `authority` field if present. + * + * This is the user-owned-DID path: the user writes a record to their PDS + * authorizing some service as the space's authority, no DID-doc edits + * required. */ +export function createPdsBindingResolver(args: { + /** DID resolver, used to look up the owner's PDS endpoint. */ + resolver: DidDocumentResolver; + /** Fetch impl. Defaults to `globalThis.fetch`. */ + fetch?: typeof fetch; + /** Per-request timeout in ms. Defaults to 5000. */ + timeoutMs?: number; +}): BindingResolver { + const fetchImpl = args.fetch ?? globalThis.fetch; + const timeoutMs = args.timeoutMs ?? 5000; + + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + const pds = await pdsEndpointFor(args.resolver, parts.ownerDid); + if (!pds) return null; + + const url = new URL(`${pds}/xrpc/com.atproto.repo.getRecord`); + url.searchParams.set("repo", parts.ownerDid); + url.searchParams.set("collection", parts.type); + url.searchParams.set("rkey", parts.key); + + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + let res: Response; + try { + res = await fetchImpl(url.toString(), { signal: ctrl.signal }); + } catch { + return null; + } finally { + clearTimeout(timer); + } + if (!res.ok) return null; + const body = (await res.json().catch(() => null)) as + | { value?: { authority?: unknown } } + | null; + const authority = body?.value?.authority; + return typeof authority === "string" && authority.startsWith("did:") ? authority : null; + }, + }; +} + +/** Reads `service[id="#atproto_space_authority"].serviceEndpoint` from the + * owner's DID doc. This is the no-PDS path — useful for provisioned space + * DIDs that exist as DID docs only. + * + * Note the service endpoint here is a *DID*, not a URL. The DID names the + * authority; the key resolver's job is to then fetch its verification key. + * For DID docs that declare a URL endpoint, we treat the URL as a + * did:web hint — caller can normalize. */ +export function createDidDocBindingResolver(args: { + resolver: DidDocumentResolver; + /** Service id to look up. Defaults to "#atproto_space_authority". */ + serviceId?: string; +}): BindingResolver { + const serviceId = args.serviceId ?? "#atproto_space_authority"; + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + let doc; + try { + doc = await args.resolver.resolve(parts.ownerDid as Did); + } catch { + return null; + } + const entry = doc.service?.find((s: { id?: string }) => s.id === serviceId); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + if (typeof endpoint !== "string") return null; + // Endpoint may be a DID (preferred) or a URL hint. Only DIDs are + // verifiable downstream; URLs require the caller to map URL → DID. + return endpoint.startsWith("did:") ? endpoint : null; + }, + }; +} + +// --------------------------------------------------------------------------- +// Key resolvers +// --------------------------------------------------------------------------- + +/** Knows the local authority's public key directly. Returns null for any + * other DID — composite with a DID-doc resolver if you also accept + * external authorities. */ +export function createLocalKeyResolver(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): KeyResolver { + return { + async resolveKey(did) { + return did === args.authorityDid ? args.publicKey : null; + }, + }; +} + +/** Resolves a DID, finds the verification method matching `kid`, returns + * its `publicKeyJwk`. */ +export function createDidDocKeyResolver(args: { + resolver: DidDocumentResolver; +}): KeyResolver { + return { + async resolveKey(did, kid) { + let doc; + try { + doc = await args.resolver.resolve(did as Did); + } catch { + return null; + } + const methods = (doc as { verificationMethod?: VerificationMethod[] }).verificationMethod; + if (!methods) return null; + // kid is "#" — we match against the method.id which DID + // docs spell as "#" too. + const method = kid + ? methods.find((m) => m.id === kid) + : methods[0]; + if (!method?.publicKeyJwk) return null; + return method.publicKeyJwk as JsonWebKey; + }, + }; +} + +/** Walks resolvers in order; returns the first non-null. */ +export function createCompositeKeyResolver( + resolvers: KeyResolver[] +): KeyResolver { + return { + async resolveKey(did, kid) { + for (const r of resolvers) { + const k = await r.resolveKey(did, kid); + if (k) return k; + } + return null; + }, + }; +} + +interface VerificationMethod { + id: string; + type?: string; + controller?: string; + publicKeyJwk?: unknown; + publicKeyMultibase?: string; +} + +// --------------------------------------------------------------------------- +// Internal: PDS endpoint lookup +// --------------------------------------------------------------------------- + +async function pdsEndpointFor( + resolver: DidDocumentResolver, + did: string +): Promise { + let doc; + try { + doc = await resolver.resolve(did as Did); + } catch { + return null; + } + const entry = doc.service?.find( + (s: { id?: string }) => s.id === "#atproto_pds" + ); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + return typeof endpoint === "string" ? endpoint : null; +} diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts index 8c4a843..d5c3029 100644 --- a/packages/contrail/src/core/spaces/credentials.ts +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -238,7 +238,8 @@ export interface CredentialVerifier { /** In-process verifier for the simple deployment: the authority and record * host run in one process and the record host has direct access to the * authority's public key. Rejects any credential whose `iss` isn't the - * configured authority. Phase 4 generalizes to multi-authority. */ + * configured authority. Phase 4 has a more general + * {@link createBindingCredentialVerifier} that does proper binding lookup. */ export function createInProcessVerifier(args: { authorityDid: string; publicKey: JsonWebKey; @@ -251,3 +252,33 @@ export function createInProcessVerifier(args: { }, }; } + +/** Verifier composed of a {@link BindingResolver} (which DID is authorized + * to issue for this space?) and a {@link KeyResolver} (what's that DID's + * public key?). This is the production-shape verifier — phase 4's main + * contribution. + * + * Verification flow: + * 1. Decode the JWT's claims (no signature check yet). + * 2. Ask the binding resolver: who's authorized for `claims.space`? + * 3. Confirm `claims.iss === authorizedDid`. + * 4. Ask the key resolver for that DID's verification key. + * 5. Verify signature + expiry + scope match. + */ +export function createBindingCredentialVerifier(args: { + bindings: import("./binding").BindingResolver; + keys: import("./binding").KeyResolver; +}): CredentialVerifier { + return { + async verify(jwt) { + const peek = decodeUnverifiedClaims(jwt); + if (!peek) return { ok: false, reason: "malformed" }; + const authorizedDid = await args.bindings.resolveAuthority(peek.space); + if (!authorizedDid) return { ok: false, reason: "unknown-issuer" }; + if (peek.iss !== authorizedDid) return { ok: false, reason: "unknown-issuer" }; + return verifyCredential(jwt, { + resolveKey: (iss, kid) => args.keys.resolveKey(iss, kid), + }); + }, + }; +} diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index fa9aeb5..c0c9e92 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -26,7 +26,7 @@ import { import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; import { - createInProcessVerifier, + createBindingCredentialVerifier, decodeUnverifiedClaims, issueCredential, verifyCredential, @@ -34,6 +34,10 @@ import { type CredentialScope, type CredentialVerifier, } from "./credentials"; +import { + createLocalBindingResolver, + createLocalKeyResolver, +} from "./binding"; import { create as createCid, toString as cidToString } from "@atcute/cid"; /** Optional hook to extend `.spaceExt.whoami` with extra fields when a @@ -57,6 +61,12 @@ export interface SpacesRoutesOptions { adapter?: StorageAdapter; /** Optional whoami extension; see {@link WhoamiExtension}. */ whoamiExtension?: WhoamiExtension; + /** Optional credential verifier for the record host. When omitted, a + * default in-process binding verifier is built from the authority's + * signing config (Local binding + Local key resolvers). Override to + * accept credentials from external authorities — wire in PDS-record / + * DID-doc binding resolvers and a DID-doc key resolver. */ + credentialVerifier?: CredentialVerifier; } /** Umbrella registration: wires both the authority and the record-host @@ -83,15 +93,24 @@ export function registerSpacesRoutes( registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); if (spacesConfig.recordHost) { - // In-process verifier: when authority and record host are colocated, - // the host has direct access to the authority's public key. Phase 4 - // adds a binding-resolving verifier for split deployments. - const verifier = authorityConfig.signing - ? createInProcessVerifier({ - authorityDid: authorityConfig.serviceDid, - publicKey: authorityConfig.signing.publicKey, - }) - : undefined; + // Build the default in-process verifier when the authority can sign: + // Local binding (always points at the configured authority) + Local + // key resolver (knows the authority's public key directly). Caller + // can override via `options.credentialVerifier` to accept external + // authorities — wire in PDS-record / DID-doc resolvers there. + const verifier = + options.credentialVerifier ?? + (authorityConfig.signing + ? createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ + authorityDid: authorityConfig.serviceDid, + }), + keys: createLocalKeyResolver({ + authorityDid: authorityConfig.serviceDid, + publicKey: authorityConfig.signing.publicKey, + }), + }) + : undefined); registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, verifier); } } diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 54e22bf..5e59dac 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -76,6 +76,7 @@ export { verifyCredential, decodeUnverifiedClaims, createInProcessVerifier, + createBindingCredentialVerifier, } from "./core/spaces/credentials"; export type { CredentialClaims, @@ -87,6 +88,19 @@ export type { VerifyOptions, } from "./core/spaces/credentials"; +// Binding + key resolution +export { + createLocalBindingResolver, + createOwnerSelfBindingResolver, + createCompositeBindingResolver, + createPdsBindingResolver, + createDidDocBindingResolver, + createLocalKeyResolver, + createDidDocKeyResolver, + createCompositeKeyResolver, +} from "./core/spaces/binding"; +export type { BindingResolver, KeyResolver } from "./core/spaces/binding"; + // Realtime export type { PubSub, diff --git a/packages/contrail/tests/spaces-binding.test.ts b/packages/contrail/tests/spaces-binding.test.ts new file mode 100644 index 0000000..427b6f2 --- /dev/null +++ b/packages/contrail/tests/spaces-binding.test.ts @@ -0,0 +1,435 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + createLocalBindingResolver, + createOwnerSelfBindingResolver, + createCompositeBindingResolver, + createPdsBindingResolver, + createDidDocBindingResolver, + createLocalKeyResolver, + createDidDocKeyResolver, + createCompositeKeyResolver, +} from "../src/core/spaces/binding"; +import { + generateAuthoritySigningKey, + issueCredential, + createBindingCredentialVerifier, +} from "../src/core/spaces/credentials"; +import type { CredentialKeyMaterial } from "../src/core/spaces/credentials"; + +const ALICE = "did:plc:alice"; +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +const SPACE_URI = "ats://did:plc:alice/com.example.event.space/main"; + +describe("BindingResolver — basic resolvers", () => { + it("Local always returns the configured DID", async () => { + const r = createLocalBindingResolver({ authorityDid: SERVICE_DID }); + expect(await r.resolveAuthority(SPACE_URI)).toBe(SERVICE_DID); + expect(await r.resolveAuthority("ats://did:plc:bob/x/y")).toBe(SERVICE_DID); + }); + + it("OwnerSelf parses the owner from the URI", async () => { + const r = createOwnerSelfBindingResolver(); + expect(await r.resolveAuthority(SPACE_URI)).toBe(ALICE); + expect(await r.resolveAuthority("not a space uri")).toBeNull(); + }); + + it("Composite returns the first non-null result", async () => { + const r = createCompositeBindingResolver([ + { resolveAuthority: async () => null }, + { resolveAuthority: async () => "did:web:second" }, + { resolveAuthority: async () => "did:web:third" }, + ]); + expect(await r.resolveAuthority(SPACE_URI)).toBe("did:web:second"); + }); + + it("Composite returns null if every resolver returns null", async () => { + const r = createCompositeBindingResolver([ + { resolveAuthority: async () => null }, + { resolveAuthority: async () => null }, + ]); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); +}); + +describe("BindingResolver — PDS record", () => { + function mockResolver(opts: { + pdsEndpoint?: string; + fail?: boolean; + }): any { + return { + resolve: async (did: string) => { + if (opts.fail) throw new Error("nope"); + return { + id: did, + service: opts.pdsEndpoint + ? [ + { + id: "#atproto_pds", + type: "AtprotoPersonalDataServer", + serviceEndpoint: opts.pdsEndpoint, + }, + ] + : [], + }; + }, + }; + } + + function mockFetch(map: Map): typeof fetch { + return (async (url: string) => { + const u = String(url); + const found = [...map.entries()].find(([k]) => u.startsWith(k)); + if (!found) return new Response("not found", { status: 404 }); + return new Response(JSON.stringify(found[1]), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + } + + it("reads `authority` from a declaration record", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + uri: "at://did:plc:alice/com.example.event.space/main", + value: { + $type: "com.example.event.space", + authority: "did:web:custom-authority.example", + recordHost: "did:web:host.example", + createdAt: "2026-04-30T00:00:00Z", + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBe("did:web:custom-authority.example"); + }); + + it("returns null when the PDS record is missing", async () => { + const fetch = mockFetch(new Map()); // 404 for everything + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when the record has no authority field", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { value: { $type: "x", createdAt: "..." } }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when the owner DID doc has no PDS endpoint", async () => { + const fetch = mockFetch(new Map()); + const r = createPdsBindingResolver({ + resolver: mockResolver({}), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); +}); + +describe("BindingResolver — DID-doc service entry", () => { + function mockResolver(serviceDid: string | null): any { + return { + resolve: async (did: string) => ({ + id: did, + service: serviceDid + ? [ + { + id: "#atproto_space_authority", + type: "AtprotoSpaceAuthority", + serviceEndpoint: serviceDid, + }, + ] + : [], + }), + }; + } + + it("reads the #atproto_space_authority service entry", async () => { + const r = createDidDocBindingResolver({ + resolver: mockResolver("did:web:authority.example"), + }); + expect(await r.resolveAuthority(SPACE_URI)).toBe("did:web:authority.example"); + }); + + it("returns null when the service entry is absent", async () => { + const r = createDidDocBindingResolver({ resolver: mockResolver(null) }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("rejects URL-shaped service endpoints (must be a DID)", async () => { + const r = createDidDocBindingResolver({ + resolver: mockResolver("https://authority.example.com"), + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); +}); + +describe("KeyResolver", () => { + it("Local matches by DID", async () => { + const r = createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }); + expect(await r.resolveKey(SERVICE_DID, undefined)).toEqual(SIGNING.publicKey); + expect(await r.resolveKey("did:web:other", undefined)).toBeNull(); + }); + + it("DidDoc finds the verification method matching kid", async () => { + const otherKey = await generateAuthoritySigningKey(); + const resolver = { + resolve: async (did: string) => ({ + id: did, + verificationMethod: [ + { + id: `${did}#atproto_space_authority`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: SIGNING.publicKey, + }, + { + id: `${did}#another-key`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: otherKey.publicKey, + }, + ], + }), + }; + const r = createDidDocKeyResolver({ resolver: resolver as any }); + const key = await r.resolveKey( + "did:web:authority.example", + "did:web:authority.example#atproto_space_authority" + ); + expect(key).toEqual(SIGNING.publicKey); + }); + + it("DidDoc returns the second key when kid points there", async () => { + const otherKey = await generateAuthoritySigningKey(); + const resolver = { + resolve: async (did: string) => ({ + id: did, + verificationMethod: [ + { + id: `${did}#atproto_space_authority`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: SIGNING.publicKey, + }, + { + id: `${did}#another-key`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: otherKey.publicKey, + }, + ], + }), + }; + const r = createDidDocKeyResolver({ resolver: resolver as any }); + const key = await r.resolveKey( + "did:web:authority.example", + "did:web:authority.example#another-key" + ); + expect(key).toEqual(otherKey.publicKey); + }); + + it("Composite walks resolvers in order", async () => { + const fallback = await generateAuthoritySigningKey(); + const r = createCompositeKeyResolver([ + { resolveKey: async () => null }, + { resolveKey: async () => fallback.publicKey }, + ]); + expect(await r.resolveKey("did:any", undefined)).toEqual(fallback.publicKey); + }); +}); + +describe("createBindingCredentialVerifier — composes binding + key resolvers", () => { + it("verifies a credential whose iss matches the binding-resolved authority", async () => { + const verifier = createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ authorityDid: SERVICE_DID }), + keys: createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }), + }); + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: SPACE_URI, scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifier.verify(credential); + expect(result.ok).toBe(true); + }); + + it("rejects when credential iss disagrees with the binding", async () => { + const verifier = createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ authorityDid: SERVICE_DID }), + keys: createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }), + }); + const { credential } = await issueCredential( + { + iss: "did:web:imposter.example", + sub: ALICE, + space: SPACE_URI, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const result = await verifier.verify(credential); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); + + it("rejects when no resolver knows the authority", async () => { + const verifier = createBindingCredentialVerifier({ + bindings: { resolveAuthority: async () => null }, + keys: { resolveKey: async () => null }, + }); + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: SPACE_URI, scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifier.verify(credential); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); +}); + +describe("end-to-end — record host accepts credential from external authority", () => { + function makeConfig(): ContrailConfig { + return { + namespace: "test.binding", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + // No `signing` — this deployment is record-host only, accepting + // credentials issued by an external authority. + }, + recordHost: {}, + }, + }; + } + + function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; + } + + it("verifies a credential issued by a separate authority via injected verifier", async () => { + // Set up: the record-host's verifier knows about an external authority + // (did:web:external-authority.example) and where to find its public key. + // No PDS / DID-doc fetches — the verifier is configured directly. + const externalAuthority = "did:web:external-authority.example"; + const externalKey = SIGNING; // simulate operator-provided key material + + const verifier = createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ authorityDid: externalAuthority }), + keys: createLocalKeyResolver({ + authorityDid: externalAuthority, + publicKey: externalKey.publicKey, + }), + }); + + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + const app = createApp(db, resolved, { + spaces: { + authMiddleware: fakeAuth(), + credentialVerifier: verifier, + }, + }); + + // Create a space directly in the local authority's tables. (In a real + // split deployment, the authority would do this; the record host would + // just enroll. Phase 5 introduces enrollment — for now, we use the + // local authority routes as a stand-in.) + const create = await app.fetch( + new Request(`http://localhost/xrpc/test.binding.space.createSpace`, { + method: "POST", + headers: { "X-Test-Did": ALICE, "Content-Type": "application/json" }, + body: "{}", + }) + ); + expect(create.status).toBe(200); + const uri = ((await create.json()) as any).space.uri; + + // External authority signs a credential. + const { credential } = await issueCredential( + { + iss: externalAuthority, + sub: ALICE, + space: uri, + scope: "rw", + ttlMs: 60_000, + }, + externalKey + ); + + // Record host accepts it on putRecord with no service-auth JWT. + const put = await app.fetch( + new Request(`http://localhost/xrpc/test.binding.space.putRecord`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Space-Credential": credential, + }, + body: JSON.stringify({ + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "hello" }, + }), + }) + ); + expect(put.status).toBe(200); + expect(((await put.json()) as any).authorDid).toBe(ALICE); + }); +}); diff --git a/packages/lexicons/lexicon-templates/spaces/declaration.json b/packages/lexicons/lexicon-templates/spaces/declaration.json new file mode 100644 index 0000000..0a5572b --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/declaration.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.declaration", + "description": "PDS-record-as-discovery: when present at `at:////` (i.e. paired with the space URI by NSID + rkey), this record declares which DID is authorized to issue credentials for the space and which DID hosts its records. The space owner's PDS write is the cryptographic proof of authorization — the binding resolver fetches this record to verify a credential's `iss` against the user's wishes, no DID-doc edits required. App-defined space-type lexicons MAY embed these fields directly instead of writing a separate record under this NSID.", + "defs": { + "main": { + "type": "record", + "key": "any", + "record": { + "type": "object", + "required": ["authority", "createdAt"], + "properties": { + "authority": { + "type": "string", + "format": "did", + "description": "DID authorized to sign credentials for this space (`iss` on emitted JWTs). May equal the space owner DID for the self-issuing case." + }, + "recordHost": { + "type": "string", + "format": "did", + "description": "DID of the record host where records for this space live. Clients use this to know where to send writes; verifiers ignore it." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} -- 2.51.2 From dc4d2bcaf15a440bc3a254044ad6f0e0760e491c Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:05:29 +0200 Subject: [PATCH 05/25] phase V --- .../src/core/realtime/publishing-adapter.ts | 4 + packages/contrail/src/core/spaces/adapter.ts | 59 +++ packages/contrail/src/core/spaces/binding.ts | 18 + packages/contrail/src/core/spaces/router.ts | 163 +++++-- packages/contrail/src/core/spaces/schema.ts | 12 + packages/contrail/src/core/spaces/types.ts | 29 +- packages/contrail/src/index.ts | 10 + .../contrail/tests/spaces-enrollment.test.ts | 402 ++++++++++++++++++ .../lexicon-templates/recordHost/enroll.json | 39 ++ 9 files changed, 704 insertions(+), 32 deletions(-) create mode 100644 packages/contrail/tests/spaces-enrollment.test.ts create mode 100644 packages/lexicons/lexicon-templates/recordHost/enroll.json diff --git a/packages/contrail/src/core/realtime/publishing-adapter.ts b/packages/contrail/src/core/realtime/publishing-adapter.ts index 07cbc00..c5dc61f 100644 --- a/packages/contrail/src/core/realtime/publishing-adapter.ts +++ b/packages/contrail/src/core/realtime/publishing-adapter.ts @@ -52,6 +52,10 @@ export function wrapWithPublishing( revokeInvite: inner.revokeInvite.bind(inner), getInvite: inner.getInvite.bind(inner), redeemInvite: inner.redeemInvite.bind(inner), + enroll: inner.enroll.bind(inner), + getEnrollment: inner.getEnrollment.bind(inner), + listEnrollments: inner.listEnrollments.bind(inner), + removeEnrollment: inner.removeEnrollment.bind(inner), getRecord: inner.getRecord.bind(inner), listRecords: inner.listRecords.bind(inner), listCollections: inner.listCollections.bind(inner), diff --git a/packages/contrail/src/core/spaces/adapter.ts b/packages/contrail/src/core/spaces/adapter.ts index b5f6b3c..f80ba5e 100644 --- a/packages/contrail/src/core/spaces/adapter.ts +++ b/packages/contrail/src/core/spaces/adapter.ts @@ -13,6 +13,7 @@ import type { BlobMetaRow, CollectionCount, CreateInviteInput, + EnrollmentRow, InviteKind, InviteRow, ListBlobsOptions, @@ -77,6 +78,15 @@ function mapBlobMetaRow(row: any): BlobMetaRow { }; } +function mapEnrollmentRow(row: any): EnrollmentRow { + return { + spaceUri: row.space_uri, + authorityDid: row.authority_did, + enrolledAt: toNum(row.enrolled_at), + enrolledBy: row.enrolled_by, + }; +} + function mapInviteRow(row: any): InviteRow { return { tokenHash: row.token_hash, @@ -362,6 +372,55 @@ export class HostedAdapter implements StorageAdapter { return row ? mapInviteRow(row) : null; } + async enroll(input: EnrollmentRow): Promise { + await this.db + .prepare( + `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri) DO UPDATE SET + authority_did = excluded.authority_did, + enrolled_at = excluded.enrolled_at, + enrolled_by = excluded.enrolled_by` + ) + .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) + .run(); + } + + async getEnrollment(spaceUri: string): Promise { + const row = await this.db + .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .first(); + return row ? mapEnrollmentRow(row) : null; + } + + async listEnrollments( + options: { authorityDid?: string; limit?: number } = {} + ): Promise { + const limit = Math.min(options.limit ?? 200, 1000); + if (options.authorityDid) { + const { results } = await this.db + .prepare( + `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` + ) + .bind(options.authorityDid, limit) + .all(); + return results.map(mapEnrollmentRow); + } + const { results } = await this.db + .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) + .bind(limit) + .all(); + return results.map(mapEnrollmentRow); + } + + async removeEnrollment(spaceUri: string): Promise { + await this.db + .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .run(); + } + async putRecord(record: StoredRecord): Promise { const table = this.tableFor(record.collection); const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts index d349d55..a81b95f 100644 --- a/packages/contrail/src/core/spaces/binding.ts +++ b/packages/contrail/src/core/spaces/binding.ts @@ -19,6 +19,7 @@ import type { DidDocumentResolver } from "@atcute/identity-resolver"; import type { Did } from "@atcute/lexicons"; import { parseSpaceUri } from "./uri"; +import type { RecordHost } from "./types"; export interface BindingResolver { /** Resolve the DID authorized to sign credentials for this space. Returns @@ -52,6 +53,23 @@ export function createLocalBindingResolver(args: { }; } +/** Reads the record host's local enrollment table. This is the *canonical* + * binding source on a record host: the host owner explicitly consented to a + * given authority for a given space (via the `recordHost.enroll` endpoint + * or auto-enrollment from the authority's createSpace). PDS-record / + * DID-doc resolvers are out-of-band discovery aids; the enrollment is what + * actually gates whether records get stored here. */ +export function createEnrollmentBindingResolver(args: { + recordHost: RecordHost; +}): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const e = await args.recordHost.getEnrollment(spaceUri); + return e?.authorityDid ?? null; + }, + }; +} + /** Returns the space owner DID as the authority. This is the implicit * fallback ("HappyView path") — when no PDS record and no DID-doc service * entry declare an issuer, the owner is taken to be its own. Whether the diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index c0c9e92..dadb7c9 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -12,7 +12,7 @@ import { } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; -import { buildSpaceUri } from "./uri"; +import { buildSpaceUri, parseSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, DEFAULT_CREDENTIAL_TTL_MS, @@ -35,6 +35,8 @@ import { type CredentialVerifier, } from "./credentials"; import { + createCompositeBindingResolver, + createEnrollmentBindingResolver, createLocalBindingResolver, createLocalKeyResolver, } from "./binding"; @@ -90,21 +92,39 @@ export function registerSpacesRoutes( const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); + // When the record host is colocated, hand it to the authority so + // createSpace can auto-enroll. This is the in-process default — + // single-call createSpace gives you a usable space without requiring an + // explicit `recordHost.enroll` afterward. Split deployments don't get + // this convenience; their createSpace caller (or operator) has to call + // enroll on the remote host explicitly. + const localRecordHost = spacesConfig.recordHost ? adapter : null; + registerAuthorityRoutes( + app, + adapter, + authorityConfig, + config, + auth, + options.whoamiExtension, + localRecordHost + ); if (spacesConfig.recordHost) { - // Build the default in-process verifier when the authority can sign: - // Local binding (always points at the configured authority) + Local - // key resolver (knows the authority's public key directly). Caller - // can override via `options.credentialVerifier` to accept external - // authorities — wire in PDS-record / DID-doc resolvers there. + // Default in-process verifier: enrollment is the canonical binding + // source (matches the host's local consent), with Local-binding as a + // fallback for the case where the authority's createSpace ran but + // auto-enroll was bypassed. Local key resolver knows the local + // authority's public key. Caller overrides via + // `options.credentialVerifier` to accept external authorities — wire + // in DID-doc key resolvers there. const verifier = options.credentialVerifier ?? (authorityConfig.signing ? createBindingCredentialVerifier({ - bindings: createLocalBindingResolver({ - authorityDid: authorityConfig.serviceDid, - }), + bindings: createCompositeBindingResolver([ + createEnrollmentBindingResolver({ recordHost: adapter }), + createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), + ]), keys: createLocalKeyResolver({ authorityDid: authorityConfig.serviceDid, publicKey: authorityConfig.signing.publicKey, @@ -116,14 +136,20 @@ export function registerSpacesRoutes( } /** Register the **space authority** XRPC surface — space lifecycle, member - * list, app policy, whoami. Does NOT touch records or blobs. */ + * list, app policy, whoami. Does NOT touch records or blobs. + * + * When a `localRecordHost` is passed, the authority's `createSpace` handler + * also enrolls the new space on that host — convenient for in-process + * deployments where the same operator runs both roles. Split deployments + * pass `null` and arrange enrollment explicitly via `recordHost.enroll`. */ export function registerAuthorityRoutes( app: Hono, authority: SpaceAuthority, authorityConfig: AuthorityConfig, config: ContrailConfig, auth: MiddlewareHandler, - whoamiExtension?: WhoamiExtension + whoamiExtension?: WhoamiExtension, + localRecordHost?: RecordHost | null ): void { /** Space endpoints are emitted per-deployment under the configured namespace; * the deployment owns and publishes its own lexicons. The library ships @@ -243,6 +269,19 @@ export function registerAuthorityRoutes( // Owner is implicit; we still write a row so membership queries are uniform. await authority.addMember(uri, sa.issuer, sa.issuer); + // Auto-enroll the space on the colocated record host (if any). Without + // this, putRecord/listRecords would 404 with "not-enrolled" until the + // caller explicitly hit recordHost.enroll. Idempotent (ON CONFLICT + // UPDATE), so calling enroll again later is harmless. + if (localRecordHost) { + await localRecordHost.enroll({ + spaceUri: uri, + authorityDid: authorityConfig.serviceDid, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + } + return c.json({ space: publicSpaceView(space, true) }); }); @@ -513,20 +552,79 @@ export function registerRecordHostRoutes( return authWithCredential(c, next); }; + /** Hard gate on every record-host operation: the space must be enrolled. + * Returns the enrollment row, or a Response to relay. */ + async function requireEnrollment( + c: Context, + spaceUri: string + ): Promise<{ authorityDid: string } | Response> { + const enrollment = await recordHost.getEnrollment(spaceUri); + if (!enrollment) { + return c.json( + { error: "NotFound", reason: "not-enrolled", message: "space is not enrolled on this record host" }, + 404 + ); + } + return enrollment; + } + + // ---- Enrollment endpoint ---- + // + // Authorize a space onto this record host. Caller must be either the space + // owner OR the declared authority — the host treats either as a sufficient + // signal of consent. Idempotent: re-enrolling the same space updates the + // authority binding. + const RECORD_HOST = `${config.namespace}.recordHost`; + + app.post(`/xrpc/${RECORD_HOST}.enroll`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; authority?: string } + | null; + if (!body?.spaceUri || !body.authority) { + return c.json({ error: "InvalidRequest", message: "spaceUri and authority required" }, 400); + } + const parts = parseSpaceUri(body.spaceUri); + if (!parts) { + return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); + } + const callerIsOwner = sa.issuer === parts.ownerDid; + const callerIsAuthority = sa.issuer === body.authority; + if (!callerIsOwner && !callerIsAuthority) { + return c.json( + { error: "Forbidden", reason: "not-owner-or-authority" }, + 403 + ); + } + await recordHost.enroll({ + spaceUri: body.spaceUri, + authorityDid: body.authority, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + return c.json({ ok: true }); + }); + app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { const spaceUri = c.req.query("spaceUri"); const collection = c.req.query("collection"); if (!spaceUri || !collection) { return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { + // JWT path consults the authority for member + app-policy checks. + // Split deployments where the host has no live authority will degrade + // here (getSpace returns null) and return 404, which is correct: a + // host that can't reach an authority can't validate JWT membership. const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", @@ -558,14 +656,16 @@ export function registerRecordHostRoutes( if (!spaceUri || !collection || !author || !rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", @@ -591,13 +691,16 @@ export function registerRecordHostRoutes( if (!body?.spaceUri || !body.collection || !body.record) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, body.spaceUri, "rw"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + // JWT path needs authority access for member + app-policy checks. + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(body.spaceUri, caller.callerDid); const result = checkAccess({ op: "write", @@ -651,13 +754,15 @@ export function registerRecordHostRoutes( if (!body?.spaceUri || !body.collection || !body.rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, body.spaceUri, "rw"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(body.spaceUri, caller.callerDid); const result = checkAccess({ op: "delete", @@ -689,13 +794,15 @@ export function registerRecordHostRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, spaceUri, "rw"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, caller.callerDid); const aclResult = checkAccess({ op: "write", @@ -764,14 +871,16 @@ export function registerRecordHostRoutes( if (!spaceUri || !cid) { return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "read", @@ -804,13 +913,15 @@ export function registerRecordHostRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, spaceUri, "read"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, caller.callerDid); const aclResult = checkAccess({ op: "read", diff --git a/packages/contrail/src/core/spaces/schema.ts b/packages/contrail/src/core/spaces/schema.ts index a08ada6..ae33a7c 100644 --- a/packages/contrail/src/core/spaces/schema.ts +++ b/packages/contrail/src/core/spaces/schema.ts @@ -59,6 +59,18 @@ export function buildSpacesBaseSchema(dialect: SqlDialect): string[] { note TEXT )`, `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, + + // Record-host enrollment table — the host's local cache of "spaces I + // accept records for, and which authority signs credentials for each." + // Filled by the recordHost.enroll endpoint, or auto-populated by the + // authority's createSpace when both roles run in the same process. + `CREATE TABLE IF NOT EXISTS record_host_enrollments ( + space_uri TEXT PRIMARY KEY, + authority_did TEXT NOT NULL, + enrolled_at ${dialect.bigintType} NOT NULL, + enrolled_by TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, ]; } diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 5429c96..6561773 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -170,6 +170,17 @@ export interface BlobMetaRow { createdAt: number; } +/** Local cache on the record host: this space is accepted here, and `authority_did` + * is the DID authorized to sign credentials for it. Populated via the + * `recordHost.enroll` endpoint or auto-populated by the authority's + * createSpace when both roles run in one process. */ +export interface EnrollmentRow { + spaceUri: string; + authorityDid: string; + enrolledAt: number; + enrolledBy: string; +} + export interface ListBlobsOptions { byUser?: string; cursor?: string; @@ -220,14 +231,20 @@ export interface SpaceAuthority { redeemInvite(tokenHash: string, now: number): Promise; } -/** **Record host** interface — stores records and blobs for a space. Trusts - * the authority's ACL: in later phases, validates incoming credentials - * rather than calling out for a member check. +/** **Record host** interface — stores records and blobs for a space, plus + * the local enrollment table that decides which spaces this host accepts. * - * Read-side note: today the record host needs `getSpace` to know the space - * exists at all (and for app-policy checks before permissioned writes). When - * enrollment lands (phase 5), this becomes a local lookup instead. */ + * Trust model: the host trusts whatever credential the authority signs, so + * long as the authority is the one named in the local enrollment for this + * space. Enrollment is the consent step — the host owner agrees to spend + * storage on a given space, scoped to a specific authority. */ export interface RecordHost { + // Enrollment + enroll(input: EnrollmentRow): Promise; + getEnrollment(spaceUri: string): Promise; + listEnrollments(options?: { authorityDid?: string; limit?: number }): Promise; + removeEnrollment(spaceUri: string): Promise; + // Records putRecord(record: StoredRecord): Promise; getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise; diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 5e59dac..bbd8efe 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -91,6 +91,7 @@ export type { // Binding + key resolution export { createLocalBindingResolver, + createEnrollmentBindingResolver, createOwnerSelfBindingResolver, createCompositeBindingResolver, createPdsBindingResolver, @@ -101,6 +102,15 @@ export { } from "./core/spaces/binding"; export type { BindingResolver, KeyResolver } from "./core/spaces/binding"; +// Route registration — exposed for split deployments where authority and +// record host run as separate Hono apps. Consumers wire them onto bare Honos +// individually instead of going through createApp's umbrella. +export { + registerAuthorityRoutes, + registerRecordHostRoutes, +} from "./core/spaces/router"; +export type { EnrollmentRow } from "./core/spaces/types"; + // Realtime export type { PubSub, diff --git a/packages/contrail/tests/spaces-enrollment.test.ts b/packages/contrail/tests/spaces-enrollment.test.ts new file mode 100644 index 0000000..3b7ef5b --- /dev/null +++ b/packages/contrail/tests/spaces-enrollment.test.ts @@ -0,0 +1,402 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + registerAuthorityRoutes, + registerRecordHostRoutes, +} from "../src/core/spaces/router"; +import { HostedAdapter } from "../src/core/spaces/adapter"; +import { + buildVerifier, + createServiceAuthMiddleware, +} from "../src/core/spaces/auth"; +import { + generateAuthoritySigningKey, + issueCredential, + createBindingCredentialVerifier, +} from "../src/core/spaces/credentials"; +import { + createEnrollmentBindingResolver, + createLocalKeyResolver, +} from "../src/core/spaces/binding"; +import type { CredentialKeyMaterial } from "../src/core/spaces/credentials"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; +} + +function call( + app: Hono, + method: string, + path: string, + did: string | null, + body?: any, + extraHeaders?: Record +) { + const headers: Record = { ...(extraHeaders ?? {}) }; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +// --------------------------------------------------------------------------- +// In-process: createSpace auto-enrolls; non-enrolled spaces 404 +// --------------------------------------------------------------------------- + +describe("auto-enrollment via createSpace", () => { + function makeConfig(): ContrailConfig { + return { + namespace: "test.enroll", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + }, + recordHost: {}, + }, + }; + } + + async function makeApp(): Promise { + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + } + + it("createSpace creates an enrollment row alongside the space row", async () => { + const app = await makeApp(); + const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); + expect(create.status).toBe(200); + const uri = ((await create.json()) as any).space.uri; + + // Subsequent putRecord should succeed because the space is enrolled. + const put = await call(app, "POST", "/xrpc/test.enroll.space.putRecord", ALICE, { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "hi" }, + }); + expect(put.status).toBe(200); + }); + + it("explicit enroll via the endpoint is idempotent", async () => { + const app = await makeApp(); + const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + // Re-enrolling the same space (e.g. to update the authority binding). + const reenroll = await call(app, "POST", "/xrpc/test.enroll.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(reenroll.status).toBe(200); + expect(((await reenroll.json()) as any).ok).toBe(true); + }); + + it("non-owner / non-authority callers cannot enroll", async () => { + const app = await makeApp(); + const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + const res = await call(app, "POST", "/xrpc/test.enroll.recordHost.enroll", BOB, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(res.status).toBe(403); + expect((await res.json()).reason).toBe("not-owner-or-authority"); + }); +}); + +// --------------------------------------------------------------------------- +// Split deployment: authority on machine A, record host on machine B +// --------------------------------------------------------------------------- + +describe("split deployment — authority and record host on separate apps", () => { + function buildAuthorityApp(db: any): Hono { + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + // Authority-only deployment: no recordHost configured. + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + }, + }, + }; + const resolved = resolveConfig(cfg); + const adapter = new HostedAdapter(db, resolved); + const app = new Hono(); + registerAuthorityRoutes( + app, + adapter, + cfg.spaces!.authority!, + resolved, + fakeAuth(), + undefined, + null // no local record host → no auto-enroll + ); + return app; + } + + function buildRecordHostApp(db: any): Hono { + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + // Authority config still required to provide the auth middleware + // and serviceDid context, but signing can be omitted (host doesn't + // sign — it verifies). + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + const adapter = new HostedAdapter(db, resolved); + const verifier = createBindingCredentialVerifier({ + bindings: createEnrollmentBindingResolver({ recordHost: adapter }), + keys: createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }), + }); + const app = new Hono(); + registerRecordHostRoutes( + app, + adapter, + adapter, + cfg.spaces!.recordHost!, + resolved, + fakeAuth(), + verifier + ); + return app; + } + + it("end-to-end: create on authority, enroll on host, write+read via credential", async () => { + // Two physically-separate DBs — proves the host doesn't peek into the + // authority's storage to know about spaces. + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + + // 1. Authority creates a space — auto-enroll did NOT happen because the + // authority has no local record host. + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + expect(create.status).toBe(200); + const uri = ((await create.json()) as any).space.uri; + + // 2. Host rejects writes for the not-yet-enrolled space. + // The credential's binding resolver (EnrollmentBindingResolver) + // finds no enrollment → returns null → verifier 401 "unknown-issuer". + // This is a different layer than the requireEnrollment 404 (which + // fires on JWT/non-credential paths), but both block writes. + const cred1 = await call(authorityApp, "POST", "/xrpc/test.split.space.getCredential", ALICE, { + spaceUri: uri, + }); + const credential = ((await cred1.json()) as any).credential; + const earlyPut = await call( + hostApp, + "POST", + "/xrpc/test.split.space.putRecord", + null, + { spaceUri: uri, collection: "app.event.message", record: { $type: "app.event.message", text: "x" } }, + { "X-Space-Credential": credential } + ); + expect(earlyPut.status).toBe(401); + expect((await earlyPut.json()).reason).toBe("unknown-issuer"); + + // 3. Owner enrolls the space on the host. + const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(enroll.status).toBe(200); + + // 4. Now the host accepts writes via credential. + const put = await call( + hostApp, + "POST", + "/xrpc/test.split.space.putRecord", + null, + { spaceUri: uri, collection: "app.event.message", record: { $type: "app.event.message", text: "hello" } }, + { "X-Space-Credential": credential } + ); + expect(put.status).toBe(200); + expect(((await put.json()) as any).authorDid).toBe(ALICE); + + // 5. Read back via credential. + const list = await call( + hostApp, + "GET", + `/xrpc/test.split.space.listRecords?spaceUri=${encodeURIComponent(uri)}&collection=app.event.message`, + null, + undefined, + { "X-Space-Credential": credential } + ); + expect(list.status).toBe(200); + expect(((await list.json()) as any).records.length).toBe(1); + }); + + it("host rejects credentials whose iss doesn't match the enrollment", async () => { + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); + + // A credential signed by a *different* DID with the same key, but iss + // doesn't match the enrolled authority. + const { credential } = await issueCredential( + { + iss: "did:web:imposter.example", + sub: ALICE, + space: uri, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const res = await call( + hostApp, + "POST", + "/xrpc/test.split.space.putRecord", + null, + { spaceUri: uri, collection: "app.event.message", record: { $type: "app.event.message", text: "x" } }, + { "X-Space-Credential": credential } + ); + expect(res.status).toBe(401); + expect((await res.json()).reason).toBe("unknown-issuer"); + }); + + it("non-enrolled spaces 404 with not-enrolled on JWT-path reads", async () => { + // Service-auth JWTs go through requireEnrollment in the route handler, + // so a non-enrolled space gives the explicit "not-enrolled" reason. + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + const list = await call( + hostApp, + "GET", + `/xrpc/test.split.space.listRecords?spaceUri=${encodeURIComponent(uri)}&collection=app.event.message`, + ALICE + ); + expect(list.status).toBe(404); + expect((await list.json()).reason).toBe("not-enrolled"); + }); + + it("the declared authority can enroll a space without the owner's involvement", async () => { + // Scenario: the authority service ('Contrail' in our naming) is a + // separate DID and acts on behalf of an owner. The authority can enroll + // because phase 5 accepts either owner-or-authority. + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + // Authority service identifies itself via the JWT issuer matching its DID. + const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", SERVICE_DID, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(enroll.status).toBe(200); + }); +}); diff --git a/packages/lexicons/lexicon-templates/recordHost/enroll.json b/packages/lexicons/lexicon-templates/recordHost/enroll.json new file mode 100644 index 0000000..2e7bc66 --- /dev/null +++ b/packages/lexicons/lexicon-templates/recordHost/enroll.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "tools.atmo.recordHost.enroll", + "defs": { + "main": { + "type": "procedure", + "description": "Enroll a space on this record host. The host stores `(spaceUri, authority)` in its local enrollment table; subsequent record / blob operations for this space go through, and the credential verifier uses the enrolled authority as the canonical binding source. Caller must be either the space owner OR the declared authority — the host treats either as a sufficient signal of consent. Idempotent: re-enrolling updates the authority binding.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["spaceUri", "authority"], + "properties": { + "spaceUri": { "type": "string" }, + "authority": { + "type": "string", + "format": "did", + "description": "DID authorized to sign credentials for this space." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["ok"], + "properties": { + "ok": { "type": "boolean" } + } + } + }, + "errors": [ + { "name": "InvalidRequest" }, + { "name": "Forbidden", "description": "Caller is neither the space owner nor the declared authority." } + ] + } + } +} -- 2.51.2 From 81d0f370e61638965f541520a60b1cb23b6321a3 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:30:32 +0200 Subject: [PATCH 06/25] phase seis --- apps/group-chat/package.json | 1 + apps/group-chat/src/lib/contrail/index.ts | 6 +- packages/contrail-community/package.json | 52 +++++++++++ .../src}/acl.ts | 0 .../src}/adapter.ts | 2 +- .../src}/credentials.ts | 0 .../src}/index.ts | 6 ++ .../contrail-community/src/integration.ts | 80 +++++++++++++++++ .../src}/invite-handler.ts | 6 +- .../src}/pds.ts | 0 .../src}/plc.ts | 0 .../src}/reconcile.ts | 2 +- .../src}/router.ts | 14 +-- .../src}/schema.ts | 5 +- .../src}/types.ts | 2 +- .../src}/whoami.ts | 2 +- .../tests/community-delegation.test.ts | 19 ++-- .../tests/community-e2e.test.ts | 19 ++-- .../tests/community-mint.test.ts | 21 +++-- .../tests/community-publishing.test.ts | 19 ++-- .../tests/invite-unified.test.ts | 19 ++-- .../contrail-community/tsconfig.build.json | 7 ++ packages/contrail-community/tsconfig.json | 7 ++ packages/contrail-community/tsup.config.ts | 11 +++ packages/contrail-community/vitest.config.ts | 17 ++++ packages/contrail/package.json | 1 + packages/contrail/src/contrail.ts | 27 +++++- .../src/core/community-integration.ts | 54 ++++++++++++ packages/contrail/src/core/db/schema.ts | 21 +++-- .../contrail/src/core/realtime/resolve.ts | 7 +- packages/contrail/src/core/realtime/router.ts | 4 +- .../contrail/src/core/router/collection.ts | 4 +- packages/contrail/src/core/router/index.ts | 86 +++++++++---------- packages/contrail/src/core/types.ts | 8 +- packages/contrail/src/index.ts | 56 ++++++------ packages/contrail/tests/realtime-e2e.test.ts | 5 +- packages/contrail/vitest.config.ts | 12 +++ pnpm-lock.yaml | 43 ++++++++++ 38 files changed, 502 insertions(+), 143 deletions(-) create mode 100644 packages/contrail-community/package.json rename packages/{contrail/src/core/community => contrail-community/src}/acl.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/adapter.ts (99%) rename packages/{contrail/src/core/community => contrail-community/src}/credentials.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/index.ts (77%) create mode 100644 packages/contrail-community/src/integration.ts rename packages/{contrail/src/core/community => contrail-community/src}/invite-handler.ts (97%) rename packages/{contrail/src/core/community => contrail-community/src}/pds.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/plc.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/reconcile.ts (96%) rename packages/{contrail/src/core/community => contrail-community/src}/router.ts (99%) rename packages/{contrail/src/core/community => contrail-community/src}/schema.ts (93%) rename packages/{contrail/src/core/community => contrail-community/src}/types.ts (98%) rename packages/{contrail/src/core/community => contrail-community/src}/whoami.ts (94%) rename packages/{contrail => contrail-community}/tests/community-delegation.test.ts (94%) rename packages/{contrail => contrail-community}/tests/community-e2e.test.ts (93%) rename packages/{contrail => contrail-community}/tests/community-mint.test.ts (90%) rename packages/{contrail => contrail-community}/tests/community-publishing.test.ts (93%) rename packages/{contrail => contrail-community}/tests/invite-unified.test.ts (92%) create mode 100644 packages/contrail-community/tsconfig.build.json create mode 100644 packages/contrail-community/tsconfig.json create mode 100644 packages/contrail-community/tsup.config.ts create mode 100644 packages/contrail-community/vitest.config.ts create mode 100644 packages/contrail/src/core/community-integration.ts diff --git a/apps/group-chat/package.json b/apps/group-chat/package.json index 54b950a..a773f5f 100644 --- a/apps/group-chat/package.json +++ b/apps/group-chat/package.json @@ -60,6 +60,7 @@ "dependencies": { "@atcute/jetstream": "^1.1.2", "@atmo-dev/contrail": "workspace:*", + "@atmo-dev/contrail-community": "workspace:*", "@atmo-dev/contrail-sync": "workspace:*", "@atmo-dev/contrail-lexicons": "workspace:*", "@foxui/core": "^0.9.1", diff --git a/apps/group-chat/src/lib/contrail/index.ts b/apps/group-chat/src/lib/contrail/index.ts index 85a6aa3..8baf918 100644 --- a/apps/group-chat/src/lib/contrail/index.ts +++ b/apps/group-chat/src/lib/contrail/index.ts @@ -4,12 +4,14 @@ import { InMemoryPubSub, MemoryBlobAdapter, R2BlobAdapter, + resolveConfig, type BlobAdapter, type ContrailConfig, type DurableObjectNamespace, type PubSub, type R2BucketLike } from '@atmo-dev/contrail'; +import { createCommunityIntegration } from '@atmo-dev/contrail-community'; import { createHandler, createServerClient } from '@atmo-dev/contrail/server'; import type { Client } from '@atcute/client'; import { dev } from '$app/environment'; @@ -65,7 +67,9 @@ function build(env: Env): Bundle { } }; - const contrail = new Contrail(config); + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ db: env.DB, config: resolved }); + const contrail = new Contrail({ ...config, db: env.DB, communityIntegration }); const handle = createHandler(contrail); const ready = contrail.init(env.DB); diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json new file mode 100644 index 0000000..2fae478 --- /dev/null +++ b/packages/contrail-community/package.json @@ -0,0 +1,52 @@ +{ + "name": "@atmo-dev/contrail-community", + "version": "0.1.0", + "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-community" + }, + "keywords": [ + "atproto", + "contrail", + "community" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@atcute/atproto": "^3.1.10", + "@atcute/cbor": "^2.3.2", + "@atcute/identity": "^1.1.4", + "@atcute/identity-resolver": "^1.2.2", + "@atcute/lexicons": "^1.2.9", + "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "tsup": "^8.5.0", + "typescript": "^5.7.3", + "vitest": "^4.1.0" + }, + "license": "MIT" +} diff --git a/packages/contrail/src/core/community/acl.ts b/packages/contrail-community/src/acl.ts similarity index 100% rename from packages/contrail/src/core/community/acl.ts rename to packages/contrail-community/src/acl.ts diff --git a/packages/contrail/src/core/community/adapter.ts b/packages/contrail-community/src/adapter.ts similarity index 99% rename from packages/contrail/src/core/community/adapter.ts rename to packages/contrail-community/src/adapter.ts index 1587e63..586fa88 100644 --- a/packages/contrail/src/core/community/adapter.ts +++ b/packages/contrail-community/src/adapter.ts @@ -1,4 +1,4 @@ -import type { Database } from "../types"; +import type { Database } from "@atmo-dev/contrail"; import type { AccessLevel, AccessLevelRow, diff --git a/packages/contrail/src/core/community/credentials.ts b/packages/contrail-community/src/credentials.ts similarity index 100% rename from packages/contrail/src/core/community/credentials.ts rename to packages/contrail-community/src/credentials.ts diff --git a/packages/contrail/src/core/community/index.ts b/packages/contrail-community/src/index.ts similarity index 77% rename from packages/contrail/src/core/community/index.ts rename to packages/contrail-community/src/index.ts index fd0cc53..d349351 100644 --- a/packages/contrail/src/core/community/index.ts +++ b/packages/contrail-community/src/index.ts @@ -35,3 +35,9 @@ export { jwkToDidKey, } from "./plc"; export type { KeyPair, GenesisOpInput, UnsignedGenesisOp, SignedGenesisOp } from "./plc"; + +// The headline export — wire community into a contrail app via: +// const community = createCommunityIntegration({ db, config }); +// const app = createApp(db, config, { community }); +export { createCommunityIntegration } from "./integration"; +export type { CommunityIntegrationOptions } from "./integration"; diff --git a/packages/contrail-community/src/integration.ts b/packages/contrail-community/src/integration.ts new file mode 100644 index 0000000..9dca172 --- /dev/null +++ b/packages/contrail-community/src/integration.ts @@ -0,0 +1,80 @@ +/** Factory that builds a {@link CommunityIntegration} for contrail's + * `createApp({ community })` option. The integration is opaque to contrail + * core — it just exposes the hooks the umbrella router needs. */ + +import type { + CommunityIntegration, + CommunityProbe, + ContrailConfig, + Database, + StorageAdapter, +} from "@atmo-dev/contrail"; +import { HostedAdapter, buildVerifier, createServiceAuthMiddleware } from "@atmo-dev/contrail"; +import { CommunityAdapter } from "./adapter"; +import { createCommunityWhoamiExtension } from "./whoami"; +import { createCommunityInviteHandler } from "./invite-handler"; +import { resolveReachableSpaces } from "./acl"; +import { registerCommunityRoutes } from "./router"; +import { buildCommunitySchema } from "./schema"; +import { getDialect } from "@atmo-dev/contrail"; + +export interface CommunityIntegrationOptions { + /** Database the community tables live in. Should be the same DB the + * spaces module uses (community rows reference space_uri). */ + db: Database; + /** Resolved contrail config. */ + config: ContrailConfig; + /** Optional override of the community adapter (for tests). */ + communityAdapter?: CommunityAdapter; + /** Optional override of the spaces adapter (for tests). Otherwise built + * from the same db using HostedAdapter. */ + spacesAdapter?: StorageAdapter; +} + +export function createCommunityIntegration( + options: CommunityIntegrationOptions +): CommunityIntegration { + const { db, config } = options; + const community = options.communityAdapter ?? new CommunityAdapter(db); + const spaces = options.spacesAdapter ?? new HostedAdapter(db, config); + + const probe: CommunityProbe = { + async getCommunity(did) { + return community.getCommunity(did); + }, + async resolveReachableSpaces(callerDid) { + return resolveReachableSpaces(community, callerDid); + }, + }; + + const whoamiExtension = createCommunityWhoamiExtension({ community }); + const inviteHandler = createCommunityInviteHandler({ + community, + authority: spaces, + }); + + return { + probe, + whoamiExtension, + inviteHandler, + registerRoutes(app, opts) { + // Reuse the spaces JWT verifier — the auth model is identical. + if (!config.spaces?.authority) return; + const verifier = buildVerifier(config.spaces.authority); + const authMiddleware = + opts?.authMiddleware ?? createServiceAuthMiddleware(verifier); + registerCommunityRoutes( + app, + db, + config, + { authMiddleware, communityAdapter: community, spacesAdapter: spaces }, + { spacesAdapter: spaces, verifier } + ); + }, + async applySchema(target) { + const dialect = getDialect(target); + const stmts = buildCommunitySchema(dialect); + await target.batch(stmts.map((s) => target.prepare(s))); + }, + }; +} diff --git a/packages/contrail/src/core/community/invite-handler.ts b/packages/contrail-community/src/invite-handler.ts similarity index 97% rename from packages/contrail/src/core/community/invite-handler.ts rename to packages/contrail-community/src/invite-handler.ts index 0b41f8a..9f27f1d 100644 --- a/packages/contrail/src/core/community/invite-handler.ts +++ b/packages/contrail-community/src/invite-handler.ts @@ -5,10 +5,10 @@ import type { CommunityInviteHandler, HandlerResponse, -} from "../invite/community-handler"; -import { mintInviteToken } from "../invite/token"; + SpaceAuthority, +} from "@atmo-dev/contrail"; +import { mintInviteToken } from "@atmo-dev/contrail"; import type { CommunityAdapter } from "./adapter"; -import type { SpaceAuthority } from "../spaces/types"; import { resolveEffectiveLevel } from "./acl"; import { reconcile } from "./reconcile"; import type { AccessLevel, CommunityInviteRow } from "./types"; diff --git a/packages/contrail/src/core/community/pds.ts b/packages/contrail-community/src/pds.ts similarity index 100% rename from packages/contrail/src/core/community/pds.ts rename to packages/contrail-community/src/pds.ts diff --git a/packages/contrail/src/core/community/plc.ts b/packages/contrail-community/src/plc.ts similarity index 100% rename from packages/contrail/src/core/community/plc.ts rename to packages/contrail-community/src/plc.ts diff --git a/packages/contrail/src/core/community/reconcile.ts b/packages/contrail-community/src/reconcile.ts similarity index 96% rename from packages/contrail/src/core/community/reconcile.ts rename to packages/contrail-community/src/reconcile.ts index 5fde1d3..1b0bbb0 100644 --- a/packages/contrail/src/core/community/reconcile.ts +++ b/packages/contrail-community/src/reconcile.ts @@ -1,4 +1,4 @@ -import type { SpaceAuthority } from "../spaces/types"; +import type { SpaceAuthority } from "@atmo-dev/contrail"; import type { CommunityAdapter } from "./adapter"; import { flattenEffectiveMembers } from "./acl"; diff --git a/packages/contrail/src/core/community/router.ts b/packages/contrail-community/src/router.ts similarity index 99% rename from packages/contrail/src/core/community/router.ts rename to packages/contrail-community/src/router.ts index 84e86d1..23c3123 100644 --- a/packages/contrail/src/core/community/router.ts +++ b/packages/contrail-community/src/router.ts @@ -1,9 +1,11 @@ import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig, Database } from "../types"; -import type { ServiceAuth } from "../spaces/auth"; -import type { StorageAdapter as SpacesAdapter } from "../spaces/types"; -import { buildSpaceUri } from "../spaces/uri"; -import { HostedAdapter } from "../spaces/adapter"; +import type { + ContrailConfig, + Database, + ServiceAuth, + StorageAdapter as SpacesAdapter, +} from "@atmo-dev/contrail"; +import { buildSpaceUri, HostedAdapter } from "@atmo-dev/contrail"; import { CommunityAdapter } from "./adapter"; import { CredentialCipher } from "./credentials"; import { resolveIdentity, createPdsSession } from "./pds"; @@ -43,7 +45,7 @@ export function registerCommunityRoutes( verifier: ServiceJwtVerifier; } | null ): void { - const cfg = config.community; + const cfg = config.community as import("./types").CommunityConfig | undefined; if (!cfg) return; if (!config.spaces?.authority) { throw new Error("community module requires spaces.authority to be enabled in config"); diff --git a/packages/contrail/src/core/community/schema.ts b/packages/contrail-community/src/schema.ts similarity index 93% rename from packages/contrail/src/core/community/schema.ts rename to packages/contrail-community/src/schema.ts index b7d382c..cdcd56e 100644 --- a/packages/contrail/src/core/community/schema.ts +++ b/packages/contrail-community/src/schema.ts @@ -1,6 +1,5 @@ -import type { Database } from "../types"; -import { getDialect } from "../dialect"; -import type { SqlDialect } from "../dialect"; +import type { Database, SqlDialect } from "@atmo-dev/contrail"; +import { getDialect } from "@atmo-dev/contrail"; export function buildCommunitySchema(dialect: SqlDialect): string[] { return [ diff --git a/packages/contrail/src/core/community/types.ts b/packages/contrail-community/src/types.ts similarity index 98% rename from packages/contrail/src/core/community/types.ts rename to packages/contrail-community/src/types.ts index 7959e72..7f74698 100644 --- a/packages/contrail/src/core/community/types.ts +++ b/packages/contrail-community/src/types.ts @@ -1,4 +1,4 @@ -import type { Database } from "../types"; +import type { Database } from "@atmo-dev/contrail"; import type { DidDocumentResolver } from "@atcute/identity-resolver"; /** Access level a subject (did or group-space) has on a given space. diff --git a/packages/contrail/src/core/community/whoami.ts b/packages/contrail-community/src/whoami.ts similarity index 94% rename from packages/contrail/src/core/community/whoami.ts rename to packages/contrail-community/src/whoami.ts index 4a75a34..5f06123 100644 --- a/packages/contrail/src/core/community/whoami.ts +++ b/packages/contrail-community/src/whoami.ts @@ -2,7 +2,7 @@ * for community-owned spaces. Returns null for non-community spaces so the * spaces module's default binary-membership logic runs. */ -import type { WhoamiExtension } from "../spaces/router"; +import type { WhoamiExtension } from "@atmo-dev/contrail"; import type { CommunityAdapter } from "./adapter"; import { resolveEffectiveLevel } from "./acl"; diff --git a/packages/contrail/tests/community-delegation.test.ts b/packages/contrail-community/tests/community-delegation.test.ts similarity index 94% rename from packages/contrail/tests/community-delegation.test.ts rename to packages/contrail-community/tests/community-delegation.test.ts index 1514964..5449e7c 100644 --- a/packages/contrail/tests/community-delegation.test.ts +++ b/packages/contrail-community/tests/community-delegation.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll, beforeEach } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -79,8 +80,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/community-e2e.test.ts b/packages/contrail-community/tests/community-e2e.test.ts similarity index 93% rename from packages/contrail/tests/community-e2e.test.ts rename to packages/contrail-community/tests/community-e2e.test.ts index db008e6..95d6048 100644 --- a/packages/contrail/tests/community-e2e.test.ts +++ b/packages/contrail-community/tests/community-e2e.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -90,8 +91,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/community-mint.test.ts b/packages/contrail-community/tests/community-mint.test.ts similarity index 90% rename from packages/contrail/tests/community-mint.test.ts rename to packages/contrail-community/tests/community-mint.test.ts index c83491f..02ea73b 100644 --- a/packages/contrail/tests/community-mint.test.ts +++ b/packages/contrail-community/tests/community-mint.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; import { buildGenesisOp, computeDidPlc, @@ -13,7 +14,7 @@ import { generateKeyPair, jwkToDidKey, signGenesisOp, -} from "../src/core/community/plc"; +} from "../src/plc"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -63,8 +64,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/community-publishing.test.ts b/packages/contrail-community/tests/community-publishing.test.ts similarity index 93% rename from packages/contrail/tests/community-publishing.test.ts rename to packages/contrail-community/tests/community-publishing.test.ts index 10a3d76..a05ee0f 100644 --- a/packages/contrail/tests/community-publishing.test.ts +++ b/packages/contrail-community/tests/community-publishing.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -93,8 +94,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/invite-unified.test.ts b/packages/contrail-community/tests/invite-unified.test.ts similarity index 92% rename from packages/contrail/tests/invite-unified.test.ts rename to packages/contrail-community/tests/invite-unified.test.ts index b56d81d..32bd6c5 100644 --- a/packages/contrail/tests/invite-unified.test.ts +++ b/packages/contrail-community/tests/invite-unified.test.ts @@ -7,11 +7,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -66,8 +67,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call(app: Hono, method: string, path: string, did: string, body?: any): Promise { diff --git a/packages/contrail-community/tsconfig.build.json b/packages/contrail-community/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-community/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-community/tsconfig.json b/packages/contrail-community/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-community/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-community/tsup.config.ts b/packages/contrail-community/tsup.config.ts new file mode 100644 index 0000000..b033415 --- /dev/null +++ b/packages/contrail-community/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["@atmo-dev/contrail"], +}); diff --git a/packages/contrail-community/vitest.config.ts b/packages/contrail-community/vitest.config.ts new file mode 100644 index 0000000..56720e0 --- /dev/null +++ b/packages/contrail-community/vitest.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "vitest/config"; +import path from "node:path"; + +const contrailSrc = path.resolve(__dirname, "../contrail/src"); + +// Alias `@atmo-dev/contrail` and its subpaths to the source so tests don't +// run through the built dist. Mirrors the in-tree-source-resolution pattern +// the contrail package's own tests use (they import via ../src/...). +export default defineConfig({ + resolve: { + alias: { + "@atmo-dev/contrail/sqlite": path.join(contrailSrc, "adapters/sqlite.ts"), + "@atmo-dev/contrail/postgres": path.join(contrailSrc, "adapters/postgres.ts"), + "@atmo-dev/contrail": path.join(contrailSrc, "index.ts"), + }, + }, +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index a64484d..846e6e8 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -78,6 +78,7 @@ "jiti": "^2.4.0" }, "devDependencies": { + "@atmo-dev/contrail-community": "workspace:*", "@cloudflare/workers-types": "^4.20250124.0", "@types/node": "^25.5.0", "@types/pg": "^8.20.0", diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index 7c52aa0..dcefac7 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -21,27 +21,43 @@ import { import type { PubSub } from "./core/realtime/types"; import { InMemoryPubSub } from "./core/realtime/in-memory"; import { createApp, type CreateAppOptions } from "./core/router"; +import type { CommunityIntegration } from "./core/community-integration"; import type { Hono } from "hono"; -export interface ContrailOptions extends ContrailConfig { +/** Note: `community` is shadowed from ContrailConfig (where it's the + * user-supplied config blob, typed as `unknown`) to be the pre-built + * integration object the Contrail instance actually consumes. */ +export interface ContrailOptions extends Omit { db?: Database; /** Optional separate DB for permissioned spaces tables. Defaults to `db`. */ spacesDb?: Database; + /** Optional user-supplied community config blob. Same shape as + * ContrailConfig.community — the community integration reads this + * via `config.community`. */ + community?: unknown; + /** Optional pre-built community integration. When set, the Contrail + * instance applies its schema during `init()` and forwards it to + * `createApp` so community routes / hooks are wired automatically. + * Construct via `createCommunityIntegration(...)` from + * `@atmo-dev/contrail-community`. */ + communityIntegration?: CommunityIntegration; } export class Contrail { readonly config: ResolvedContrailConfig; private _db?: Database; private _spacesDb?: Database; + private _community?: CommunityIntegration; private _ingestState: IngestState = createIngestState(); private _pubsub: PubSub | null = null; constructor(options: ContrailOptions) { - const { db, spacesDb, ...configInput } = options; - this.config = resolveConfig(configInput); + const { db, spacesDb, communityIntegration, ...configInput } = options; + this.config = resolveConfig(configInput as ContrailConfig); validateConfig(this.config); this._db = db; this._spacesDb = spacesDb; + this._community = communityIntegration; // Build the pubsub instance up-front so ingestion and HTTP routes share // it. Caller overrides via `config.realtime.pubsub` (e.g. DurableObject). if (this.config.realtime) { @@ -72,7 +88,8 @@ export class Contrail { async init(db?: Database, spacesDb?: Database): Promise { const main = this.getDb(db); const spaces = spacesDb ?? this._spacesDb; - await initSchema(main, this.config, { spacesDb: spaces }); + const extraSchemas = this._community ? [this._community.applySchema] : []; + await initSchema(main, this.config, { spacesDb: spaces, extraSchemas }); } /** Query records from a collection. */ @@ -273,6 +290,8 @@ export class Contrail { return createApp(main, this.config, { ...appOpts, spacesDb: spaces, + // Per-call community override falls back to the constructor's. + community: appOpts.community ?? this._community ?? null, realtime: { ...appOpts.realtime, pubsub: this._pubsub ?? undefined }, }); } diff --git a/packages/contrail/src/core/community-integration.ts b/packages/contrail/src/core/community-integration.ts new file mode 100644 index 0000000..d91b3a4 --- /dev/null +++ b/packages/contrail/src/core/community-integration.ts @@ -0,0 +1,54 @@ +/** Pluggable integration surface for the community module. + * + * Phase 6 extracted community to its own package (`@atmo-dev/contrail-community`). + * The contrail core package never imports from it — couplings only flow + * through these interfaces. The community package's + * `createCommunityIntegration({ ... })` returns a {@link CommunityIntegration}, + * which the consumer hands to `createApp` via `options.community`. + * + * Two layers: + * - {@link CommunityProbe}: minimal "is this a community DID" / "what + * spaces does this caller reach" surface used by realtime + collection + * routes for community-aware dispatch. + * - {@link CommunityIntegration}: the umbrella bundle — probe, whoami + * extension, invite handler, plus route + schema wiring that the + * umbrella router calls during setup. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { Database } from "./types"; +import type { CommunityInviteHandler } from "./invite/community-handler"; +import type { WhoamiExtension } from "./spaces/router"; + +/** Narrow interface for the deep callers (realtime/resolve, router/collection) + * that just need to ask "is this a community DID?" or "what spaces does this + * caller reach via community membership?" */ +export interface CommunityProbe { + /** Look up a community row by DID. Returns null for non-community DIDs. + * Callers usually only check truthiness — community-specific fields stay + * inside the community package. */ + getCommunity(did: string): Promise<{ did: string } | null>; + + /** Resolve the set of space URIs reachable by `callerDid` through community + * membership (direct grants + delegations). Used by realtime to expand + * community: topics into the caller's concrete space: topics. */ + resolveReachableSpaces(callerDid: string): Promise>; +} + +/** Umbrella integration the consumer constructs once and hands to createApp. + * contrail core treats this as an opaque bundle — it doesn't introspect + * community state, just calls these methods at the right wiring points. */ +export interface CommunityIntegration { + /** Probe used by realtime + collection cross-cutting concerns. */ + probe: CommunityProbe; + /** Whoami extension that returns `accessLevel` for community-owned spaces. */ + whoamiExtension: WhoamiExtension; + /** Handler for the community-grant path of the unified invite surface. */ + inviteHandler: CommunityInviteHandler; + /** Register `.community.*` routes onto the Hono app. */ + registerRoutes( + app: Hono, + options?: { authMiddleware?: MiddlewareHandler } + ): void; + /** Apply community schema (DDL) to the database. Called by initSchema. */ + applySchema(db: Database): Promise; +} diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 5ef78e8..562a6ee 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -11,7 +11,6 @@ import { } from "../types"; import { getSearchableFields } from "../search"; import { buildSpacesBaseSchema } from "../spaces/schema"; -import { buildCommunitySchema } from "../community/schema"; import { buildLabelsSchema } from "../labels/schema"; function getResolved(config: ContrailConfig): ResolvedMaps { @@ -262,9 +261,17 @@ async function runMigrations(db: Database): Promise { } } +/** Pluggable schema applier — passed in by extension packages (community, + * third-party plugins) to install their own tables alongside contrail's. */ +export type SchemaModule = (db: Database) => Promise; + export interface InitSchemaOptions { /** Separate DB for the spaces tables. Defaults to the main `db`. */ spacesDb?: Database; + /** Extra schema modules to apply after contrail's own DDL. Used by the + * community package to install its tables — contrail core no longer + * imports community schema directly. */ + extraSchemas?: SchemaModule[]; } async function applySpacesSchema( @@ -311,11 +318,13 @@ export async function initSchema( await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); } - if (config.community) { - // Community tables live on the same DB as spaces (they reference space_uri). - const target = spacesSharesMainDb ? db : spacesDb!; - const communityStmts = buildCommunitySchema(dialect); - await target.batch(communityStmts.map((s) => target.prepare(s))); + // Extension schemas (e.g. community) — applied to the spacesDb when one's + // configured separately, since they typically reference space_uri. The + // caller is responsible for routing the schema to the right db; we just + // hand it the spaces-or-main DB as a sensible default. + const extensionTarget = spacesSharesMainDb ? db : spacesDb!; + for (const apply of options.extraSchemas ?? []) { + await apply(extensionTarget); } if (config.labels) { diff --git a/packages/contrail/src/core/realtime/resolve.ts b/packages/contrail/src/core/realtime/resolve.ts index f8a0ab4..c16abe5 100644 --- a/packages/contrail/src/core/realtime/resolve.ts +++ b/packages/contrail/src/core/realtime/resolve.ts @@ -12,8 +12,7 @@ * (not yet implemented). */ import type { StorageAdapter } from "../spaces/types"; -import type { CommunityAdapter } from "../community/adapter"; -import { resolveReachableSpaces } from "../community/acl"; +import type { CommunityProbe } from "../community-integration"; import { spaceTopic, parseCommunityTopic, parseSpaceTopic } from "./types"; export interface TopicResolutionContext { @@ -22,7 +21,7 @@ export interface TopicResolutionContext { * (`collection:`, `actor:`) still resolve. */ spaces: StorageAdapter | null; /** May be null if the community module is not enabled. */ - community: CommunityAdapter | null; + community: CommunityProbe | null; } export interface TopicResolution { @@ -63,7 +62,7 @@ export async function resolveTopicForCaller( } const row = await ctx.community.getCommunity(communityDid); if (!row) return { ok: false, error: "NotFound", reason: "community-not-found" }; - const reachable = await resolveReachableSpaces(ctx.community, callerDid); + const reachable = await ctx.community.resolveReachableSpaces(callerDid); // Filter to spaces owned by THIS community — reachable may include spaces // from other communities via cross-community delegation. const ownedList = await ctx.spaces.listSpaces({ ownerDid: communityDid, limit: 1000 }); diff --git a/packages/contrail/src/core/realtime/router.ts b/packages/contrail/src/core/realtime/router.ts index 2726226..d10e45e 100644 --- a/packages/contrail/src/core/realtime/router.ts +++ b/packages/contrail/src/core/realtime/router.ts @@ -4,7 +4,7 @@ import type { Context, Hono, MiddlewareHandler } from "hono"; import type { ContrailConfig } from "../types"; import type { ServiceAuth } from "../spaces/auth"; import type { StorageAdapter } from "../spaces/types"; -import type { CommunityAdapter } from "../community/adapter"; +import type { CommunityProbe } from "../community-integration"; import { InMemoryPubSub } from "./in-memory"; import { TicketSigner } from "./ticket"; import { sseResponse } from "./sse"; @@ -40,7 +40,7 @@ export function registerRealtimeRoutes( app: Hono, config: ContrailConfig, spaces: StorageAdapter | null, - community: CommunityAdapter | null, + community: CommunityProbe | null, options: RealtimeRoutesOptions ): void { const cfg = config.realtime; diff --git a/packages/contrail/src/core/router/collection.ts b/packages/contrail/src/core/router/collection.ts index e440522..6d43d00 100644 --- a/packages/contrail/src/core/router/collection.ts +++ b/packages/contrail/src/core/router/collection.ts @@ -31,7 +31,7 @@ import { DurableObjectPubSub } from "../realtime/durable-object"; import { TicketSigner, type TicketQuerySpec } from "../realtime/ticket"; import { resolveTopicForCaller } from "../realtime/resolve"; import { mergeAsyncIterables } from "../realtime/merge"; -import type { CommunityAdapter } from "../community/adapter"; +import type { CommunityProbe } from "../community-integration"; import { getRelationField, getNestedValue } from "../types"; /** Scope of a watch stream. @@ -497,7 +497,7 @@ export function registerCollectionRoutes( spacesCtx?: SpacesContext | null, options: { pubsub?: import("../realtime/types").PubSub | null; - community?: CommunityAdapter | null; + community?: CommunityProbe | null; } = {} ): void { const ns = config.namespace; diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 535c15f..0c5e058 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -12,11 +12,7 @@ import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; import { HostedAdapter } from "../spaces/adapter"; import type { StorageAdapter } from "../spaces/types"; import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; -import { registerCommunityRoutes } from "../community/router"; -import type { CommunityRoutesOptions } from "../community/router"; -import { CommunityAdapter } from "../community/adapter"; -import { createCommunityInviteHandler } from "../community/invite-handler"; -import { createCommunityWhoamiExtension } from "../community/whoami"; +import type { CommunityIntegration } from "../community-integration"; import { registerRealtimeRoutes } from "../realtime/router"; import type { RealtimeRoutesOptions } from "../realtime/router"; import { registerInviteRoutes } from "../invite/router"; @@ -28,6 +24,7 @@ import { resolveProfiles } from "./profiles"; import { backfillUser } from "../backfill"; import { selectAcceptedLabelers } from "../labels/select"; import { hydrateLabels } from "../labels/hydrate"; +import type { MiddlewareHandler } from "hono"; export interface SpacesContext { adapter: StorageAdapter; @@ -36,7 +33,13 @@ export interface SpacesContext { export interface CreateAppOptions { spaces?: SpacesRoutesOptions; - community?: CommunityRoutesOptions; + /** Pre-built community integration. Construct via the community package's + * `createCommunityIntegration({ ... })`. When set, contrail wires + * community whoami extension, invite handler, route registration, etc. + * When omitted, deployment runs without community features. */ + community?: CommunityIntegration | null; + /** Auth middleware override for community routes (rare — mostly for tests). */ + communityAuthMiddleware?: MiddlewareHandler; realtime?: Partial; /** Separate DB for the spaces tables. Defaults to `db`. */ spacesDb?: Database; @@ -131,11 +134,11 @@ export function createApp( } : null; - // Community is wired up at this layer, not from inside spaces / invite — - // those modules consume injected hooks, not community internals. The - // adapter is shared across every call site that needs it (publishing - // wrapper, collection routes, whoami extension, invite handler, realtime). - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; + // Community is provided as a pre-built integration — contrail core never + // imports from the community package. The integration object is opaque; + // we just pass through its probe / whoamiExtension / inviteHandler / + // registerRoutes hooks at the right wiring points. + const community = options.community ?? null; // Realtime pubsub is built whenever realtime is configured — independent of // spaces. With spaces, the spaces adapter is wrapped so private record/member @@ -149,8 +152,8 @@ export function createApp( queueBound: config.realtime.queueBound, }); if (spacesCtx) { - const isCommunityDid = communityAdapter - ? cachedIsCommunityDid(communityAdapter) + const isCommunityDid = community + ? cachedIsCommunityDid(community.probe) : undefined; spacesCtx = { ...spacesCtx, @@ -163,52 +166,43 @@ export function createApp( registerCollectionRoutes(app, db, config, spacesCtx, { pubsub: realtimePubsub, - community: communityAdapter, + community: community?.probe ?? null, }); registerFeedRoutes(app, db, config); registerNotifyRoute(app, db, config); - // Spaces routes — get a whoami extension when community is configured so - // community-owned spaces get an `accessLevel` field. + // Spaces routes — get a whoami extension from the community integration + // when one's wired so community-owned spaces get an `accessLevel` field. const spacesOptions = { ...options.spaces, whoamiExtension: - options.spaces?.whoamiExtension ?? - (communityAdapter - ? createCommunityWhoamiExtension({ community: communityAdapter }) - : undefined), + options.spaces?.whoamiExtension ?? community?.whoamiExtension, }; registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); - if (config.community && spacesCtx) { + if (community && spacesCtx) { // Community routes reuse the spaces service-auth middleware (same JWT verifier). const authMiddleware = - options.community?.authMiddleware ?? + options.communityAuthMiddleware ?? options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier); - registerCommunityRoutes( - app, - spacesDb, - config, - { ...options.community, authMiddleware }, - { spacesAdapter: spacesCtx.adapter, verifier: spacesCtx.verifier } - ); + community.registerRoutes(app, { authMiddleware }); } if (config.spaces?.authority && spacesCtx) { // Unified invite surface: one `.invite.*` family that dispatches on // space ownership (user-owned → addMember; community-owned → grant via - // an injected community-invite handler). + // the integration's invite handler). const authMiddleware = options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier); - const inviteHandler = communityAdapter - ? createCommunityInviteHandler({ - community: communityAdapter, - authority: spacesCtx.adapter, - }) - : null; - registerInviteRoutes(app, config, spacesCtx.adapter, inviteHandler, { authMiddleware }); + registerInviteRoutes( + app, + config, + spacesCtx.adapter, + community?.inviteHandler ?? null, + { authMiddleware } + ); } if (config.realtime && realtimePubsub) { @@ -221,17 +215,23 @@ export function createApp( options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier) : null; - registerRealtimeRoutes(app, config, spacesCtx?.adapter ?? null, communityAdapter, { - authMiddleware, - pubsub: realtimePubsub, - }); + registerRealtimeRoutes( + app, + config, + spacesCtx?.adapter ?? null, + community?.probe ?? null, + { + authMiddleware, + pubsub: realtimePubsub, + } + ); } return app; } function cachedIsCommunityDid( - community: CommunityAdapter + probe: import("../community-integration").CommunityProbe ): (did: string) => Promise { const TTL = 60_000; const cache = new Map(); @@ -239,7 +239,7 @@ function cachedIsCommunityDid( const now = Date.now(); const hit = cache.get(did); if (hit && hit.expires > now) return hit.value; - const row = await community.getCommunity(did); + const row = await probe.getCommunity(did); const value = row != null; cache.set(did, { value, expires: now + TTL }); return value; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index b895141..3b9dc48 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -155,9 +155,11 @@ export interface ContrailConfig { notify?: boolean | string; /** Permissioned spaces configuration. When set, the service exposes space XRPCs. */ spaces?: import("./spaces/types").SpacesConfig; - /** Community module configuration. When set, the service exposes community XRPCs - * for managing community-owned spaces and tiered access levels. Requires `spaces`. */ - community?: import("./community/types").CommunityConfig; + /** Community module configuration. Typed by the community package via + * declaration merging — contrail core only knows it's "something the + * community package consumes." Set when wiring community via + * `createCommunityIntegration({ ... })`. Requires `spaces.authority`. */ + community?: unknown; /** Realtime module configuration. When set, the service exposes ticket + SSE/WS * subscribe XRPCs, and wraps the spaces adapter to publish events after writes. */ realtime?: import("./realtime/types").RealtimeConfig; diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index bbd8efe..63d07ec 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -110,6 +110,34 @@ export { registerRecordHostRoutes, } from "./core/spaces/router"; export type { EnrollmentRow } from "./core/spaces/types"; +export type { WhoamiExtension } from "./core/spaces/router"; + +// Internal-but-exposed bits — extension packages (community, etc.) need +// these to wire themselves up. Consumer apps generally don't. +export { buildSpaceUri, parseSpaceUri } from "./core/spaces/uri"; +export type { ServiceAuth } from "./core/spaces/auth"; +export { buildVerifier, createServiceAuthMiddleware } from "./core/spaces/auth"; +export { getDialect } from "./core/dialect"; +export type { SqlDialect } from "./core/dialect"; + +// App + schema wiring — extension packages and tests use these. +export { createApp } from "./core/router"; +export type { CreateAppOptions, SpacesContext } from "./core/router"; +export { initSchema } from "./core/db/schema"; +export type { InitSchemaOptions } from "./core/db/schema"; + +// Community integration interfaces — contrail core defines the shapes +// extension packages implement. The @atmo-dev/contrail-community package +// provides the concrete implementations. +export type { + CommunityIntegration, + CommunityProbe, +} from "./core/community-integration"; +export type { + CommunityInviteHandler, + HandlerResponse, +} from "./core/invite/community-handler"; +export type { SchemaModule } from "./core/db/schema"; // Realtime export type { @@ -160,28 +188,6 @@ export { export type { PersistentLabelsOptions } from "./core/labels/subscribe"; export { resolveLabelerEndpoint } from "./core/labels/resolve"; -// Community -export { - CommunityAdapter, - CredentialCipher, - registerCommunityRoutes, - ACCESS_LEVELS, - RESERVED_KEYS, - isAccessLevel, - isReservedKey, - rankOf, - resolveEffectiveLevel, - flattenEffectiveMembers, - wouldCycle, - reconcile, -} from "./core/community"; -export type { - CommunityConfig, - CommunityMode, - CommunityRow, - CommunityInviteRow, - CreateCommunityInviteInput, - AccessLevel, - AccessLevelRow, - ReservedKey, -} from "./core/community"; +// Community has moved to @atmo-dev/contrail-community. Import from there: +// import { createCommunityIntegration, CommunityAdapter, ... } from "@atmo-dev/contrail-community"; +// const app = createApp(db, config, { community: createCommunityIntegration(...) }); diff --git a/packages/contrail/tests/realtime-e2e.test.ts b/packages/contrail/tests/realtime-e2e.test.ts index 59e9338..0ef8aa2 100644 --- a/packages/contrail/tests/realtime-e2e.test.ts +++ b/packages/contrail/tests/realtime-e2e.test.ts @@ -7,6 +7,7 @@ import { createApp } from "../src/core/router"; import { resolveConfig } from "../src/core/types"; import type { ContrailConfig } from "../src/core/types"; import type { RealtimeEvent } from "../src/core/realtime/types"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -72,9 +73,11 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, + community, }); } diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 04f1a15..70a22e6 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from "vitest/config"; +import path from "node:path"; export default defineConfig({ test: { @@ -6,4 +7,15 @@ export default defineConfig({ // PostgreSQL tests share a single database and cannot run in parallel fileParallelism: false, }, + resolve: { + alias: { + // Point at contrail-community's source so tests don't require a built + // dist. Mirrors the contrail-community package's own vitest alias for + // `@atmo-dev/contrail` → contrail/src. + "@atmo-dev/contrail-community": path.resolve( + __dirname, + "../contrail-community/src/index.ts" + ), + }, + }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e648bf..11a1eb6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -82,6 +82,9 @@ importers: '@atmo-dev/contrail': specifier: workspace:* version: link:../../packages/contrail + '@atmo-dev/contrail-community': + specifier: workspace:* + version: link:../../packages/contrail-community '@atmo-dev/contrail-lexicons': specifier: workspace:* version: link:../../packages/lexicons @@ -383,6 +386,9 @@ importers: specifier: ^2.4.0 version: 2.6.1 devDependencies: + '@atmo-dev/contrail-community': + specifier: workspace:* + version: link:../contrail-community '@cloudflare/workers-types': specifier: ^4.20250124.0 version: 4.20260424.1 @@ -408,6 +414,43 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-community: + dependencies: + '@atcute/atproto': + specifier: ^3.1.10 + version: 3.1.11 + '@atcute/cbor': + specifier: ^2.3.2 + version: 2.3.2 + '@atcute/identity': + specifier: ^1.1.4 + version: 1.1.4 + '@atcute/identity-resolver': + specifier: ^1.2.2 + version: 1.2.2(@atcute/identity@1.1.4) + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atcute/xrpc-server': + specifier: ^0.1.12 + version: 0.1.12 + '@atmo-dev/contrail': + specifier: workspace:* + version: link:../contrail + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + vitest: + specifier: ^4.1.0 + version: 4.1.5(@types/node@25.6.0)(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.6.1)(tsx@4.21.0)) + packages/lexicons: dependencies: '@atcute/lex-cli': -- 2.51.2 From 7e3145b3950f161513798054f9535e8300451e9b Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:14:39 +0200 Subject: [PATCH 07/25] fix deploy, update docs, update ci --- .changeset/config.json | 2 +- .changeset/spaces-host-authority-split.md | 101 +++++++++ README.md | 4 +- docs/05-auth.md | 102 +++++++-- docs/06-spaces.md | 115 ++++++++-- docs/07-communities.md | 77 +++++-- docs/10-deployment-shapes.md | 199 ++++++++++++++++++ packages/contrail-community/package.json | 2 +- .../tests/realtime-community.test.ts | 148 +++++++++++++ packages/contrail/package.json | 1 - packages/contrail/tests/realtime-e2e.test.ts | 69 +----- packages/contrail/vitest.config.ts | 12 -- packages/lexicons/tests/generate.test.ts | 5 +- pnpm-lock.yaml | 3 - refs/spaces-later.md | 127 ++++++++--- refs/spaces-spec-mapping.md | 195 +++++++++-------- 16 files changed, 908 insertions(+), 254 deletions(-) create mode 100644 .changeset/spaces-host-authority-split.md create mode 100644 docs/10-deployment-shapes.md create mode 100644 packages/contrail-community/tests/realtime-community.test.ts diff --git a/.changeset/config.json b/.changeset/config.json index 237216e..745d87f 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -3,7 +3,7 @@ "changelog": "@changesets/cli/changelog", "commit": false, "fixed": [], - "linked": [["@atmo-dev/contrail", "@atmo-dev/contrail-sync"]], + "linked": [["@atmo-dev/contrail", "@atmo-dev/contrail-sync", "@atmo-dev/contrail-community"]], "access": "public", "baseBranch": "main", "updateInternalDependencies": "patch", diff --git a/.changeset/spaces-host-authority-split.md b/.changeset/spaces-host-authority-split.md new file mode 100644 index 0000000..e14e46c --- /dev/null +++ b/.changeset/spaces-host-authority-split.md @@ -0,0 +1,101 @@ +--- +"@atmo-dev/contrail": minor +"@atmo-dev/contrail-community": minor +"@atmo-dev/contrail-sync": minor +--- + +Spaces refactor: split authority + record host into independently runnable +roles, add space credentials, extract community into its own package. + +**Breaking — config shape** + +`spaces` is no longer flat — split into `authority` and `recordHost`: + +```ts +// before +spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, +} + +// after +spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, +} +``` + +**Breaking — community moved to its own package** + +Community has been extracted to `@atmo-dev/contrail-community`. Wire it via +`createCommunityIntegration`: + +```ts +import { Contrail, resolveConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + +const resolved = resolveConfig(config); +const communityIntegration = createCommunityIntegration({ db, config: resolved }); +const contrail = new Contrail({ ...config, communityIntegration }); +``` + +The community config (`config.community`) stays the same; only the wiring +moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, +`reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of +`@atmo-dev/contrail`. + +**New — space credentials (`X-Space-Credential`)** + +The space authority issues short-lived ES256 JWTs (default 2h TTL) via +`.space.getCredential` and `refreshCredential`. The record host accepts +them on read/write paths in lieu of per-request service-auth JWTs. Skips +DID-doc fetches and member checks; the credential's signature is the proof. + +Generate a signing key once at deploy time: + +```ts +import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; +const signing = await generateAuthoritySigningKey(); +// Store the JWK; pass to spaces.authority.signing. +``` + +**New — binding resolution** + +Verifiers can resolve "which authority signs for this space?" from three +sources, in order: local enrollment table, PDS records at +`at:////`, DID-doc `#atproto_space_authority` service +entry, owner-self fallback. Lets user-owned DIDs authorize a third-party +authority via a normal PDS write — no DID-doc surgery. + +**New — independent deployments + enrollment** + +The authority and record host can run as separate processes/operators. +A new `.recordHost.enroll` endpoint lets owners (or authorities) +register a space onto a host. In-process deployments auto-enroll on +`createSpace`; nothing changes for single-instance setups. + +See `docs/10-deployment-shapes.md` for all-in-one / authority-only / +host-only configurations and when to choose each. + +**Migration** + +For most deployments running spaces today, the migration is: + +1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. +2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. +3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + +Existing service-auth JWT clients keep working as a fallback path. +Migrate to space credentials when convenient — exchange a JWT for a +credential once via `getCredential`, then reuse it. diff --git a/README.md b/README.md index 1f88ec9..c4203e0 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,15 @@ returns every `community.lexicon.calendar.event` record published anywhere on at - [Communities](https://github.com/flo-bit/contrail/blob/main/docs/07-communities.md) — group-controlled atproto DIDs - [Sync](https://github.com/flo-bit/contrail/blob/main/docs/08-sync.md) — reactive client-side store over `watchRecords` - [Labels](https://github.com/flo-bit/contrail/blob/main/docs/09-labels.md) — atproto-native moderation hydration from external labelers +- [Deployment shapes](https://github.com/flo-bit/contrail/blob/main/docs/10-deployment-shapes.md) — all-in-one vs split-authority vs split-host configurations - Frameworks: [SvelteKit + Cloudflare](https://github.com/flo-bit/contrail/blob/main/docs/frameworks/sveltekit-cloudflare.md) ## Packages | Package | | |---|---| -| `@atmo-dev/contrail` | Core library — indexing, XRPC server, spaces, communities, realtime | +| `@atmo-dev/contrail` | Core library — indexing, XRPC server, spaces, realtime | +| `@atmo-dev/contrail-community` | Community module — group-controlled DIDs, access-level ladder. Plugs into core via an integration | | `@atmo-dev/contrail-sync` | Client-side reactive watch-store with optional IndexedDB cache | | `@atmo-dev/contrail-lexicons` | Codegen + `contrail-lex` CLI | diff --git a/docs/05-auth.md b/docs/05-auth.md index ad8f659..7cb7578 100644 --- a/docs/05-auth.md +++ b/docs/05-auth.md @@ -1,15 +1,18 @@ # Auth -Contrail has four auth mechanisms. Which one applies depends on who's calling and what they're asking for. +Contrail has six auth mechanisms. Which one applies depends on who's calling, where they're calling, and what they're asking for. | Mechanism | Used by | For | |---|---|---| | Anonymous | anyone | public reads | -| Service-auth JWT | third-party apps acting on behalf of a user | anything permissioned (spaces, communities) | +| Service-auth JWT | third-party apps acting on behalf of a user | authority-side ops + record-host fallback | +| **Space credential** (`X-Space-Credential`) | callers after exchange via `getCredential` | record-host reads/writes — primary path | | In-process server client | your own server code | loaders / actions that skip HTTP entirely | | Invite token | anonymous bearers | read-only access to a specific space | | Watch ticket | browsers | realtime subscriptions (`watchRecords`) | +The two "atproto-y" mechanisms (service-auth JWTs and space credentials) work in tandem on permissioned routes: a caller exchanges a JWT for a credential once via `space.getCredential`, then presents the credential on every subsequent record-host request until it expires. + ## Service-auth JWTs The standard atproto mechanism. When a third-party app wants to call your contrail service as a user, it: @@ -20,23 +23,87 @@ The standard atproto mechanism. When a third-party app wants to call your contra Contrail verifies every request against the public key in the issuer's DID doc (`@atcute/xrpc-server` does the heavy lifting). It checks: - Signature valid -- `aud` matches the `serviceDid` you configured +- `aud` matches the `serviceDid` you configured (under `spaces.authority.serviceDid`) - `lxm` covers the method being called - Token hasn't expired On pass, your handler sees a populated `serviceAuth = { issuer, audience, lxm, clientId }` context and can proceed. On fail, 401 or 403 with a structured reason. +### Where service-auth JWTs apply + +- **Authority routes** (`.space.createSpace`, `addMember`, `getCredential`, etc.) — JWT-only. Credentials are scoped to record-host operations; you can't use one to manage spaces. +- **Record-host routes** (`putRecord`, `listRecords`, `uploadBlob`, etc.) — accept JWTs as a fallback path. The credential path (below) is preferred. + ### The `serviceDid` gotcha Use the **plain DID** (no `#fragment`) when configuring contrail: ```ts -spaces: { serviceDid: "did:web:example.com" } // right -spaces: { serviceDid: "did:web:example.com#com_example_space" } // wrong +spaces: { authority: { serviceDid: "did:web:example.com" } } // right +spaces: { authority: { serviceDid: "did:web:example.com#com_example_x" } } // wrong ``` Many PDS implementations reject `aud` values containing `#fragment` in `com.atproto.server.getServiceAuth`, and contrail does strict string equality on `aud`. The fragment form belongs only in your DID doc's `service` entry, where PDSes use it to resolve the service endpoint URL for `Atproto-Proxy` routing — that's separate from JWT audience validation. +## Space credentials + +Short-lived (default 2h) ES256 JWTs minted by the space authority. Once a caller has one, they present it via `X-Space-Credential: ` on every record-host request and skip the per-request JWT mint dance. This matches the rough atproto permissioned-data spec. + +### Lifecycle + +``` +1. Caller mints a service-auth JWT { aud, lxm: ".space.getCredential" }. +2. POST .space.getCredential { spaceUri } Authorization: Bearer + → { credential: "", expiresAt: } +3. Caller stores the credential. For ~2 hours, every record-host request: + X-Space-Credential: + succeeds without going back through the user's PDS. +4. Before expiry, refresh: + POST .space.refreshCredential { credential } + → { credential: , expiresAt: } +``` + +### Claims + +```json +{ + "iss": "", + "sub": "", + "space": "ats:////", + "scope": "rw", + "iat": 1746000000, + "exp": 1746014400 +} +``` + +- Signed with the authority's ES256 key (kid = `#atproto_space_authority`). +- Stateless — verifiable by anyone who can resolve the authority DID's verification key. +- `scope` is `"rw"` or `"read"`. Today only `rw` is issued via `getCredential`; the read-only path is a future read-grant invite replacement. + +### How verification works + +When the record host receives a credential, it: + +1. Decodes the JWT, reads `iss` and `space`. +2. Asks its **binding resolver** "who's authorized to sign for this space?" — primary source is the local enrollment table; fallbacks include PDS records and DID-doc service entries. +3. Confirms `iss` matches the authorized DID. +4. Resolves the issuer's verification key (local in-process, or via DID doc). +5. Verifies signature, expiry, scope, space match. + +In an in-process deployment, this is one DB lookup + one signature verification. No DID-doc fetches per request. See [Spaces](./06-spaces.md#discovery--binding-resolution) for the binding details. + +### When the credential gets rejected + +| Reason | Response | +|---|---| +| `malformed` | 401 — JWT structure invalid | +| `bad-alg` | 401 — header alg ≠ ES256 | +| `bad-signature` | 401 — signature didn't verify against the resolved key | +| `expired` | 401 — past `exp`. Refresh, or re-mint. | +| `wrong-space` | 403 — credential's `space` ≠ request's space | +| `wrong-scope` | 403 — read-only credential on a write | +| `unknown-issuer` | 401 — `iss` doesn't match the binding for the space (most often: not enrolled here) | + ## In-process server client When your own server code wants to call contrail, the service-auth dance is pointless — it's your code talking to your code. `createServerClient` skips it: @@ -50,7 +117,7 @@ const client = createServerClient(async (req) => handle(req, env.DB), userDid); const res = await client.get("com.example.event.listRecords", { params: {...} }); ``` -Pass `did` to act as that user; omit it for anonymous calls against public endpoints. This is a trust boundary — anything that actually crosses a network needs a real service-auth JWT, not this shortcut. +Pass `did` to act as that user; omit it for anonymous calls against public endpoints. This is a trust boundary — anything that actually crosses a network needs a real service-auth JWT or space credential, not this shortcut. See [SvelteKit + Cloudflare](./frameworks/sveltekit-cloudflare.md) for the typical loader pattern. @@ -59,7 +126,7 @@ See [SvelteKit + Cloudflare](./frameworks/sveltekit-cloudflare.md) for the typic First-class auth for spaces. When a space owner creates an invite: ``` -com.example.space.invite.create { spaceUri, ttl?, maxUses? } +.invite.create { spaceUri, kind, ttl?, maxUses? } → { token: "...plaintext..." } // returned once, never again ``` @@ -67,7 +134,7 @@ The plaintext token is handed to the user out-of-band (link, QR, email). Contrai Three invite kinds, depending on what the token does: -- **`join`** — redeemed via `com.example.space.invite.redeem` with a service-auth JWT. Adds the caller's DID to the member list. Members have full read + write inside the space; there's no per-member permission axis beyond "is a member." +- **`join`** — redeemed via `.invite.redeem` with a service-auth JWT. Adds the caller's DID to the member list. Members have full read + write inside the space; there's no per-member permission axis beyond "is a member." - **`read`** — bearer-only. The token itself grants read access when passed as `?inviteToken=`, no DID, no redemption. Good for sharing a read-only link that doesn't add anyone to the member list. - **`read-join`** — both. Works anonymously as a read token; can also be redeemed with a JWT to promote the caller to member. @@ -79,7 +146,7 @@ Realtime subscriptions (`watchRecords`) can't use regular service-auth JWTs for Server-side minting comes in two flavours: -- `com.example.realtime.ticket` — POST `{ topic }` (e.g. `"space:ats://..."`) → `{ ticket, topics, expiresAt }`. Bare topic-list ticket, used with the generic `<ns>.realtime.subscribe` endpoint. +- `<ns>.realtime.ticket` — POST `{ topic }` (e.g. `"space:ats://..."`) → `{ ticket, topics, expiresAt }`. Bare topic-list ticket, used with the generic `<ns>.realtime.subscribe` endpoint. - `<collection>.watchRecords?mode=ws&spaceUri=…` (or `&actor=…`) handshake — returns `{ snapshot, ticket, wsUrl, sinceTs, ticketTtlMs, querySpec }`. The ticket is bound to `(did, topics, querySpec)` and is the one to use for the per-collection `watchRecords` stream — both for SSE (`?ticket=…`) and the subsequent WS upgrade. Both flavours are signed by `realtime.ticketSecret` (a 32-byte random, configured once). Clients hand the ticket off via `?ticket=...` on connect. @@ -129,9 +196,14 @@ A typical flow for a third-party app acting as a user in a space: 1. App registers OAuth client pointing at your permission set NSID. 2. User grants consent — PDS fetches your permission set lexicon via DNS, shows the user the methods, records the scope. -3. App calls `com.atproto.server.getServiceAuth` on the user's PDS: `{ aud: "did:web:example.com", lxm: "com.example.space.putRecord", exp: <60s> }`. PDS signs, returns JWT. -4. App sends `PUT /xrpc/com.example.space.putRecord` with `Authorization: Bearer <jwt>` and `Atproto-Proxy: did:web:example.com#com_example_space`. -5. User's PDS reads the `Atproto-Proxy` header, resolves the service endpoint from your DID doc, forwards the request. -6. Contrail verifies the JWT (signature, `aud`, `lxm`, expiry), runs ACL check (is this DID a member of that space?), dispatches the write. - -For your own loaders/actions, steps 3–6 collapse into a single `createServerClient({did}).post(...)` call. For a browser subscribing to a feed, steps 3–5 are replaced by a ticket mint from your server. Same auth model, different surface. +3. App calls `com.atproto.server.getServiceAuth` on the user's PDS: `{ aud: "did:web:example.com", lxm: "com.example.space.getCredential", exp: <60s> }`. PDS signs, returns JWT. +4. App POSTs `<ns>.space.getCredential { spaceUri }` with `Authorization: Bearer <jwt>` and `Atproto-Proxy: did:web:example.com#com_example_space`. Authority verifies the JWT, checks membership, mints a 2h credential. +5. App caches the credential. Every subsequent `putRecord` / `listRecords` / `uploadBlob`: + ``` + POST <ns>.space.putRecord + X-Space-Credential: <credential> + ``` + The record host verifies the credential against its enrolled authority's key — no PDS roundtrip, no DID-doc fetch. +6. Before expiry, app calls `refreshCredential` to get a fresh one. + +For your own loaders/actions, the credential dance disappears — `createServerClient({did}).post(...)` bypasses it entirely. For a browser subscribing to a feed, steps 3–5 are replaced by a ticket mint from your server. Same auth model, different surface. diff --git a/docs/06-spaces.md b/docs/06-spaces.md index 8bb9fb8..338abc7 100644 --- a/docs/06-spaces.md +++ b/docs/06-spaces.md @@ -12,17 +12,40 @@ Auth-gated store for records that can't live on public PDSes — private events, Every permission boundary is its own space. No nested ACLs. Richer roles = more spaces or app-layer checks. +### Two roles, one or two services + +A space has two operational roles: + +- **Space authority** — owns the member list, signs short-lived credentials. Identified by a service DID. +- **Record host** — stores records and blobs for spaces it has *enrolled*. + +In the default deployment both run in the same Contrail instance against the same DB; you don't notice the split. But the roles can also run separately — see [deployment shapes](./10-deployment-shapes.md) for ACL-on-arbiter / records-on-contrail patterns. + ## Enable ```ts import type { ContrailConfig } from "@atmo-dev/contrail"; +import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + +// One-time setup: generate a signing key and store it. The authority signs +// space credentials with this key; verifiers find the public key in the +// authority DID's DID document or via the binding-resolver chain. +const signing = await generateAuthoritySigningKey(); const config: ContrailConfig = { namespace: "com.example", collections: { /* ... */ }, spaces: { - type: "com.example.event.space", - serviceDid: "did:web:example.com", + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing, // omit to disable credential issuance + credentialTtlMs: 2 * 60 * 60 * 1000, // 2h, matches the rough spec + }, + recordHost: { + // blobs is optional; omit to disable blob endpoints + blobs: { adapter: blobsAdapter }, + }, }, }; ``` @@ -33,42 +56,98 @@ Each collection gets a parallel `spaces_records_<short>` table. Opt out per-coll public_only: { collection: "com.example.public", allowInSpaces: false } ``` -## Auth +## Auth — three paths + +The record host accepts three forms of auth on read/write paths, in this precedence order: -Spaces use the standard contrail auth surface — service-auth JWTs for third-party apps, in-process server clients for your own loaders, invite tokens for anonymous read-grant links. See [Auth](./05-auth.md) for the full picture. +1. **`X-Space-Credential` header** — a short-lived JWT minted by the space authority. The primary path: callers exchange a service-auth JWT once via `space.getCredential`, then present the credential on every request until it expires. Skips per-request DID-doc fetches and member checks; the credential's signature is the proof. +2. **`?inviteToken=...` query** (read-only) — bearer access for shareable links. See [Auth § Invite tokens](./05-auth.md#invite-tokens). +3. **`Authorization: Bearer <service-auth-jwt>`** — the standard atproto path. Useful for one-off calls (the credential exchange itself, space-management endpoints) or as a fallback when the caller doesn't want to manage credentials. + +Authority-side endpoints (`createSpace`, `addMember`, `getCredential`, etc.) only accept service-auth JWTs — credentials are scoped to record-host operations. + +See [Auth](./05-auth.md) for the full picture. + +## Credential flow + +```text + ┌──────────────┐ + │ user PDS │ mints service-auth JWT (lxm: getCredential) + └──────┬───────┘ + ▼ + ┌──────────────┐ + │ authority │ validates JWT, checks membership, + │ (Contrail) │ signs ES256 credential (2h TTL) + └──────┬───────┘ + │ { credential, expiresAt } + ▼ + ┌──────────────┐ + │ record host │ verifies credential signature against + │ (Contrail or │ authority DID's published key, + │ elsewhere) │ checks scope/space/expiry, serves request + └──────────────┘ +``` -Space-specific wiring: +`space.refreshCredential` re-issues a fresh credential from an unexpired one without going back through the JWT mint dance — useful for long-running clients. -- `serviceDid` in the config is the `aud` contrail expects on incoming JWTs. Plain DID, no `#fragment`. -- Apps acting in a space send `Atproto-Proxy: <serviceDid>#<service-id-from-your-did-doc>` so the user's PDS routes correctly. -- Invite redemption via service-auth JWT grants membership; via `?inviteToken=...` query param grants read-only bearer access to that space. +## Enrollment + +The record host maintains a local table of which spaces it accepts records for and which authority signs credentials for each. Two ways enrollment happens: + +- **Auto-enroll** (default for in-process deployments): the authority's `createSpace` automatically enrolls the new space on the colocated record host. New users see no enrollment surface; it just works. +- **Explicit `recordHost.enroll`**: for split deployments where the authority and record host run in different processes/operators, the owner (or the authority itself) calls `<ns>.recordHost.enroll { spaceUri, authority }` to consent. Idempotent — re-enrolling updates the binding. + +A non-enrolled space gets 404 "not-enrolled" on every record-host route. This is the host's consent layer — without it, anyone with a valid credential could create unbounded storage on your host. + +## Discovery — binding resolution + +When a record host receives a credential, it needs to know whether the credential's `iss` is authorized to sign for that space. Three sources, tried in order: + +1. **Local enrollment** — primary on the record host. `(spaceUri → authorityDid)` from the enrollment table. +2. **PDS record** at `at://<owner>/<type>/<key>` — for user-owned DIDs that declared a host via a normal PDS write. Lexicon: `tools.atmo.space.declaration` (or your namespaced variant). +3. **DID-doc service entry** — `#atproto_space_authority` on the owner's DID doc. For provisioned (no-PDS) DIDs. +4. **Owner self-issues** (fallback) — for the trivial case where the owner DID's own key signs credentials. + +For in-process deployments, step 1 is the only one that fires. The other resolvers are wired in by deployments that accept credentials from external authorities — see [deployment shapes](./10-deployment-shapes.md). ## Unified `listRecords` | Call | Returns | |---|---| | no auth, no `spaceUri` | public only | -| `?spaceUri=…` + JWT | one space (ACL-gated) | -| JWT, no `spaceUri` | public **unioned** with every space the caller is a member of | +| `?spaceUri=…` + credential or JWT | one space (ACL-gated) | +| credential / JWT, no `spaceUri` | public **unioned** with every space the caller is a member of | Filters, sorts, hydration, and references work across all three. Records from a space carry a `space: <spaceUri>` field — same on `listRecords`/`getRecord` responses and `watchRecords` stream events. ## Invites -First-class primitive — see [Auth § Invite tokens](./05-auth.md#invite-tokens) for the mechanism. Space-specific: create via `com.example.space.invite.create`, redeem via `.redeem` (membership grant) or `?inviteToken=...` query param (read-only bearer grant). +First-class primitive — see [Auth § Invite tokens](./05-auth.md#invite-tokens) for the mechanism. Space-specific: create via `<ns>.invite.create`, redeem via `.redeem` (membership grant) or `?inviteToken=...` query param (read-only bearer grant). ## XRPCs -- `com.example.space.create | get | list | delete` -- `com.example.space.putRecord | deleteRecord | listRecords | getRecord` -- `com.example.space.invite.create | redeem | revoke | list` -- `com.example.space.listMembers | removeMember` +### Authority routes (`<ns>.space.*` — spec-aligned) +- `createSpace` `getSpace` `listSpaces` `deleteSpace` +- `listMembers` `addMember` `removeMember` +- `getCredential` `refreshCredential` +- `leaveSpace` (contrail extra) + +### Record-host routes (`<ns>.space.*` for records, `<ns>.recordHost.*` for management) +- `putRecord` `deleteRecord` `getRecord` `listRecords` +- `uploadBlob` `getBlob` `listBlobs` (when `recordHost.blobs` is configured) +- `recordHost.enroll` + +### Contrail extras (`<ns>.spaceExt.*`) +- `whoami` — caller's relationship to a space (extensions plug in via the integration's whoami hook) + +### Invite (`<ns>.invite.*`) +- `create` `redeem` `revoke` `list` ## What's not here - No E2EE (data is operator-readable). - No FTS on `?spaceUri=…` yet. -- No per-space sharding — one DB, one operator. -- Not a long-term replacement for real atproto permissioned repos. +- Records still live in the operator's DB rather than user PDSes — federation is greenfield. (See `refs/spaces-spec-mapping.md` for the migration notes.) +- No managing-app routing (join requests, approval queues — see `refs/spaces-later.md`). -The design follows Daniel Holmgren's [permissioned data rough spec](https://dholms.leaflet.pub/3mhj6bcqats2o). The goal is that when real atproto permissioned repos ship, migration is mostly data movement — the API your app speaks doesn't change. +The design follows Daniel Holmgren's [permissioned data rough spec](https://dholms.leaflet.pub/3mhj6bcqats2o). When real atproto permissioned repos ship, migration is mostly data movement — the wire surface your app speaks doesn't change. diff --git a/docs/07-communities.md b/docs/07-communities.md index a7f2e26..0268c42 100644 --- a/docs/07-communities.md +++ b/docs/07-communities.md @@ -2,6 +2,56 @@ Group-controlled atproto DIDs. A community is a DID whose signing/rotation keys are held by the appview on behalf of multiple members, with tiered access levels. Built on top of [spaces](./06-spaces.md). +Communities live in a separate package — `@atmo-dev/contrail-community` — that plugs into Contrail via an integration object. The contrail core has no knowledge of community-specific concepts; the package wires itself in via injectable hooks (whoami extension, invite handler, route registration, schema). + +## Install + +```bash +pnpm add @atmo-dev/contrail @atmo-dev/contrail-community +``` + +## Wire it up + +Construct the integration once, hand it to `Contrail` (or directly to `createApp`): + +```ts +import { Contrail, resolveConfig, type ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + +const config: ContrailConfig = { + namespace: "com.example", + collections: { /* ... */ }, + spaces: { + authority: { type: "com.example.event.space", serviceDid: "did:web:example.com", signing }, + recordHost: {}, + }, + community: { + masterKey: env.COMMUNITY_MASTER_KEY, // 32-byte encryption key for stored credentials + serviceDid: "did:web:example.com", + levels: ["admin", "moderator"], // ranked, highest-first + }, +}; + +const resolved = resolveConfig(config); +const communityIntegration = createCommunityIntegration({ db, config: resolved }); + +const contrail = new Contrail({ ...config, db, communityIntegration }); +await contrail.init(); // applies community schema alongside contrail's own +``` + +Or with `createApp` directly: + +```ts +import { createApp, initSchema } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + +const community = createCommunityIntegration({ db, config }); +await initSchema(db, config, { extraSchemas: [community.applySchema] }); +const app = createApp(db, config, { community }); +``` + +Stored credentials (app passwords for adopted communities, signing keys for minted) are envelope-encrypted with `masterKey`. Never ship the placeholder. + ## When to use this When you want atproto records published under a *shared* identity — a team, a project, a channel — not a single user. Think: a group's published calendar events, a community's published posts. @@ -15,17 +65,7 @@ Either way, the result is the same: a DID that multiple members can act through, ## Access levels -Each member has a level (ranked). Levels map to write permissions. Owners can grant/revoke levels. Two reserved levels exist: `owner` and `member`. Your deployment defines the rest: - -```ts -community: { - masterKey: ENV.COMMUNITY_MASTER_KEY, // 32-byte encryption key for stored credentials - serviceDid: "did:web:example.com", - levels: ["admin", "moderator"], // ranked, highest-first -} -``` - -Stored credentials (app passwords for adopted communities, signing keys for minted) are envelope-encrypted with `masterKey`. Never ship the placeholder. +Each member has a level (ranked). Levels map to write permissions. Owners can grant/revoke levels. Two reserved levels exist: `owner` and `member`. Your deployment defines the rest via `config.community.levels`. ## How it composes with spaces @@ -37,13 +77,18 @@ community.space.grant { spaceUri, subject: { did: "did:plc:..." }, accessLevel: The spaces layer stays ignorant of access levels — it just sees "this DID is a member." The community layer projects member × level → membership in specific spaces. Once a DID is a member of a space (through a community grant or otherwise), they have full read + write inside it. +The integration plugs in to two contrail extension points: + +- **Whoami** — `<ns>.spaceExt.whoami` returns `accessLevel` for community-owned spaces (the community whoami extension overrides the default binary-membership response). +- **Invites** — the unified `<ns>.invite.*` family dispatches community-owned spaces through the community invite handler (which uses access levels) and user-owned spaces through the spaces module's binary-membership handler. + ## XRPCs -- `com.example.community.mint | adopt | list | delete` -- `com.example.community.invite.create | redeem | revoke | list` -- `com.example.community.setAccessLevel | revoke | listMembers` -- `com.example.community.space.create | grant | revoke | ...` — community-owned spaces -- `com.example.community.putRecord | deleteRecord` — publish records as the community DID +- `<ns>.community.mint | adopt | list | delete` +- `<ns>.community.invite.create | redeem | revoke | list` +- `<ns>.community.setAccessLevel | revoke | listMembers` +- `<ns>.community.space.create | grant | revoke | ...` — community-owned spaces +- `<ns>.community.putRecord | deleteRecord` — publish records as the community DID ## What's not here diff --git a/docs/10-deployment-shapes.md b/docs/10-deployment-shapes.md new file mode 100644 index 0000000..25464ed --- /dev/null +++ b/docs/10-deployment-shapes.md @@ -0,0 +1,199 @@ +# Deployment shapes + +Spaces split into two roles, run together by default. Three deployment shapes, in increasing order of complexity: + +1. **All-in-one** — authority + record host + (optional) community in one process. The default; what you get from `createApp` with both `spaces.authority` and `spaces.recordHost` configured. Most apps want this. +2. **Authority-only** — a service that controls ACL and signs credentials, but doesn't store records. Useful when records live on someone else's host (e.g. a community arbiter that delegates storage to a heavier appview). +3. **Record-host-only** — a service that stores records, accepting credentials signed by an external authority. Useful when storage lives separately from governance — e.g. Contrail-as-host backing spaces that an "Arbiter" or HappyView manages. + +The split is real at the wire level (different XRPCs, different auth shapes) but the same Contrail codebase handles all three. This doc walks through each. + +## Shape 1: all-in-one (default) + +``` +┌────────────────────────────────────┐ +│ Contrail │ +│ ┌────────┐ ┌────────────┐ │ +│ │authority│ │ record host│ │ +│ │ + signing│ │ + enrollment│ │ +│ └────────┘ └────────────┘ │ +│ shared DB, single process │ +└────────────────────────────────────┘ +``` + +Config: + +```ts +spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter: blobsAdapter }, // optional + }, +}, +``` + +What happens at startup: + +- `initSchema` creates both authority tables (`spaces`, `spaces_members`, `spaces_invites`) and record-host tables (`spaces_records_<short>`, `spaces_blobs`, `record_host_enrollments`). +- The umbrella router wires `registerAuthorityRoutes` + `registerRecordHostRoutes` against the same `HostedAdapter`. +- The credential verifier is built from `Local` binding + `Local` key — no DID-doc fetches; the host knows the authority's public key directly. + +What happens when a user creates a space: + +1. `createSpace` writes a row in `spaces` (authority) and immediately a row in `record_host_enrollments` (host). One round-trip, two DB writes. +2. From there, `getCredential` works, `putRecord` works, the world is in sync. + +This is the path most apps run. You don't notice the role split. + +## Shape 2: authority-only + +A lightweight service that holds ACL and signs credentials. Records live on someone else's host. + +``` + ┌────────────────┐ + │ this Contrail │ + │ authority │ + └────────────────┘ + ▲ + │ getCredential + │ + ┌─────────────┐ + │ client │ + └─────────────┘ + │ X-Space-Credential + ▼ + ┌─────────────┐ + │ external │ enrolled with this authority + │ record host │ (different operator, different DID) + └─────────────┘ +``` + +Config: + +```ts +spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:authority.example.com", + signing: await generateAuthoritySigningKey(), + }, + // recordHost omitted — this deployment doesn't store records +}, +``` + +`createSpace` here does NOT auto-enroll anywhere. The space owner (or the authority itself) calls `recordHost.enroll` on whichever host they want to use; the host then accepts credentials signed by this authority for that space. + +The authority's DID document needs to publish the verification key under `#atproto_space_authority` so external hosts can resolve it. + +## Shape 3: record-host-only + +A storage tier that accepts credentials signed by external authorities. + +``` +┌────────────┐ +│ external │ signs credentials +│ authority │ +└────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ this Contrail │ +│ record host (no authority) │ +│ │ +│ verifies credentials via: │ +│ - enrollment table │ +│ - DID-doc key resolver │ +│ (for external authorities) │ +└─────────────────────────────────────┘ +``` + +Config: + +```ts +spaces: { + // authority is still needed for the JWT verifier infrastructure (so the + // record host can validate JWTs on the recordHost.enroll endpoint), but + // no signing key is configured — this deployment doesn't issue creds. + authority: { + type: "com.example.event.space", + serviceDid: "did:web:host.example.com", + }, + recordHost: { + blobs: { adapter: blobsAdapter }, + }, +}, +``` + +To accept credentials from an external authority, wire a custom verifier: + +```ts +import { + createApp, + createBindingCredentialVerifier, + createEnrollmentBindingResolver, + createDidDocKeyResolver, +} from "@atmo-dev/contrail"; +import { CompositeDidDocumentResolver, PlcDidDocumentResolver, WebDidDocumentResolver } + from "@atcute/identity-resolver"; + +const didResolver = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, +}); + +const verifier = createBindingCredentialVerifier({ + // Local enrollment is the canonical binding source — only spaces this + // host has explicitly opted into are accepted. + bindings: createEnrollmentBindingResolver({ recordHost: hostAdapter }), + // For credential signature verification, walk DID docs of external + // authorities to find their published verification keys. + keys: createDidDocKeyResolver({ resolver: didResolver }), +}); + +const app = createApp(db, config, { + spaces: { credentialVerifier: verifier }, +}); +``` + +The flow when a request arrives: + +1. Caller presents `X-Space-Credential: <jwt>`. +2. Verifier reads `iss` from the JWT, looks up enrollment for `claims.space`. If the enrollment's `authorityDid` matches `iss` → continue. If not → 401 `unknown-issuer`. +3. Resolves the issuer DID, finds the verification method with id matching the JWT's `kid`, verifies the signature. +4. Checks expiry, scope, space match. +5. Serves the request. + +Enrollment is the host's consent layer: a credential can only be presented for spaces the host has agreed to store. Without enrollment, no records get written. + +## Mixing shapes + +You can run all three simultaneously in one Contrail instance. The umbrella router enables each set of routes based on what's configured: + +- `spaces.authority` → authority routes registered (`createSpace`, `getCredential`, etc.) +- `spaces.recordHost` → record-host routes registered (`putRecord`, `recordHost.enroll`, etc.) +- Both → today's default. + +A deployment can act as the authority for spaces it owns *and* a record host for spaces other authorities own. Auto-enroll fires only for spaces this deployment is the authority for; external authorities still enroll explicitly. + +## Choosing a shape + +| Need | Shape | +|---|---| +| One operator, one process, want it to work | All-in-one | +| You're running a "DAO governance / arbiter" service that decides ACL but not storage | Authority-only | +| You're running an appview / heavier storage tier and want to accept ACL decisions from external services | Record-host-only | +| You're an existing Contrail deployment that wants to also accept external authorities | All-in-one + custom verifier | + +When in doubt, all-in-one. Splitting is for when you have a real operational reason to separate the two — different teams running them, different latency profiles, different scaling targets, different governance. + +## What's not here + +- **Authority migration** — moving a space's authority from DID A to DID B. The architecture supports it (re-enroll on the host with the new authority binding) but no helper API yet. +- **Multi-authority per space** — could in principle allow several authorities to all sign for one space (replication scenarios). Not modeled today; spec is silent. +- **PDS-backed records** — when atproto's permissioned-repos protocol ships, records will federate from user PDSes. The host becomes an aggregator rather than a store. The role split here generalizes to that world without changes. diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index 2fae478..43fe2e1 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-community", - "version": "0.1.0", + "version": "0.4.2", "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-community/tests/realtime-community.test.ts b/packages/contrail-community/tests/realtime-community.test.ts new file mode 100644 index 0000000..c6c6b7a --- /dev/null +++ b/packages/contrail-community/tests/realtime-community.test.ts @@ -0,0 +1,148 @@ +/** Realtime + community integration test. Lives here (not in contrail) so + * contrail's package.json doesn't have to dev-depend on contrail-community + * (which would create a build-graph cycle in turbo). + * + * Tests that `community:<did>` topics expand to the caller's reachable + * community spaces — the cross-cutting concern that needs both modules. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { + createApp, + initSchema, + resolveConfig, + type ContrailConfig, +} from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; + +const ALICE = "did:plc:alice"; +const CHARLIE = "did:plc:charlie"; + +const MASTER_KEY = new Uint8Array(32).fill(5); +const REALTIME_SECRET = new Uint8Array(32).fill(9); + +const CONFIG: ContrailConfig = { + namespace: "test.rt", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, + }, + community: { + masterKey: MASTER_KEY, + plcDirectory: "https://plc.test", + fetch: mockFetch, + resolver: mockResolver(), + }, + realtime: { + ticketSecret: REALTIME_SECRET, + keepaliveMs: 60_000, + }, +}; + +function mockResolver(): any { + return { + resolve: async (_did: string) => ({ + id: _did, + service: [ + { id: "#atproto_pds", type: "AtprotoPersonalDataServer", serviceEndpoint: "https://pds.test" }, + ], + }), + }; +} + +async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.endsWith("/xrpc/com.atproto.server.createSession")) { + return new Response( + JSON.stringify({ accessJwt: "a.b.c", refreshJwt: "r.r.r", did: "did:plc:community" }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + if (url.startsWith("https://plc.test/")) return new Response("{}", { status: 200 }); + return new Response("not found", { status: 404 }); +} + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); + await next(); + }; +} + +async function makeApp(): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const resolved = resolveConfig(CONFIG); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); +} + +function call( + app: Hono, + method: string, + path: string, + did: string | null, + body?: any +): Promise<Response> { + const headers: Record<string, string> = {}; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +describe("realtime + community", () => { + let app: Hono; + beforeAll(async () => { + app = await makeApp(); + }); + + it("community:<did> alias expands to reachable spaces", async () => { + // Adopt a community, create a child space, grant Charlie member. + const adoptRes = await call(app, "POST", "/xrpc/test.rt.community.adopt", ALICE, { + identifier: "did:plc:community", + appPassword: "anything", // mockFetch returns 200 for createSession + }); + expect(adoptRes.status).toBe(200); + const { communityDid } = (await adoptRes.json()) as any; + + const c1 = await call(app, "POST", "/xrpc/test.rt.community.space.create", ALICE, { + communityDid, + key: "general", + }); + expect(c1.status).toBe(200); + const general = ((await c1.json()) as any).space.uri as string; + + // Grant Charlie as member in general. + const g = await call(app, "POST", "/xrpc/test.rt.community.space.grant", ALICE, { + spaceUri: general, + subject: { did: CHARLIE }, + accessLevel: "member", + }); + expect(g.status).toBe(200); + + // Charlie mints a community-alias ticket; should expand to [space:general]. + const ticketRes = await call(app, "POST", "/xrpc/test.rt.realtime.ticket", CHARLIE, { + topic: `community:${communityDid}`, + }); + expect(ticketRes.status).toBe(200); + const body = (await ticketRes.json()) as any; + expect(body.topics).toContain(`space:${general}`); + }); +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 846e6e8..a64484d 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -78,7 +78,6 @@ "jiti": "^2.4.0" }, "devDependencies": { - "@atmo-dev/contrail-community": "workspace:*", "@cloudflare/workers-types": "^4.20250124.0", "@types/node": "^25.5.0", "@types/pg": "^8.20.0", diff --git a/packages/contrail/tests/realtime-e2e.test.ts b/packages/contrail/tests/realtime-e2e.test.ts index 0ef8aa2..01d1767 100644 --- a/packages/contrail/tests/realtime-e2e.test.ts +++ b/packages/contrail/tests/realtime-e2e.test.ts @@ -7,13 +7,10 @@ import { createApp } from "../src/core/router"; import { resolveConfig } from "../src/core/types"; import type { ContrailConfig } from "../src/core/types"; import type { RealtimeEvent } from "../src/core/realtime/types"; -import { createCommunityIntegration } from "@atmo-dev/contrail-community"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; -const CHARLIE = "did:plc:charlie"; -const MASTER_KEY = new Uint8Array(32).fill(5); const REALTIME_SECRET = new Uint8Array(32).fill(9); const CONFIG: ContrailConfig = { @@ -26,41 +23,12 @@ const CONFIG: ContrailConfig = { }, recordHost: {}, }, - community: { - masterKey: MASTER_KEY, - plcDirectory: "https://plc.test", - fetch: mockFetch, - resolver: mockResolver(), - }, realtime: { ticketSecret: REALTIME_SECRET, keepaliveMs: 60_000, }, }; -function mockResolver(): any { - return { - resolve: async (_did: string) => ({ - id: _did, - service: [ - { id: "#atproto_pds", type: "AtprotoPersonalDataServer", serviceEndpoint: "https://pds.test" }, - ], - }), - }; -} - -async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> { - const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; - if (url.endsWith("/xrpc/com.atproto.server.createSession")) { - return new Response( - JSON.stringify({ accessJwt: "a.b.c", refreshJwt: "r.r.r", did: "did:plc:community" }), - { status: 200, headers: { "content-type": "application/json" } } - ); - } - if (url.startsWith("https://plc.test/")) return new Response("{}", { status: 200 }); - return new Response("not found", { status: 404 }); -} - function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); @@ -73,11 +41,9 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise<Hono> { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - const community = createCommunityIntegration({ db, config: resolved }); - await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + await initSchema(db, resolved); return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, - community, }); } @@ -280,37 +246,4 @@ describe("realtime e2e (in-memory pubsub, SSE transport)", () => { ); expect(res.status).toBe(401); }); - - it("community:<did> alias expands to reachable spaces", async () => { - // Adopt a community, create two child spaces, grant Charlie member on one. - const adoptRes = await call(app, "POST", "/xrpc/test.rt.community.adopt", ALICE, { - identifier: "did:plc:community", - appPassword: "anything", // mockFetch returns 200 for createSession - }); - expect(adoptRes.status).toBe(200); - const { communityDid } = (await adoptRes.json()) as any; - - const c1 = await call(app, "POST", "/xrpc/test.rt.community.space.create", ALICE, { - communityDid, - key: "general", - }); - expect(c1.status).toBe(200); - const general = ((await c1.json()) as any).space.uri as string; - - // Grant Charlie as member in general. - const g = await call(app, "POST", "/xrpc/test.rt.community.space.grant", ALICE, { - spaceUri: general, - subject: { did: CHARLIE }, - accessLevel: "member", - }); - expect(g.status).toBe(200); - - // Charlie mints a community-alias ticket; should expand to [space:general]. - const ticketRes = await call(app, "POST", "/xrpc/test.rt.realtime.ticket", CHARLIE, { - topic: `community:${communityDid}`, - }); - expect(ticketRes.status).toBe(200); - const body = (await ticketRes.json()) as any; - expect(body.topics).toContain(`space:${general}`); - }); }); diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 70a22e6..04f1a15 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -1,5 +1,4 @@ import { defineConfig } from "vitest/config"; -import path from "node:path"; export default defineConfig({ test: { @@ -7,15 +6,4 @@ export default defineConfig({ // PostgreSQL tests share a single database and cannot run in parallel fileParallelism: false, }, - resolve: { - alias: { - // Point at contrail-community's source so tests don't require a built - // dist. Mirrors the contrail-community package's own vitest alias for - // `@atmo-dev/contrail` → contrail/src. - "@atmo-dev/contrail-community": path.resolve( - __dirname, - "../contrail-community/src/index.ts" - ), - }, - }, }); diff --git a/packages/lexicons/tests/generate.test.ts b/packages/lexicons/tests/generate.test.ts index 44f45c5..eaf57bc 100644 --- a/packages/lexicons/tests/generate.test.ts +++ b/packages/lexicons/tests/generate.test.ts @@ -273,7 +273,10 @@ describe("extractXrpcMethods / listXrpcMethods", () => { const config: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, - spaces: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + recordHost: {}, + }, community: { masterKey: new Uint8Array(32).fill(1) }, realtime: { ticketSecret: new Uint8Array(32).fill(2) }, }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 11a1eb6..48ffc2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -386,9 +386,6 @@ importers: specifier: ^2.4.0 version: 2.6.1 devDependencies: - '@atmo-dev/contrail-community': - specifier: workspace:* - version: link:../contrail-community '@cloudflare/workers-types': specifier: ^4.20250124.0 version: 4.20260424.1 diff --git a/refs/spaces-later.md b/refs/spaces-later.md index 270b55c..47a0714 100644 --- a/refs/spaces-later.md +++ b/refs/spaces-later.md @@ -4,26 +4,65 @@ Deferred items from the spaces design review. Not blocking shipping; keep an eye on these as usage grows or as the permissioned-data spec firms up. See also [spaces-spec-mapping.md](./spaces-spec-mapping.md). +Items resolved by the six-phase refactor (credential flow, host/authority +split, enrollment, community as separate package, etc.) have been removed +from this list — see the spec-mapping doc for the post-refactor state. + ## Hydrated members endpoint -`space.listMembers` today returns raw `{did, perms, addedAt, addedBy}` rows. + +`space.listMembers` today returns raw `{did, addedAt, addedBy}` rows. Every client ends up wanting profile hydration (handle, displayName, avatar). Add `space.getMembers` (or extend `listMembers`) with: - Cursor-based pagination (current endpoint is unbounded) - Optional `hydrate=true` that joins against the configured profile collection - Sort options (joined-at, alphabetical by handle) +## DID-doc publication helper + +The authority's signing key needs to be published in its DID document under +`#atproto_space_authority` (verification method) — that's how external +verifiers find it. Today the deployer does this manually: +- For `did:web:contrail.example.com`: edit `.well-known/did.json`. +- For `did:plc`: a PLC operation signed with the rotation key. + +A `contrail authority publish-key` CLI subcommand could: +- For `did:web`, emit the verification-method JSON to stdout for the deployer to drop into their DID doc. +- For `did:plc`, build and submit the PLC operation. + +Without this, external verifiers can't validate the authority's credentials. +The in-process default works either way (host has the key directly). + +## Auto-wiring discovery resolvers + +`createPdsBindingResolver` and `createDidDocBindingResolver` are exported but +not wired into the default verifier. Today the in-process verifier uses only +`Local` (configured authority) + `Enrollment` (locally consented spaces). + +For deployments that want to accept credentials from any authority that's +properly bound on a user's PDS or DID doc, the deployer composes the +resolvers manually (see [deployment-shapes](../docs/10-deployment-shapes.md)). + +A higher-level config knob — `spaces.recordHost.acceptExternalAuthorities: true` +or similar — could auto-wire the full discovery chain. Decide whether the +fast-path (Local+Enrollment only) or the universal-path (full chain) is the +right default once we have real cross-host deployments. + ## More tests -The e2e + invite tests cover the happy paths. Gaps: -- Non-owner calling `createSpace` (should succeed — anyone can create their - own) vs non-member trying to use someone else's space URI -- App policy enforcement in both `allow` and `deny` modes (clientId checks) -- `deleteRecord` by owner on another author's record -- Re-querying a soft-deleted space returns NotFound -- `leaveSpace` by owner (should error) -- `whoami` for owner, member, non-member + +Phase 3-5 added good coverage for credentials, binding, enrollment. Gaps that +predate the refactor and still apply: + +- Non-owner calling `createSpace` (should succeed — anyone can create their own). +- App policy enforcement in both `allow` and `deny` modes (clientId checks). +- `deleteRecord` by owner on another author's record (should fail). +- Re-querying a soft-deleted space returns NotFound. +- `leaveSpace` by owner (should error). +- `whoami` for owner, member, non-member, with and without community integration. ## Config-change behavior + What happens today if a deployment: + - Adds a new collection after spaces already contain data? The per-collection table (`spaces_records_<short>`) won't exist until schema init re-runs. `listCollections` swallows the missing-table error, but `putRecord` / @@ -36,42 +75,80 @@ What happens today if a deployment: Need a config-drift audit (or migration) story. ## Verify `clientId` actually flows through + `checkAccess` uses `ServiceAuth.clientId` for app policy checks. Confirm: + - JWT verifier actually extracts `client_id` from real atproto service tokens - (not just our test fixture) -- App policy with a populated `apps[]` blocks/allows correctly in practice -- Empty `apps[]` under `mode: "deny"` blocks everyone (is that what we want?) + (not just our test fixture). +- App policy with a populated `apps[]` blocks/allows correctly in practice. +- Empty `apps[]` under `mode: "deny"` blocks everyone (is that what we want?). If `clientId` is `undefined` in the wild, app policy is decorative. +App policy is also currently checked at credential-issuance time but not +enforced again on the record host. For very long-lived credentials (>2h), an +app removed from the allowlist could continue acting until expiry. The TTL is +the spec's revocation bound; live with it. + ## Join requests (spec-adjacent, not in spec) + The rough spec punts invite/onboarding mechanics to apps. A natural fit given our invite system: a fourth kind `request` where `redeem` creates a pending row for the owner to approve. Likely wants: + - `space.requestJoin` → creates pending row - `space.listJoinRequests` (owner) → pending rows - `space.approveJoinRequest` / `space.denyJoinRequest` -Should this live under `space.*` or a separate extras namespace? +Should this live under `<ns>.space.*` or `<ns>.spaceExt.*`? + +## Authority migration + +A space's authority can change in principle — the `recordHost.enroll` row maps +`spaceUri → authorityDid`, re-enroll with a new authority and credentials +from the new authority will start verifying. But: + +- Existing credentials from the old authority don't auto-revoke; they expire + within their TTL. +- The PDS-record / DID-doc discovery sources (if used) need updating in lockstep. +- No helper API for this — the deployer or owner does each step manually. -## Real-time: SSE / subscriptions -Every collaborative app wants "new records in this space, as they land." -The spec's sync model uses write-notifications through the space owner; we -don't have that yet. Lightweight interim: Server-Sent Events on -`space.subscribeRecords?spaceUri=&collection=`. Works for first-party apps -right away; swap to the real thing later. +A `<ns>.recordHost.transferAuthority` endpoint could automate the wire-level +parts (re-enroll, optionally short-circuit credential cache). -## Namespace split for contrail-specific extras — done -`space.invite.*` and `space.whoami` moved to `<ns>.spaceExt.*`. See -[spaces-spec-mapping.md § Contrail extras](./spaces-spec-mapping.md#contrail-extras-namespace-nsspaceext). -`leaveSpace` is still in `space.*` — revisit if the spec lands with different -self-remove semantics. +## Multi-authority spaces + +In principle the architecture allows several authorities to all sign for one +space (replication / failover scenarios). The host's enrollment is 1:1 today +(one authority per space) but could become 1:N with a small schema change. +Spec is silent. Defer until a real use case. ## Ownership transfer -Dropped for now. The space URI is `at://<ownerDid>/<type>/<key>` — owner DID + +Dropped for now. The space URI is `ats://<ownerDid>/<type>/<key>` — owner DID is baked into the URI, and every record/member/invite row keys off that URI string. Transferring would mean either rewriting every referencing row in a transaction (and breaking external refs to the old URI) or decoupling storage from URI with an internal stable space id (bigger refactor). Revisit once the spec pins down whether ownership transfer exists and what the URI authority is supposed to be post-transfer. + +## Real-time over credentials + +Realtime tickets are signed by `realtime.ticketSecret`, not by the space +authority. If the host/authority split goes far enough that they're operated +by different parties, the realtime ticket model may need rethinking — does +the host mint tickets it then validates itself, or does the authority issue +realtime grants the host honors? Today both run in one process so it doesn't +matter. + +--- + +## Resolved by the six-phase refactor (kept for history) + +- ~~Space-credential flow~~ — done in phase 3. +- ~~Binding resolution (PDS records, DID-doc service entries)~~ — done in phase 4. +- ~~Independent host/authority deployments~~ — done in phase 5. +- ~~Real-time SSE / subscriptions~~ — landed via the realtime module. +- ~~Namespace split for contrail-specific extras~~ — `<ns>.spaceExt.*` shipped. +- ~~Community as separate package~~ — done in phase 6 (`@atmo-dev/contrail-community`). diff --git a/refs/spaces-spec-mapping.md b/refs/spaces-spec-mapping.md index 293a425..70d697b 100644 --- a/refs/spaces-spec-mapping.md +++ b/refs/spaces-spec-mapping.md @@ -7,70 +7,71 @@ is this doc. The goal is to make it obvious, when the real spec lands, where contrail already lines up and where it needs to change. Contrail is a backend-in-a-bottle / simple appview, not a PDS. For permissioned -data it currently stores everything in its own database; the plan is to -switch permissioned reads to come from users' PDSes once the protocol-level -flow is shipped (same story we already have for public records via jetstream). +data it currently stores everything in its own database; the long-term plan +is to switch permissioned reads to come from users' PDSes once the +protocol-level flow is shipped (same story we already have for public records +via jetstream). + +The spaces implementation went through a six-phase refactor (phases 1–6, +documented in conversation history) that aligned the architecture with the +spec and split community out into its own package. This doc reflects the +post-refactor state. --- ## Concept-by-concept alignment -| Spec concept | Contrail | Alignment | Notes | -| ------------------------------ | ------------------------------------------------------------ | --------- | ------------------------------------------------------------------------------ | -| Space owner (DID) | `spaces.owner_did` | ✅ | 1:1 | -| Space type (NSID) | `spaces.type` | ✅ | 1:1 | -| Space key / skey | `spaces.key` | ✅ | TID-generated when caller omits it | -| Record addressing 6-tuple | `(owner, type, key, author-did, collection, rkey)` | ✅ | Storage is keyed by `(space_uri, did, rkey)`; `space_uri` encodes the first 3 | -| Single ACL = member list | `spaces_members (did)` | ✅ | Membership is binary: you're in or you're out. Owner is implicit member. No read/write tiering — apps filter writes themselves | -| Space credential (2–4h token) | _none; service-auth JWTs used directly_ | ❌ | Fine while contrail is a single appview. Add a shim when real PDS sync lands | -| App allow/deny | `appPolicy {mode, apps[]}` | ✅ | Matches spec's default-allow / default-deny model. Visible only to the owner | -| Permissioned repo per user | single DB (`spaces_records_<short>`) | ⚠️ | Structurally compatible — keyed per `(space, author)`. Federation is future | -| ECMH commit / sync log | _none_ | ❌ | Out of scope until federated sync exists | -| Pull-based sync, write notifs | _none_ | ❌ | Same | -| URI scheme | `at://<owner>/<type>/<key>` for spaces; records not exposed | ⚠️ | Spec floats `ats://`. We centralize construction in `src/core/spaces/uri.ts` | -| Authority model for record URI | sidestepped (records keyed, not URI-addressed) | ✅ | Spec is undecided; we don't commit either way | -| Managing app routing | _none (join-requests etc. not modeled yet)_ | ⚠️ | See [spaces-later.md](./spaces-later.md) | +| Spec concept | Contrail | Alignment | Notes | +| ------------------------------ | --------------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------- | +| Space owner (DID) | `spaces.owner_did` | ✅ | 1:1 | +| Space type (NSID) | `spaces.type` | ✅ | 1:1 | +| Space key / skey | `spaces.key` | ✅ | TID-generated when caller omits it | +| Record addressing 6-tuple | `(owner, type, key, author-did, collection, rkey)` | ✅ | Storage is keyed by `(space_uri, did, rkey)`; `space_uri` encodes the first 3 | +| `ats://` URI scheme | `ats://<owner>/<type>/<key>` | ✅ | Centralized in `src/core/spaces/uri.ts` | +| Single ACL = member list | `spaces_members (did)` | ✅ | Membership is binary. Owner is implicit member. No read/write tiering. | +| Member list `(did, read\|write)` tuples | binary membership only | ⚠️ | Spec says read/write tiered. Pragmatic divergence; flip if/when spec firms | +| Member list as PDS record | server-side state on authority | ⚠️ | Spec says published on owner's PDS, synced. Future migration; the authority caches today | +| Space credential (2–4h token) | ES256 JWTs via `<ns>.space.getCredential` / `refreshCredential` | ✅ | Phase 3. Default 2h TTL. `iss` = authority DID; signed with the authority's published key | +| Credential signed by owner key | signed by **issuer** DID; binding from PDS record / DID-doc / owner-self | ⚠️ ext | Spec says owner-key. We extend so user-owned DIDs can authorize a separate issuer without DID-doc surgery | +| App allow/deny | `appPolicy {mode, apps[]}` | ✅ | Matches default-allow / default-deny. Checked at credential issuance only | +| Discovery via DID doc | `#atproto_space_authority` service entry resolver | ✅ | Phase 4. Plus PDS-record fallback (extension) and owner-self fallback | +| Permissioned repo per user | single DB (`spaces_records_<short>`) | ⚠️ | Structurally compatible — keyed per `(space, author)`. Federation is future | +| ECMH commit / sync log | _none_ | ❌ | Out of scope until federated sync exists | +| Pull-based sync, write notifs | _none_ | ❌ | Same | +| Authority model for record URI | sidestepped (records keyed, not URI-addressed) | ✅ | Spec is undecided; we don't commit either way | +| Managing app routing | _none (join-requests etc. not modeled yet)_ | ⚠️ | See [spaces-later.md](./spaces-later.md) | +| Host/AppView split | `spaces.authority` + `spaces.recordHost` independently runnable | ➕ | Phase 5. Spec implies but doesn't fully model. See [deployment-shapes](../docs/10-deployment-shapes.md) | +| Enrollment as host consent | `<ns>.recordHost.enroll` + `record_host_enrollments` table | ➕ | Phase 5. Spec doesn't address consent — we add explicit binding registration | + +Legend: ✅ aligned · ⚠️ pragmatic divergence · ❌ unimplemented (deliberate) · ➕ extension over the spec --- ## Endpoints -All endpoints are emitted under `<config.namespace>.space.*` from templates in -`lexicon-templates/spaces/`. - -### Read -- `space.listSpaces` — caller's spaces (scope=member|owner) -- `space.getSpace` — metadata; supports `?inviteToken=` bearer read -- `space.listMembers` — members for a space (member/owner only) -- `space.listRecords` — space-scoped record listing; bearer-read supported -- `space.getRecord` — single record; bearer-read supported - -### Write -- `space.putRecord` -- `space.deleteRecord` - -### Owner-gated (space management) -- `space.createSpace` -- `space.addMember` -- `space.removeMember` -- `space.leaveSpace` — self-remove; owner cannot leave (extra) - -### Contrail extras (namespace: `<ns>.spaceExt.*`) -Clearly-off-spec features live under a separate namespace so the `space.*` -surface stays close to whatever the permissioned-data spec becomes. Moved here -from `space.*` in an earlier refactor. - -- `spaceExt.whoami` — caller's relationship to a space (owner / member flags) -- `spaceExt.invite.create` — returns raw token once; hash stored -- `spaceExt.invite.redeem` -- `spaceExt.invite.list` -- `spaceExt.invite.revoke` - -Invites have three kinds: `join`, `read`, `read-join`. `read` tokens grant -bearer-only anonymous read access; `read-join` does both; `join` requires a -signed-in caller and grants a membership row. None of this is in the spec — -it lives here because the spec explicitly defers invite/onboarding mechanics -to apps, and shipping a working invite primitive is useful for every consumer. +All endpoints are emitted under `<config.namespace>.*` from templates in +`packages/lexicons/lexicon-templates/`. + +### Authority (`<ns>.space.*` — spec-aligned) +- `createSpace` `getSpace` `listSpaces` `deleteSpace` +- `listMembers` `addMember` `removeMember` `leaveSpace` +- `getCredential` `refreshCredential` + +### Record host +- `<ns>.space.putRecord` `deleteRecord` `getRecord` `listRecords` +- `<ns>.space.uploadBlob` `getBlob` `listBlobs` (optional) +- `<ns>.recordHost.enroll` + +### Contrail extras (`<ns>.spaceExt.*`) +Clearly-off-spec features that don't map cleanly to the rough spec. +- `whoami` — caller's relationship to a space (owner / member / extension fields) + +### Invites (`<ns>.invite.*`) +- `create` `redeem` `revoke` `list` + +Invites have three kinds: `join`, `read`, `read-join`. Spec defers +invite/onboarding mechanics to apps; we ship a working primitive because +every consumer needs one. ### Collection integration Per-collection `listRecords` / `getRecord` accept `?spaceUri=` (space-scoped) @@ -79,44 +80,54 @@ public + own-member-spaces union (see `src/core/router/collection.ts`). --- +## What changed in the six-phase refactor + +| Phase | Brought us | Notes | +|---|---|---| +| 1 | `SpaceAuthority` + `RecordHost` interface boundary | Pure refactor; `StorageAdapter` is their union | +| 2 | Spaces no longer imports community | Whoami extension hook + `CommunityInviteHandler` interface | +| 3 | Credential issuance + verification | ES256 JWTs, `X-Space-Credential` header, in-process verifier | +| 4 | Binding resolution | PDS-record + DID-doc resolvers; `iss != owner` allowed via the binding | +| 5 | Independent deployment + enrollment | Authority and host runnable as separate processes; `recordHost.enroll` consent | +| 6 | Community as separate package | `@atmo-dev/contrail-community` with integration interface | + +After phase 6 the architecture maps to the spec roughly as: + +``` + spec concept contrail mapping + ──────────── ──────────────── + "space host" ─→ space authority (signs creds, holds members) + "permissioned repo" ─→ record host (stores records, enrolls spaces) + "external space hosts" ─→ binding resolver chain (multi-authority support) + "managing app routing" ─→ not yet (deferred — spaces-later.md) +``` + +--- + ## Migration readiness -Hasn't shipped yet → nothing to migrate, but the shape of what changes when -the real spec lands: - -1. **URI scheme swap (if any).** Centralized in `src/core/spaces/uri.ts` — - flip `at://` to `ats://` (or whatever) in two helpers and every caller - follows. -2. **Space-credential flow.** Needs an endpoint that mints short-lived tokens - from an owner key, and a verifier that accepts them in place of a - service-auth JWT on read paths. The current JWT middleware - (`src/core/spaces/auth.ts`) is the right anchor for this. -3. **Read records from PDSes.** Mirrors the jetstream ingestion we already do - for public data: consume permissioned-repo sync, write into the same - `spaces_records_<short>` tables. The storage schema is already keyed per - `(space, author)` so no migration needed on that side. -4. **ECMH commits & sync log.** Greenfield; unrelated to existing storage. -5. **Endpoint naming.** Spec doesn't pin XRPC names. When it does, rename - lexicon template files + routes. No storage churn. - -### Design decisions worth preserving -- Keep the member list as the single ACL. Don't add roles or per-collection - policies just because it's easy — the spec is emphatic that the member list - is _the_ ACL. -- Membership is binary, not tiered. Previously had `perms: "read" | "write"` - per member row; collapsed to plain membership because the rough spec is - moving toward "member = access, apps filter writes." Delete keeps the - owner / own-record rule, but that's about *which records you can affect*, - not a permission tier on the member row. -- Don't over-engineer the space row with pre-emptive extension columns. - Previously had `member_list_ref` as a hook for externally-managed - membership; dropped because the community-module case is handled via - ownership (community-owned spaces are managed by the community module, no - flag column needed). If a future need for external membership sources - shows up, add the column then. -- Keep `space.whoami`, `space.leaveSpace`, and the invite endpoints clearly - labeled as contrail extras in docs. If the spec ends up naming some of - them, renaming is cheap; relying on them from the base spec isn't. -- Don't mint a canonical record URI. The spec is undecided on the authority - (user DID vs space owner DID); storing records by tuple avoids picking. +What still needs to change when the real spec lands: + +1. **Member list moves to PDS records.** The spec says it's a record on the owner's PDS, synced. Our authority holds it server-side. Future: a watcher consumes member-list records via Jetstream and reconciles into `spaces_members`. Auth-side `addMember` becomes a PDS write rather than an internal API. + +2. **Records federate from user PDSes.** Today the record host *is* the source of truth. When permissioned-repos ship, records federate; the host becomes an aggregator. Storage schema (`spaces_records_<short>`, keyed per `(space, author)`) already supports this — the change is in the write path, not the read path. + +3. **ECMH commits & sync log.** Greenfield. Required for federation. + +4. **Endpoint naming.** Spec doesn't pin XRPC names. When it does, rename lexicon template files + routes. No storage churn. + +5. **Possibly: `(did, read|write)` member tuples.** If the spec stays at tiered membership and doesn't move to binary, add an `access` column on `spaces_members` and branch the ACL check in `acl.ts`. One-day change. + +6. **Possibly: credential `iss = owner DID`.** If the spec forbids the issuer-DID indirection we use for user-owned DIDs, fall back to "owner adds a host-controlled verification method to their DID doc" (HappyView's hidden assumption). Operationally heavier; we hold the looser reading until forced to tighten. + +--- + +## Design decisions worth preserving +- **Keep the member list as the single ACL.** Don't add roles or per-collection policies just because it's easy — the spec is emphatic that the member list is _the_ ACL. +- **Membership is binary, not tiered.** Previously had `perms: "read" | "write"` per member row; collapsed to plain membership because the rough spec is moving toward "member = access, apps filter writes." Delete keeps the owner / own-record rule, but that's about *which records you can affect*, not a permission tier on the member row. +- **Don't over-engineer the space row with pre-emptive extension columns.** Previously had `member_list_ref` as a hook for externally-managed membership; dropped because the community-module case is handled via ownership (community-owned spaces are managed by the community module, no flag column needed). If a future need for external membership sources shows up, add the column then. +- **Keep `spaceExt.whoami`, `space.leaveSpace`, and the invite endpoints clearly labeled as contrail extras in docs.** If the spec ends up naming some of them, renaming is cheap; relying on them from the base spec isn't. +- **Don't mint a canonical record URI.** The spec is undecided on the authority (user DID vs space owner DID); storing records by tuple avoids picking. +- **Enrollment is the host's source of truth.** Even when PDS records and DID-doc service entries declare authority bindings, the host's local enrollment is what actually gates record acceptance. Keeps the host's consent explicit and prevents abuse of the open-ended discovery layer. +- **Keep the issuer-DID indirection as an extension, not a hard architectural choice.** The credential verifier supports `iss == owner` (literal-spec) and `iss != owner` (with binding). If the spec forbids the latter, we degrade gracefully. -- 2.51.2 From 0e9378293f595e8bc8c409734f82c93240839ebd Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 00:56:35 +0200 Subject: [PATCH 08/25] uff, split stuff up --- packages/contrail-base/package.json | 67 +++ .../contrail-base/src/adapters/postgres.ts | 93 +++ packages/contrail-base/src/adapters/sqlite.ts | 39 ++ packages/contrail-base/src/client.ts | 194 ++++++ .../src/community-integration.ts | 67 +++ packages/contrail-base/src/dialect.ts | 127 ++++ packages/contrail-base/src/identity.ts | 159 +++++ packages/contrail-base/src/index.ts | 67 +++ .../src/invite/community-handler.ts | 67 +++ packages/contrail-base/src/invite/token.ts | 43 ++ packages/contrail-base/src/labels/types.ts | 56 ++ .../src/realtime/durable-object.ts | 567 +++++++++++++++++ .../contrail-base/src/realtime/in-memory.ts | 116 ++++ packages/contrail-base/src/realtime/merge.ts | 77 +++ .../src/realtime/query-filter.ts | 235 ++++++++ packages/contrail-base/src/realtime/sse.ts | 98 +++ packages/contrail-base/src/realtime/ticket.ts | 179 ++++++ packages/contrail-base/src/realtime/types.ts | 127 ++++ .../contrail-base/src/realtime/websocket.ts | 84 +++ packages/contrail-base/src/spaces/acl.ts | 69 +++ packages/contrail-base/src/spaces/auth.ts | 177 ++++++ packages/contrail-base/src/spaces/binding.ts | 274 +++++++++ .../contrail-base/src/spaces/blob-adapter.ts | 94 +++ .../contrail-base/src/spaces/credentials.ts | 284 +++++++++ .../contrail-base/src/spaces/in-process.ts | 34 ++ packages/contrail-base/src/spaces/tid.ts | 20 + packages/contrail-base/src/spaces/types.ts | 273 +++++++++ packages/contrail-base/src/spaces/uri.ts | 37 ++ packages/contrail-base/src/types.ts | 513 ++++++++++++++++ packages/contrail-base/tsconfig.build.json | 7 + packages/contrail-base/tsconfig.json | 7 + packages/contrail-base/tsup.config.ts | 11 + packages/contrail-community/package.json | 1 + packages/contrail-community/vitest.config.ts | 9 +- packages/contrail/package.json | 1 + packages/contrail/src/adapters/postgres.ts | 94 +-- packages/contrail/src/adapters/sqlite.ts | 40 +- packages/contrail/src/core/backfill.ts | 1 + packages/contrail/src/core/client.ts | 195 +----- .../src/core/community-integration.ts | 55 +- packages/contrail/src/core/dialect.ts | 128 +--- packages/contrail/src/core/identity.ts | 160 +---- .../src/core/invite/community-handler.ts | 68 +-- packages/contrail/src/core/invite/token.ts | 44 +- packages/contrail/src/core/labels/types.ts | 60 +- .../src/core/realtime/durable-object.ts | 568 +----------------- .../contrail/src/core/realtime/in-memory.ts | 117 +--- packages/contrail/src/core/realtime/merge.ts | 78 +-- .../src/core/realtime/query-filter.ts | 236 +------- packages/contrail/src/core/realtime/sse.ts | 99 +-- packages/contrail/src/core/realtime/ticket.ts | 180 +----- packages/contrail/src/core/realtime/types.ts | 128 +--- .../contrail/src/core/realtime/websocket.ts | 85 +-- packages/contrail/src/core/refresh.ts | 1 + packages/contrail/src/core/spaces/acl.ts | 70 +-- packages/contrail/src/core/spaces/auth.ts | 178 +----- packages/contrail/src/core/spaces/binding.ts | 275 +-------- .../contrail/src/core/spaces/blob-adapter.ts | 95 +-- .../contrail/src/core/spaces/credentials.ts | 285 +-------- .../contrail/src/core/spaces/in-process.ts | 35 +- packages/contrail/src/core/spaces/tid.ts | 21 +- packages/contrail/src/core/spaces/types.ts | 274 +-------- packages/contrail/src/core/spaces/uri.ts | 38 +- packages/contrail/src/core/types.ts | 514 +--------------- packages/contrail/vitest.config.ts | 12 + pnpm-lock.yaml | 49 ++ 66 files changed, 4360 insertions(+), 4096 deletions(-) create mode 100644 packages/contrail-base/package.json create mode 100644 packages/contrail-base/src/adapters/postgres.ts create mode 100644 packages/contrail-base/src/adapters/sqlite.ts create mode 100644 packages/contrail-base/src/client.ts create mode 100644 packages/contrail-base/src/community-integration.ts create mode 100644 packages/contrail-base/src/dialect.ts create mode 100644 packages/contrail-base/src/identity.ts create mode 100644 packages/contrail-base/src/index.ts create mode 100644 packages/contrail-base/src/invite/community-handler.ts create mode 100644 packages/contrail-base/src/invite/token.ts create mode 100644 packages/contrail-base/src/labels/types.ts create mode 100644 packages/contrail-base/src/realtime/durable-object.ts create mode 100644 packages/contrail-base/src/realtime/in-memory.ts create mode 100644 packages/contrail-base/src/realtime/merge.ts create mode 100644 packages/contrail-base/src/realtime/query-filter.ts create mode 100644 packages/contrail-base/src/realtime/sse.ts create mode 100644 packages/contrail-base/src/realtime/ticket.ts create mode 100644 packages/contrail-base/src/realtime/types.ts create mode 100644 packages/contrail-base/src/realtime/websocket.ts create mode 100644 packages/contrail-base/src/spaces/acl.ts create mode 100644 packages/contrail-base/src/spaces/auth.ts create mode 100644 packages/contrail-base/src/spaces/binding.ts create mode 100644 packages/contrail-base/src/spaces/blob-adapter.ts create mode 100644 packages/contrail-base/src/spaces/credentials.ts create mode 100644 packages/contrail-base/src/spaces/in-process.ts create mode 100644 packages/contrail-base/src/spaces/tid.ts create mode 100644 packages/contrail-base/src/spaces/types.ts create mode 100644 packages/contrail-base/src/spaces/uri.ts create mode 100644 packages/contrail-base/src/types.ts create mode 100644 packages/contrail-base/tsconfig.build.json create mode 100644 packages/contrail-base/tsconfig.json create mode 100644 packages/contrail-base/tsup.config.ts diff --git a/packages/contrail-base/package.json b/packages/contrail-base/package.json new file mode 100644 index 0000000..e72201a --- /dev/null +++ b/packages/contrail-base/package.json @@ -0,0 +1,67 @@ +{ + "name": "@atmo-dev/contrail-base", + "version": "0.6.0", + "description": "Shared infrastructure for the contrail family of packages — interfaces (SpaceAuthority, RecordHost, CommunityIntegration), credential primitives, binding resolvers, realtime infra, schema scaffolding. No routes, no tables of its own.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./sqlite": { + "types": "./dist/adapters/sqlite.d.ts", + "import": "./dist/adapters/sqlite.js" + }, + "./postgres": { + "types": "./dist/adapters/postgres.d.ts", + "import": "./dist/adapters/postgres.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-base" + }, + "keywords": [ + "atproto", + "contrail" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/atproto": "^3.1.10", + "@atcute/cid": "^2.4.1", + "@atcute/client": "^4.2.1", + "@atcute/identity": "^1.1.4", + "@atcute/identity-resolver": "^1.2.2", + "@atcute/lexicons": "^1.2.9", + "@atcute/xrpc-server": "^0.1.12", + "hono": "^4.12.8" + }, + "devDependencies": { + "@types/node": "^25.5.0", + "@types/pg": "^8.20.0", + "pg": "^8.20.0", + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "peerDependencies": { + "pg": "^8.0.0" + }, + "peerDependenciesMeta": { + "pg": { + "optional": true + } + }, + "license": "MIT" +} diff --git a/packages/contrail-base/src/adapters/postgres.ts b/packages/contrail-base/src/adapters/postgres.ts new file mode 100644 index 0000000..e8c2b14 --- /dev/null +++ b/packages/contrail-base/src/adapters/postgres.ts @@ -0,0 +1,93 @@ +import pg from "pg"; +import type { Database, Statement } from "../types"; +import { postgresDialect } from "../dialect"; + +/** Internal interface for statements that can run on a specific client */ +interface PgStatement extends Statement { + /** Execute on a specific client (used by batch for transaction isolation) */ + _runOn(client: pg.PoolClient): Promise<any>; +} + +/** Column names known to be BIGINT — PostgreSQL returns these as strings */ +const BIGINT_COLUMNS = new Set(["time_us", "indexed_at", "resolved_at"]); + +function normalizeRow(row: any): any { + if (!row) return row; + if (typeof row.record === "object" && row.record !== null) { + row.record = JSON.stringify(row.record); + } + for (const col of BIGINT_COLUMNS) { + if (typeof row[col] === "string") row[col] = Number(row[col]); + } + return row; +} + +export function createPostgresDatabase(pool: pg.Pool): Database { + function rewritePlaceholders(sql: string): string { + let idx = 0; + let inString = false; + let result = ""; + for (let i = 0; i < sql.length; i++) { + const ch = sql[i]; + if (ch === "'" && sql[i - 1] !== "\\") { + inString = !inString; + result += ch; + } else if (ch === "?" && !inString) { + result += `$${++idx}`; + } else { + result += ch; + } + } + return result; + } + + function wrapStatement(sql: string, boundValues: any[] = []): PgStatement { + const pgSql = rewritePlaceholders(sql); + + return { + bind(...values: any[]): PgStatement { + return wrapStatement(sql, values); + }, + async run() { + const result = await pool.query(pgSql, boundValues); + return { changes: result.rowCount }; + }, + async _runOn(client: pg.PoolClient) { + const result = await client.query(pgSql, boundValues); + return { changes: result.rowCount }; + }, + async all<T>() { + const result = await pool.query(pgSql, boundValues); + return { results: result.rows.map(normalizeRow) as T[] }; + }, + async first<T>() { + const result = await pool.query(pgSql, boundValues); + return result.rows[0] ? (normalizeRow(result.rows[0]) as T) : null; + }, + }; + } + + return { + prepare(sql: string): Statement { + return wrapStatement(sql); + }, + async batch(stmts: Statement[]): Promise<any[]> { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + const results: any[] = []; + for (const stmt of stmts) { + results.push(await (stmt as PgStatement)._runOn(client)); + } + await client.query("COMMIT"); + return results; + } catch (e) { + await client.query("ROLLBACK"); + throw e; + } finally { + client.release(); + } + }, + dialect: postgresDialect, + }; +} diff --git a/packages/contrail-base/src/adapters/sqlite.ts b/packages/contrail-base/src/adapters/sqlite.ts new file mode 100644 index 0000000..305dc39 --- /dev/null +++ b/packages/contrail-base/src/adapters/sqlite.ts @@ -0,0 +1,39 @@ +import { DatabaseSync } from "node:sqlite"; +import type { Database, Statement } from "../types"; +import { sqliteDialect } from "../dialect"; + +export function createSqliteDatabase(path: string): Database { + const raw = new DatabaseSync(path); + raw.exec("PRAGMA journal_mode = WAL"); + + function wrapStatement(sql: string, boundValues: any[] = []): Statement { + return { + bind(...values: any[]): Statement { + return wrapStatement(sql, values); + }, + async run() { + return raw.prepare(sql).run(...boundValues); + }, + async all<T>() { + return { results: raw.prepare(sql).all(...boundValues) as T[] }; + }, + async first<T>() { + return (raw.prepare(sql).get(...boundValues) as T) ?? null; + }, + }; + } + + return { + prepare(sql: string): Statement { + return wrapStatement(sql); + }, + async batch(stmts: Statement[]): Promise<any[]> { + const results: any[] = []; + for (const stmt of stmts) { + results.push(await stmt.run()); + } + return results; + }, + dialect: sqliteDialect, + }; +} diff --git a/packages/contrail-base/src/client.ts b/packages/contrail-base/src/client.ts new file mode 100644 index 0000000..f37bea3 --- /dev/null +++ b/packages/contrail-base/src/client.ts @@ -0,0 +1,194 @@ +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, +} from "@atcute/identity-resolver"; +import { type Did } from "@atcute/lexicons"; +import { Client, simpleFetchHandler } from "@atcute/client"; +import type {} from "@atcute/atproto"; +import type { Database } from "./types"; + +// Slingshot-first PDS resolution with fallback to DID document resolution +const SLINGSHOT_URL = + "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; + +export interface ResolvedIdentity { + did: string; + handle: string | null; + pds: string | null; +} + +/** Reject PDS URLs that point to private/internal addresses or non-HTTPS */ +function validatePdsUrl(url: string): boolean { + try { + const parsed = new URL(url); + if (parsed.protocol !== "https:") return false; + const host = parsed.hostname; + // Block private/internal IP ranges + if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; + if (host.startsWith("10.")) return false; + if (host.startsWith("192.168.")) return false; + if (host.startsWith("169.254.")) return false; + if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; + return true; + } catch { + return false; + } +} + +async function resolveViaSlingshot( + identifier: string +): Promise<ResolvedIdentity | undefined> { + const url = new URL(SLINGSHOT_URL); + url.searchParams.set("identifier", identifier); + + try { + const response = await fetch(url.toString()); + if (!response.ok) return undefined; + const data = (await response.json()) as { + did?: string; + handle?: string; + pds?: string; + }; + if (!data.did && !data.pds) return undefined; + return { + did: data.did ?? identifier, + handle: data.handle ?? null, + pds: data.pds ?? null, + }; + } catch { + return undefined; + } +} + +const didResolver = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, +}); + +async function getPDSViaDidDoc(did: Did): Promise<string | undefined> { + const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); + return doc.service + ?.find((s) => s.id === "#atproto_pds") + ?.serviceEndpoint.toString(); +} + +/** + * Resolve identity info (did, handle, pds) for a DID or handle. + * Uses slingshot first, falls back to DID doc for PDS. + */ +export async function resolvePDS( + identifier: string +): Promise<ResolvedIdentity | undefined> { + const result = await resolveViaSlingshot(identifier); + if (result?.pds) { + if (!validatePdsUrl(result.pds)) return { ...result, pds: null }; + return result; + } + + // Fall back to DID doc resolution (only works for DIDs, not handles) + if (identifier.startsWith("did:")) { + try { + const pds = await getPDSViaDidDoc(identifier as Did); + if (pds && validatePdsUrl(pds)) { + return { + did: identifier, + handle: result?.handle ?? null, + pds, + }; + } + } catch { + // ignore + } + } + + return result; +} + +// In-memory PDS cache with TTL + size limit, plus in-flight deduplication +const PDS_CACHE_TTL = 60 * 60 * 1000; // 1 hour +const PDS_CACHE_MAX = 10_000; +const pdsCache = new Map<string, { pds: string; at: number }>(); +const pdsInflight = new Map<string, Promise<string | undefined>>(); + +function pdsCacheGet(did: string): string | undefined { + const entry = pdsCache.get(did); + if (!entry) return undefined; + if (Date.now() - entry.at > PDS_CACHE_TTL) { + pdsCache.delete(did); + return undefined; + } + return entry.pds; +} + +function pdsCacheSet(did: string, pds: string): void { + // Evict oldest entries if over limit + if (pdsCache.size >= PDS_CACHE_MAX) { + const first = pdsCache.keys().next().value; + if (first) pdsCache.delete(first); + } + pdsCache.set(did, { pds, at: Date.now() }); +} + +export async function getPDS( + did: Did, + db?: Database +): Promise<string | undefined> { + const mem = pdsCacheGet(did); + if (mem) return mem; + + // Deduplicate concurrent calls for the same DID + const inflight = pdsInflight.get(did); + if (inflight) return inflight; + + const promise = resolvePDSCached(did, db); + pdsInflight.set(did, promise); + try { + return await promise; + } finally { + pdsInflight.delete(did); + } +} + +async function resolvePDSCached( + did: Did, + db?: Database +): Promise<string | undefined> { + if (db) { + const cached = await db + .prepare("SELECT pds FROM identities WHERE did = ? AND pds IS NOT NULL") + .bind(did) + .first<{ pds: string }>(); + if (cached?.pds) { + pdsCacheSet(did, cached.pds); + return cached.pds; + } + } + + const resolved = await resolvePDS(did); + if (!resolved?.pds) return undefined; + + pdsCacheSet(did, resolved.pds); + + // Persist to DB for future runs + if (db) { + await db + .prepare( + "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET pds = excluded.pds, handle = COALESCE(excluded.handle, identities.handle), resolved_at = excluded.resolved_at" + ) + .bind(did, resolved.handle, resolved.pds, Date.now()) + .run(); + } + + return resolved.pds; +} + +export async function getClient(did: Did, db?: Database): Promise<Client> { + const pds = await getPDS(did, db); + if (!pds) throw new Error(`PDS not found for ${did}`); + return new Client({ + handler: simpleFetchHandler({ service: pds }), + }); +} diff --git a/packages/contrail-base/src/community-integration.ts b/packages/contrail-base/src/community-integration.ts new file mode 100644 index 0000000..bc2f936 --- /dev/null +++ b/packages/contrail-base/src/community-integration.ts @@ -0,0 +1,67 @@ +/** Pluggable integration surface for the community module. + * + * Phase 6 extracted community to its own package (`@atmo-dev/contrail-community`). + * The contrail core package never imports from it — couplings only flow + * through these interfaces. The community package's + * `createCommunityIntegration({ ... })` returns a {@link CommunityIntegration}, + * which the consumer hands to `createApp` via `options.community`. + * + * Two layers: + * - {@link CommunityProbe}: minimal "is this a community DID" / "what + * spaces does this caller reach" surface used by realtime + collection + * routes for community-aware dispatch. + * - {@link CommunityIntegration}: the umbrella bundle — probe, whoami + * extension, invite handler, plus route + schema wiring that the + * umbrella router calls during setup. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { Database } from "./types"; +import type { CommunityInviteHandler } from "./invite/community-handler"; + +/** Optional hook to extend `<ns>.spaceExt.whoami` with extra fields when a + * module above spaces (e.g. community) wants to override the default + * binary-membership response. If the hook returns a non-null object, that + * object is the entire response body. If null, falls through to the + * default behavior (just `isOwner`/`isMember`). + * + * Spaces stays community-agnostic: any consumer can plug in here. */ +export type WhoamiExtension = (input: { + spaceUri: string; + callerDid: string; + isOwner: boolean; + ownerDid: string; +}) => Promise<Record<string, unknown> | null>; + +/** Narrow interface for the deep callers (realtime/resolve, router/collection) + * that just need to ask "is this a community DID?" or "what spaces does this + * caller reach via community membership?" */ +export interface CommunityProbe { + /** Look up a community row by DID. Returns null for non-community DIDs. + * Callers usually only check truthiness — community-specific fields stay + * inside the community package. */ + getCommunity(did: string): Promise<{ did: string } | null>; + + /** Resolve the set of space URIs reachable by `callerDid` through community + * membership (direct grants + delegations). Used by realtime to expand + * community: topics into the caller's concrete space: topics. */ + resolveReachableSpaces(callerDid: string): Promise<Set<string>>; +} + +/** Umbrella integration the consumer constructs once and hands to createApp. + * contrail core treats this as an opaque bundle — it doesn't introspect + * community state, just calls these methods at the right wiring points. */ +export interface CommunityIntegration { + /** Probe used by realtime + collection cross-cutting concerns. */ + probe: CommunityProbe; + /** Whoami extension that returns `accessLevel` for community-owned spaces. */ + whoamiExtension: WhoamiExtension; + /** Handler for the community-grant path of the unified invite surface. */ + inviteHandler: CommunityInviteHandler; + /** Register `<ns>.community.*` routes onto the Hono app. */ + registerRoutes( + app: Hono, + options?: { authMiddleware?: MiddlewareHandler } + ): void; + /** Apply community schema (DDL) to the database. Called by initSchema. */ + applySchema(db: Database): Promise<void>; +} diff --git a/packages/contrail-base/src/dialect.ts b/packages/contrail-base/src/dialect.ts new file mode 100644 index 0000000..e3bb790 --- /dev/null +++ b/packages/contrail-base/src/dialect.ts @@ -0,0 +1,127 @@ +/** Get the dialect from a Database, defaulting to SQLite (for D1 compatibility) */ +export function getDialect(db: { dialect?: SqlDialect }): SqlDialect { + return db.dialect ?? sqliteDialect; +} + +const SAFE_FIELD = /^[a-zA-Z0-9_.]+$/; + +function assertSafeField(field: string): void { + if (!SAFE_FIELD.test(field)) { + throw new Error(`Invalid field name: ${field}`); + } +} + +export interface SqlDialect { + /** json_extract(col, '$.field') or col->>'field' */ + jsonExtract(column: string, field: string): string; + + /** Convert INSERT INTO to ignore-duplicates form. + * SQLite: INSERT INTO → INSERT OR IGNORE INTO + * PG: appends ON CONFLICT DO NOTHING + * Accepts full SQL starting with "INSERT INTO" (works with both VALUES and SELECT). */ + insertOrIgnore(sql: string): string; + + /** Column type for the record column: TEXT (SQLite) or JSONB (PostgreSQL) */ + readonly recordColumnType: string; + + /** FTS strategy: 'virtual-table' (SQLite FTS5) or 'generated-column' (PG tsvector) */ + readonly ftsStrategy: "virtual-table" | "generated-column"; + + /** INTEGER type name — same on both, but PostgreSQL may want BIGINT for time_us */ + readonly integerType: string; + + /** BIGINT type name for timestamps */ + readonly bigintType: string; + + /** Wrap an expression for use in CREATE INDEX — PostgreSQL requires parens around expressions */ + indexExpression(expr: string): string; +} + +export const sqliteDialect: SqlDialect = { + jsonExtract(column: string, field: string): string { + assertSafeField(field); + return `json_extract(${column}, '$.${field}')`; + }, + + insertOrIgnore(sql: string): string { + return sql.replace(/^INSERT INTO/, "INSERT OR IGNORE INTO"); + }, + + recordColumnType: "TEXT", + ftsStrategy: "virtual-table", + integerType: "INTEGER", + bigintType: "INTEGER", + + indexExpression(expr: string): string { + return expr; + }, +}; + +export const postgresDialect: SqlDialect = { + jsonExtract(column: string, field: string): string { + assertSafeField(field); + const parts = field.split("."); + if (parts.length === 1) { + return `${column}->>'${parts[0]}'`; + } + // a.b.c → col->'a'->'b'->>'c' + const intermediate = parts.slice(0, -1).map((p) => `->'${p}'`).join(""); + return `${column}${intermediate}->>'${parts[parts.length - 1]}'`; + }, + + insertOrIgnore(sql: string): string { + return `${sql} ON CONFLICT DO NOTHING`; + }, + + recordColumnType: "JSONB", + ftsStrategy: "generated-column", + integerType: "INTEGER", + bigintType: "BIGINT", + + indexExpression(expr: string): string { + return `(${expr})`; + }, +}; + +/** Generate FTS schema statements based on dialect */ +export function buildFtsSchema( + dialect: SqlDialect, + recordsTable: string, + fields: string[] +): string[] { + if (dialect.ftsStrategy === "virtual-table") { + const ftsTable = recordsTable.replace("records_", "fts_"); + return [ + `CREATE VIRTUAL TABLE IF NOT EXISTS ${ftsTable} USING fts5(uri UNINDEXED, content)` + ]; + } else { + const concatExpr = fields + .map((f) => `COALESCE(${dialect.jsonExtract("record", f)}, '')`) + .join(" || ' ' || "); + return [ + `ALTER TABLE ${recordsTable} ADD COLUMN IF NOT EXISTS search_vector TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', ${concatExpr})) STORED`, + `CREATE INDEX IF NOT EXISTS idx_${recordsTable}_search ON ${recordsTable} USING GIN (search_vector)`, + ]; + } +} + +/** Generate FTS query clause based on dialect */ +export function ftsQueryClause( + dialect: SqlDialect, + recordsTable: string +): { join: string; condition: string; orderExpr: string } { + if (dialect.ftsStrategy === "virtual-table") { + const ftsTable = recordsTable.replace("records_", "fts_"); + return { + join: `JOIN ${ftsTable} fts ON fts.uri = r.uri`, + condition: "fts.content MATCH ?", + orderExpr: "fts.rank", + }; + } else { + return { + join: "", + condition: "r.search_vector @@ plainto_tsquery('english', ?)", + orderExpr: "ts_rank(r.search_vector, plainto_tsquery('english', ?))", + }; + } +} diff --git a/packages/contrail-base/src/identity.ts b/packages/contrail-base/src/identity.ts new file mode 100644 index 0000000..05040fd --- /dev/null +++ b/packages/contrail-base/src/identity.ts @@ -0,0 +1,159 @@ +import type { Did } from "@atcute/lexicons"; +import type { Database, Logger } from "./types"; +import { isDid, isHandle } from "@atcute/lexicons/syntax"; +import { resolvePDS } from "./client"; + +const STALE_MS = 24 * 60 * 60 * 1000; // 24 hours + +export interface Identity { + did: string; + handle: string | null; + pds: string | null; + resolved_at: number; +} + +async function saveIdentity(db: Database, identity: Identity): Promise<void> { + await db + .prepare( + "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET handle = excluded.handle, pds = excluded.pds, resolved_at = excluded.resolved_at" + ) + .bind(identity.did, identity.handle, identity.pds, identity.resolved_at) + .run(); +} + +function isStale(resolvedAt: number): boolean { + return Date.now() - resolvedAt >= STALE_MS; +} + +async function fetchAndSave( + db: Database, + identifier: string, + cached?: Identity | null +): Promise<Identity> { + const resolved = await resolvePDS(identifier); + const identity: Identity = { + did: resolved?.did ?? identifier, + handle: resolved?.handle ?? cached?.handle ?? null, + pds: resolved?.pds ?? cached?.pds ?? null, + resolved_at: Date.now(), + }; + await saveIdentity(db, identity); + return identity; +} + +export async function resolveIdentity( + db: Database, + did: Did +): Promise<Identity> { + const cached = await db + .prepare("SELECT did, handle, pds, resolved_at FROM identities WHERE did = ?") + .bind(did) + .first<Identity>(); + + if (cached && !isStale(cached.resolved_at)) return cached; + + return fetchAndSave(db, did, cached); +} + +export async function resolveIdentities( + db: Database, + dids: string[] +): Promise<Map<string, Identity>> { + const map = new Map<string, Identity>(); + if (dids.length === 0) return map; + + // Batch lookup from DB + const BATCH = 50; + for (let i = 0; i < dids.length; i += BATCH) { + const chunk = dids.slice(i, i + BATCH); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare(`SELECT did, handle, pds, resolved_at FROM identities WHERE did IN (${placeholders})`) + .bind(...chunk) + .all<Identity>(); + for (const row of rows.results ?? []) { + map.set(row.did, row); + } + } + + // Resolve missing via slingshot directly (no redundant DB lookup) + for (const did of dids) { + if (map.has(did) || !isDid(did)) continue; + try { + const identity = await fetchAndSave(db, did); + map.set(did, identity); + } catch { + // Silently skip unresolvable identities + } + } + + return map; +} + +export async function resolveActor( + db: Database, + actor: string +): Promise<string | null> { + if (isDid(actor)) return actor; + if (!isHandle(actor)) return null; + + // Look up handle in identities table + const cached = await db + .prepare("SELECT did, resolved_at FROM identities WHERE handle = ?") + .bind(actor) + .first<{ did: string; resolved_at: number }>(); + + if (cached && !isStale(cached.resolved_at)) return cached.did; + + // Resolve via slingshot + const resolved = await resolvePDS(actor); + if (!resolved?.did || !isDid(resolved.did)) return null; + + await saveIdentity(db, { + did: resolved.did, + handle: resolved.handle ?? actor, + pds: resolved.pds ?? null, + resolved_at: Date.now(), + }); + + return resolved.did; +} + +export async function refreshStaleIdentities( + db: Database, + dids: string[] +): Promise<void> { + if (dids.length === 0) return; + + const unique = [...new Set(dids)].filter(isDid); + if (unique.length === 0) return; + + const staleThreshold = Date.now() - STALE_MS; + const toRefresh: string[] = []; + + const BATCH = 50; + for (let i = 0; i < unique.length; i += BATCH) { + const chunk = unique.slice(i, i + BATCH); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare(`SELECT did, resolved_at FROM identities WHERE did IN (${placeholders})`) + .bind(...chunk) + .all<{ did: string; resolved_at: number }>(); + + const found = new Map((rows.results ?? []).map((r) => [r.did, r.resolved_at])); + for (const did of chunk) { + const resolvedAt = found.get(did); + if (resolvedAt === undefined || resolvedAt < staleThreshold) { + toRefresh.push(did); + } + } + } + + for (const did of toRefresh) { + try { + await fetchAndSave(db, did); + } catch { + // Silently skip unresolvable identities + } + } +} diff --git a/packages/contrail-base/src/index.ts b/packages/contrail-base/src/index.ts new file mode 100644 index 0000000..e11f46c --- /dev/null +++ b/packages/contrail-base/src/index.ts @@ -0,0 +1,67 @@ +/** @atmo-dev/contrail-base — shared infrastructure for the contrail family. + * + * No routes. No tables of its own. Pure types, interfaces, primitives, and + * shared utilities used across contrail / contrail-appview / contrail-authority / + * contrail-record-host / contrail-community. + * + * Re-exported wholesale from each source module — anything internal that + * needed to be hidden would have an explicit subpath export instead. */ + +// Core types + config + helpers (Database, ContrailConfig, dialect helpers, etc.) +export * from "./types"; + +// Dialect (SqlDialect, getDialect, sqliteDialect, postgresDialect, buildFtsSchema) +export * from "./dialect"; + +// Identity (resolveActor, resolveIdentities, refreshStaleIdentities) +export * from "./identity"; + +// PDS client helpers (getPDS, getClient) +export * from "./client"; + +// Spaces interfaces + shared types +export * from "./spaces/types"; + +// Spaces URI helpers +export * from "./spaces/uri"; + +// TID generator +export * from "./spaces/tid"; + +// In-process auth marker +export * from "./spaces/in-process"; + +// Service-auth verification +export * from "./spaces/auth"; + +// ACL pure functions +export * from "./spaces/acl"; + +// Credentials +export * from "./spaces/credentials"; + +// Binding + key resolution +export * from "./spaces/binding"; + +// Blob adapter interface + built-in impls +export * from "./spaces/blob-adapter"; + +// Invite token primitives + community-handler interface +export * from "./invite/token"; +export * from "./invite/community-handler"; + +// Community integration interface + WhoamiExtension +export * from "./community-integration"; + +// Labels types +export * from "./labels/types"; + +// Realtime infrastructure +export * from "./realtime/types"; +export * from "./realtime/in-memory"; +export * from "./realtime/ticket"; +export * from "./realtime/durable-object"; +export * from "./realtime/sse"; +export * from "./realtime/websocket"; +export * from "./realtime/merge"; +export * from "./realtime/query-filter"; diff --git a/packages/contrail-base/src/invite/community-handler.ts b/packages/contrail-base/src/invite/community-handler.ts new file mode 100644 index 0000000..1043525 --- /dev/null +++ b/packages/contrail-base/src/invite/community-handler.ts @@ -0,0 +1,67 @@ +/** Pluggable handler for community-grant invites within the unified invite + * surface. The invite router calls into this when the target space is + * community-owned, or "tries" it on the redeem / revoke-without-spaceUri + * paths. Community module provides the impl; invite/router doesn't import + * from community at all. + * + * Each method returns a `HandlerResponse`: a `{status, body}` envelope that + * the router relays as JSON, or `null` (only on the "try" methods) meaning + * "not applicable, fall through to the user-owned path." */ + +export type HandlerResponse = { + status: number; + body: Record<string, unknown>; +}; + +export interface CommunityInviteHandler { + /** True iff this space is owned by a community (vs. a regular user DID). + * Used by the invite router to choose the dispatch path on + * create / list / revoke-with-spaceUri. */ + isCommunityOwned(spaceUri: string): Promise<boolean>; + + /** Create a community-grant invite. Caller is validated upstream for + * having a JWT; this method handles the access-level checks. */ + create(input: { + spaceUri: string; + callerDid: string; + /** Raw caller-supplied access level — implementation validates. */ + accessLevel?: string; + /** Caller-supplied `kind` field — community spaces don't accept this; the + * handler returns an InvalidRequest if set. */ + kind?: string; + expiresAt: number | null; + maxUses: number | null; + note: string | null; + }): Promise<HandlerResponse>; + + /** List invites for a community-owned space. */ + list(input: { + spaceUri: string; + callerDid: string; + includeRevoked: boolean; + }): Promise<HandlerResponse>; + + /** Revoke a known community-owned invite (caller already passed spaceUri + * and the router classified it as community-owned). */ + revoke(input: { + spaceUri: string; + tokenHash: string; + callerDid: string; + }): Promise<HandlerResponse>; + + /** Revoke without a spaceUri — try to find the invite in the community + * table; return null if not a community invite (router falls through). */ + tryRevokeByToken(input: { + tokenHash: string; + callerDid: string; + }): Promise<HandlerResponse | null>; + + /** Try to redeem a token as a community invite. Returns null if the token + * is not a community invite, in which case the router falls through to + * the user-owned redeem path. */ + tryRedeem(input: { + tokenHash: string; + callerDid: string; + now: number; + }): Promise<HandlerResponse | null>; +} diff --git a/packages/contrail-base/src/invite/token.ts b/packages/contrail-base/src/invite/token.ts new file mode 100644 index 0000000..76f4880 --- /dev/null +++ b/packages/contrail-base/src/invite/token.ts @@ -0,0 +1,43 @@ +const B64U_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +function bytesToB64Url(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += 3) { + const b0 = bytes[i]; + const b1 = bytes[i + 1] ?? 0; + const b2 = bytes[i + 2] ?? 0; + out += B64U_ALPHABET[b0 >> 2]; + out += B64U_ALPHABET[((b0 & 3) << 4) | (b1 >> 4)]; + if (i + 1 < bytes.length) out += B64U_ALPHABET[((b1 & 15) << 2) | (b2 >> 6)]; + if (i + 2 < bytes.length) out += B64U_ALPHABET[b2 & 63]; + } + return out; +} + +function bytesToHex(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i++) out += bytes[i].toString(16).padStart(2, "0"); + return out; +} + +/** Generate a fresh invite token (cryptographically random, 32 bytes base64url-encoded). */ +export function generateInviteToken(): string { + const bytes = new Uint8Array(32); + crypto.getRandomValues(bytes); + return bytesToB64Url(bytes); +} + +/** SHA-256 hash of a token, hex-encoded. Used as the PK in storage so raw tokens are never persisted. */ +export async function hashInviteToken(token: string): Promise<string> { + const encoded = new TextEncoder().encode(token); + const digest = await crypto.subtle.digest("SHA-256", encoded); + return bytesToHex(new Uint8Array(digest)); +} + +/** Convenience: generate a token and return both the raw form (returned to + * the creator once) and its hash (persisted as the stable ID). */ +export async function mintInviteToken(): Promise<{ token: string; tokenHash: string }> { + const token = generateInviteToken(); + const tokenHash = await hashInviteToken(token); + return { token, tokenHash }; +} diff --git a/packages/contrail-base/src/labels/types.ts b/packages/contrail-base/src/labels/types.ts new file mode 100644 index 0000000..f2e36c2 --- /dev/null +++ b/packages/contrail-base/src/labels/types.ts @@ -0,0 +1,56 @@ +import type { Database } from "../types"; + +/** A labeler the operator wants contrail to track. */ +export interface LabelerSource { + /** Labeler DID — `did:plc:...` or `did:web:...`. */ + did: string; + /** Override the service endpoint resolution. Otherwise resolved from the + * DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ + endpoint?: string; + /** Backfill from `cursor=0` on first sight. Defaults to true. Set false + * for "start from now" — useful for very chatty labelers. */ + backfill?: boolean; +} + +export interface LabelsConfig { + /** Labelers to subscribe to and index. */ + sources: LabelerSource[]; + /** DIDs honored when the caller sends no `atproto-accept-labelers` / + * `?labelers=`. Defaults to every entry in `sources`. Set `[]` for + * opt-in-only — clients see no labels unless they ask. */ + defaults?: string[]; + /** Per-request cap. Default: 20 (matches Bluesky). */ + maxPerRequest?: number; +} + +export const DEFAULT_LABELS_MAX_PER_REQUEST = 20; + +/** A single label as stored. Matches `com.atproto.label.defs#label`. */ +export interface LabelRow { + /** Issuing labeler DID. */ + src: string; + /** Subject — at-URI for record labels, plain DID for account labels. */ + uri: string; + /** Label value — kebab-case, ≤128 bytes per spec. */ + val: string; + /** Optional CID pin to a specific record version. */ + cid: string | null; + /** When true, retracts a previously-emitted label for the same (src, uri, val). */ + neg: boolean; + /** Expiry, unix seconds. Past this, hydration drops the row. */ + exp: number | null; + /** Creation timestamp, unix seconds — what we collapse on. */ + cts: number; + /** Raw signature bytes. Stored when present so we can re-emit later; + * not verified in v1. */ + sig: Uint8Array | null; +} + +/** Per-labeler state row — endpoint cache and last-seen seq cursor. */ +export interface LabelerCursorRow { + did: string; + cursor: number; + endpoint: string | null; + resolved_at: number | null; +} + diff --git a/packages/contrail-base/src/realtime/durable-object.ts b/packages/contrail-base/src/realtime/durable-object.ts new file mode 100644 index 0000000..966ab53 --- /dev/null +++ b/packages/contrail-base/src/realtime/durable-object.ts @@ -0,0 +1,567 @@ +/** Durable Object backend for realtime PubSub. + * + * Two pieces live here: + * 1. `RealtimePubSubDO` — the DO class. Ship it from your Worker via + * `export { RealtimePubSubDO } from "@atmo-dev/contrail";` and bind it in + * your `wrangler.toml`. One DO = one topic; addressed by name. + * 2. `DurableObjectPubSub` — client-side adapter implementing the PubSub + * interface against a DO namespace binding. + * + * Wire format between Worker and DO (internal, not a stable public contract): + * POST /publish — body = RealtimeEvent JSON + * GET /subscribe — server-sent events stream, optionally with + * `Upgrade: websocket` for WS connections. + * Auth/ACL is already checked at the Worker edge; + * the DO trusts anything that reaches it. */ + +import type { PubSub, RealtimeEvent } from "./types"; +import { translateForQuery, type TranslatedEnvelope } from "./query-filter"; +type TranslatedEvent = TranslatedEnvelope; + +/** Query spec attached to a WS subscriber, used to filter events before + * delivery. Shape matches what the Worker's `watchRecords` handler builds; + * forwarded to the DO via trusted internal headers on the WS upgrade. */ +export interface SubscriberQuerySpec { + /** NSID of the primary collection the client is watching. */ + collection: string; + /** Space URI this subscription is scoped to. Events outside are dropped. */ + spaceUri: string; + /** Hydrated relations. Keyed by relName — value is the child collection + * NSID and the field on the child record that references the parent. */ + hydrate?: Record<string, { childCollection: string; matchField: string }>; +} + +// ---- Minimal structural typings so we don't depend on @cloudflare/workers-types +// at the library level. Callers on Workers will have proper types. +// ---------------------------------------------------------------------------- + +export interface DurableObjectId { + toString(): string; +} + +export interface DurableObjectStub { + fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>; +} + +export interface DurableObjectNamespace { + idFromName(name: string): DurableObjectId; + get(id: DurableObjectId): DurableObjectStub; +} + +export interface DurableObjectState { + acceptWebSocket(ws: any, tags?: string[]): void; + getWebSockets(tag?: string): any[]; +} + +// ---------------------------------------------------------------------------- +// Client adapter +// ---------------------------------------------------------------------------- + +export class DurableObjectPubSub implements PubSub { + constructor(private readonly namespace: DurableObjectNamespace) {} + + private stub(topic: string): DurableObjectStub { + return this.namespace.get(this.namespace.idFromName(topic)); + } + + async publish(event: RealtimeEvent): Promise<void> { + const res = await this.stub(event.topic).fetch("https://do/publish", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(event), + }); + if (!res.ok) { + // Consume the body so the edge doesn't hold a dangling response. + await res.text().catch(() => ""); + throw new Error(`DO publish failed (${res.status})`); + } + } + + /** In-Worker server-side subscribe. Browsers should hit the SSE endpoint + * directly; this is the path for an in-process consumer that wants an + * AsyncIterable (tests, bots embedded in the Worker). */ + subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { + const stub = this.stub(topic); + return { + [Symbol.asyncIterator]() { + return pullIterator(stub, signal); + }, + }; + } + + /** Forward an incoming browser WS upgrade (or SSE GET) through to the DO + * that owns this topic, attaching a query-filter spec that the DO will use + * to decide what to deliver. The Worker must verify auth + spec validity + * before calling this — the DO trusts the headers. */ + async forwardSubscribe( + topic: string, + request: Request, + opts: { + did?: string; + querySpec?: SubscriberQuerySpec; + /** Unix ms. DO replays any buffered event with ts > sinceTs before + * going live — closes the snapshot→WS race window on the client. */ + sinceTs?: number; + } = {} + ): Promise<Response> { + const headers = new Headers(request.headers); + if (opts.querySpec) { + headers.set("X-Contrail-Query-Spec", JSON.stringify(opts.querySpec)); + } + const url = new URL("https://do/subscribe"); + if (opts.did) url.searchParams.set("did", opts.did); + if (opts.sinceTs && opts.sinceTs > 0) { + url.searchParams.set("sinceTs", String(opts.sinceTs)); + } + return this.stub(topic).fetch(url.toString(), { + method: "GET", + headers + }); + } +} + +function pullIterator( + stub: DurableObjectStub, + signal?: AbortSignal +): AsyncIterator<RealtimeEvent> { + let reader: ReadableStreamDefaultReader<Uint8Array> | null = null; + let buf = ""; + const decoder = new TextDecoder(); + const ac = new AbortController(); + if (signal) { + if (signal.aborted) ac.abort(); + else signal.addEventListener("abort", () => ac.abort(), { once: true }); + } + + const open = async () => { + const res = await stub.fetch("https://do/subscribe", { + method: "GET", + headers: { accept: "text/event-stream" }, + signal: ac.signal, + }); + if (!res.ok || !res.body) throw new Error(`DO subscribe failed (${res.status})`); + reader = res.body.getReader(); + }; + + return { + async next(): Promise<IteratorResult<RealtimeEvent>> { + if (!reader) await open(); + while (true) { + // Drain buffered frames. + while (true) { + const sep = buf.indexOf("\n\n"); + if (sep < 0) break; + const frame = buf.slice(0, sep); + buf = buf.slice(sep + 2); + let data: string | null = null; + for (const line of frame.split("\n")) { + if (line.startsWith(":")) continue; + if (line.startsWith("data:")) data = line.slice(5).trim(); + } + if (data) { + try { + return { value: JSON.parse(data) as RealtimeEvent, done: false }; + } catch { + /* skip malformed */ + } + } + } + if (ac.signal.aborted) return { value: undefined, done: true }; + const r = await reader!.read(); + if (r.done) return { value: undefined, done: true }; + buf += decoder.decode(r.value, { stream: true }); + } + }, + async return(): Promise<IteratorResult<RealtimeEvent>> { + ac.abort(); + try { + await reader?.cancel(); + } catch { + /* ignore */ + } + return { value: undefined, done: true }; + }, + }; +} + +// ---------------------------------------------------------------------------- +// Durable Object class +// ---------------------------------------------------------------------------- + +/** The Durable Object implementation. Each DO instance owns the fan-out for + * exactly one topic. WebSocket connections are stored via the Hibernation + * API (`state.acceptWebSocket`) so idle rooms cost near-zero. + * + * This class intentionally avoids the `DurableObject` base class so we don't + * have to depend on @cloudflare/workers-types at the library level — users + * wire it up directly in their Worker entry. */ +/** Rolling buffer of recent events, used to close the snapshot→WS race: + * when a new subscriber connects with `?sinceTs=X`, replay any buffered + * event with `event.ts > X` before going live. Bounded by count + age so + * memory stays small. */ +const RECENT_BUFFER_MS = 15_000; +const RECENT_BUFFER_MAX = 500; + +export class RealtimePubSubDO { + private readonly recentEvents: RealtimeEvent[] = []; + + constructor( + protected readonly state: DurableObjectState, + _env?: unknown + ) {} + + private pushRecent(event: RealtimeEvent): void { + this.recentEvents.push(event); + const cutoff = Date.now() - RECENT_BUFFER_MS; + while ( + this.recentEvents.length > RECENT_BUFFER_MAX || + (this.recentEvents.length > 0 && this.recentEvents[0]!.ts < cutoff) + ) { + this.recentEvents.shift(); + } + } + + /** Worker entry delegates `fetch` to this method. */ + async fetch(request: Request): Promise<Response> { + const url = new URL(request.url); + if (request.method === "POST" && url.pathname === "/publish") { + let event: RealtimeEvent; + try { + event = (await request.json()) as RealtimeEvent; + } catch { + return new Response(JSON.stringify({ error: "InvalidRequest" }), { status: 400 }); + } + this.publishEvent(event); + return new Response("{}", { status: 200 }); + } + if (request.method === "GET" && url.pathname === "/subscribe") { + const did = url.searchParams.get("did") ?? undefined; + const sinceTsRaw = url.searchParams.get("sinceTs"); + const sinceTs = sinceTsRaw ? Number(sinceTsRaw) : 0; + // Optional query-filter spec, forwarded by the Worker after it has + // verified the caller's auth + access. Parsed once here; the parsed + // object is serialized into the WS attachment so the DO can filter + // events on publish without re-parsing. + let querySpec: SubscriberQuerySpec | undefined; + const rawSpec = request.headers.get("X-Contrail-Query-Spec"); + if (rawSpec) { + try { + querySpec = JSON.parse(rawSpec) as SubscriberQuerySpec; + } catch { + return new Response( + JSON.stringify({ error: "InvalidRequest", message: "bad X-Contrail-Query-Spec" }), + { status: 400 } + ); + } + } + + if (request.headers.get("Upgrade")?.toLowerCase() === "websocket") { + const Pair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; + if (!Pair) return new Response("websockets require Workers", { status: 426 }); + const pair = new Pair(); + this.acceptWebSocketSubscriber(pair[1], did, querySpec); + if (sinceTs > 0) this.replayRecentTo(pair[1], sinceTs); + return new Response(null, { + status: 101, + // Workers-specific init field + webSocket: pair[0], + } as ResponseInit & { webSocket: unknown }); + } + return this.openSseResponse(did, querySpec, sinceTs); + } + return new Response("not found", { status: 404 }); + } + + /** Fan-out an event to every connected subscriber (WS + SSE). + * Public so tests + advanced callers can skip the HTTP layer. + * + * If a subscriber has attached a `querySpec`, we translate the raw event + * into 0–1 watchRecords-shaped events (record.created, record.deleted, + * hydration.added, hydration.removed) and deliver only those. Otherwise + * the raw event is delivered as-is (topic-firehose behaviour for the + * `realtime.subscribe` endpoint). */ + publishEvent(event: RealtimeEvent): void { + // Buffer first so a subscriber connecting mid-publish (race-window + // replay) can pick up this event too once they provide their sinceTs. + this.pushRecent(event); + + const rawPayload = JSON.stringify(event); + const rawFrame = `event: ${event.kind}\ndata: ${rawPayload}\n\n`; + + for (const ws of this.state.getWebSockets()) { + const attachment = getAttachment(ws); + + if (attachment?.querySpec) { + const translated = translateForQuery(event, attachment); + if (translated) this.writeSubscriberState(ws, attachment, translated); + for (const msg of translated ?? []) { + try { + ws.send(JSON.stringify(msg)); + } catch { + /* ignore */ + } + } + } else { + try { + ws.send(rawPayload); + } catch { + /* ignore */ + } + } + + if ( + event.kind === "member.removed" && + attachment?.did && + event.payload.did === attachment.did + ) { + try { + ws.close(4003, "membership-revoked"); + } catch { + /* ignore */ + } + } + } + + for (const entry of this.sseControllers) { + if (entry.querySpec) { + const translated = translateForQuery(event, entry); + if (translated) this.writeSubscriberStateForSse(entry, translated); + for (const msg of translated ?? []) { + try { + entry.controller.enqueue( + this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) + ); + } catch { + /* drop */ + } + } + } else { + try { + entry.controller.enqueue(this.encoder.encode(rawFrame)); + } catch { + /* drop; cleanup happens on the subscribe-side */ + } + } + + if ( + event.kind === "member.removed" && + entry.did && + event.payload.did === entry.did + ) { + try { + entry.controller.close(); + } catch { + /* ignore */ + } + } + } + } + + /** Register a server-side WebSocket as a subscriber. Wires the DID + + * optional query spec into the hibernation attachment so this DO can + * filter and route events after going to sleep. */ + acceptWebSocketSubscriber( + serverWs: any, + did?: string, + querySpec?: SubscriberQuerySpec + ): void { + this.state.acceptWebSocket(serverWs, did ? [did] : undefined); + if (did || querySpec) { + setAttachment(serverWs, { + did, + querySpec, + parentUris: [], + childToParent: {} + }); + } + } + + /** Open an SSE subscriber; returns the streaming Response. */ + openSseResponse( + did?: string, + querySpec?: SubscriberQuerySpec, + sinceTs = 0 + ): Response { + let entry: SseEntry; + const stream = new ReadableStream<Uint8Array>({ + start: (controller) => { + entry = { + controller, + did, + querySpec, + parentUris: new Set(), + childToParent: new Map() + }; + this.sseControllers.add(entry); + controller.enqueue(this.encoder.encode(`: open\n\n`)); + if (sinceTs > 0) this.replayRecentToSse(entry, sinceTs); + }, + cancel: () => { + this.sseControllers.delete(entry); + }, + }); + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + }, + }); + } + + /** Replay buffered events with ts > sinceTs through this subscriber's + * query-spec filter. Called once, synchronously, on WS connect. */ + private replayRecentTo(ws: any, sinceTs: number): void { + const attachment = getAttachment(ws); + for (const event of this.recentEvents) { + if (event.ts <= sinceTs) continue; + if (attachment?.querySpec) { + const translated = translateForQuery(event, attachment); + if (translated) this.writeSubscriberState(ws, attachment, translated); + for (const msg of translated ?? []) { + try { + ws.send(JSON.stringify(msg)); + } catch { + /* ignore */ + } + } + } else { + try { + ws.send(JSON.stringify(event)); + } catch { + /* ignore */ + } + } + } + } + + private replayRecentToSse(entry: SseEntry, sinceTs: number): void { + for (const event of this.recentEvents) { + if (event.ts <= sinceTs) continue; + if (entry.querySpec) { + const translated = translateForQuery(event, entry); + if (translated) this.writeSubscriberStateForSse(entry, translated); + for (const msg of translated ?? []) { + try { + entry.controller.enqueue( + this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) + ); + } catch { + /* drop */ + } + } + } else { + try { + entry.controller.enqueue( + this.encoder.encode(`event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`) + ); + } catch { + /* drop */ + } + } + } + } + + /** Update the persisted WS attachment state after we've decided which + * events to forward. Keeps the parent/child tracking tables warm across + * hibernation. */ + private writeSubscriberState( + ws: any, + attachment: WsAttachment, + translated: TranslatedEvent[] + ): void { + let dirty = false; + for (const msg of translated) { + if (msg.kind === "record.created" && msg.data.record?.uri) { + attachment.parentUris = Array.from( + new Set([...(attachment.parentUris ?? []), msg.data.record.uri]) + ); + dirty = true; + } else if (msg.kind === "record.deleted" && msg.data.uri) { + const before = attachment.parentUris ?? []; + attachment.parentUris = before.filter((u) => u !== msg.data.uri); + if (attachment.parentUris.length !== before.length) dirty = true; + } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { + attachment.childToParent = { + ...(attachment.childToParent ?? {}), + [msg.data.child.rkey]: { + parentUri: msg.data.parentUri, + relName: msg.data.relation + } + }; + dirty = true; + } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { + const next = { ...(attachment.childToParent ?? {}) }; + if (next[msg.data.childRkey]) { + delete next[msg.data.childRkey]; + attachment.childToParent = next; + dirty = true; + } + } + } + if (dirty) setAttachment(ws, attachment); + } + + private writeSubscriberStateForSse( + entry: SseEntry, + translated: TranslatedEvent[] + ): void { + for (const msg of translated) { + if (msg.kind === "record.created" && msg.data.record?.uri) { + entry.parentUris?.add(msg.data.record.uri); + } else if (msg.kind === "record.deleted" && msg.data.uri) { + entry.parentUris?.delete(msg.data.uri); + } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { + entry.childToParent?.set(msg.data.child.rkey, { + parentUri: msg.data.parentUri, + relName: msg.data.relation + }); + } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { + entry.childToParent?.delete(msg.data.childRkey); + } + } + } + + private readonly sseControllers = new Set<SseEntry>(); + private readonly encoder = new TextEncoder(); +} + +interface SseEntry { + controller: ReadableStreamDefaultController<Uint8Array>; + did: string | undefined; + querySpec?: SubscriberQuerySpec; + parentUris?: Set<string>; + childToParent?: Map<string, { parentUri: string; relName: string }>; +} + +interface WsAttachment { + did?: string; + querySpec?: SubscriberQuerySpec; + /** URIs of primary records currently in this subscriber's result set. */ + parentUris?: string[]; + /** childRkey → parent info, for routing child delete events. */ + childToParent?: Record<string, { parentUri: string; relName: string }>; +} + +function setAttachment(ws: any, attachment: WsAttachment): void { + try { + ws.serializeAttachment?.(attachment); + } catch { + /* non-hibernating socket — fall back to a direct property */ + ws.__attachment = attachment; + } +} + +function getAttachment(ws: any): WsAttachment | null { + try { + const a = ws.deserializeAttachment?.(); + if (a) return a as WsAttachment; + } catch { + /* ignore */ + } + return (ws.__attachment as WsAttachment | undefined) ?? null; +} + +// Query-spec filtering lives in ./query-filter so the Worker can reuse it for +// non-DO (InMemoryPubSub) watchRecords paths without bundling the whole DO. diff --git a/packages/contrail-base/src/realtime/in-memory.ts b/packages/contrail-base/src/realtime/in-memory.ts new file mode 100644 index 0000000..311c0a5 --- /dev/null +++ b/packages/contrail-base/src/realtime/in-memory.ts @@ -0,0 +1,116 @@ +import type { PubSub, RealtimeEvent } from "./types"; +import { DEFAULT_QUEUE_BOUND } from "./types"; + +/** Single-process PubSub backed by in-memory subscriber sets. + * + * Each subscriber owns a bounded queue; when full, oldest events are dropped. + * `publish` returns once every subscriber has been offered the event — it + * never awaits a subscriber's consumption, so a slow consumer can't block + * producers. The cost of that guarantee is the drop-oldest policy. */ +export class InMemoryPubSub implements PubSub { + private readonly subscribers = new Map<string, Set<Subscriber>>(); + private readonly queueBound: number; + + constructor(opts: { queueBound?: number } = {}) { + this.queueBound = opts.queueBound ?? DEFAULT_QUEUE_BOUND; + } + + async publish(event: RealtimeEvent): Promise<void> { + const set = this.subscribers.get(event.topic); + if (!set) return; + for (const sub of set) sub.push(event); + } + + subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { + const sub = new Subscriber(this.queueBound); + let set = this.subscribers.get(topic); + if (!set) { + set = new Set(); + this.subscribers.set(topic, set); + } + set.add(sub); + + const cleanup = () => { + sub.close(); + const s = this.subscribers.get(topic); + if (s) { + s.delete(sub); + if (s.size === 0) this.subscribers.delete(topic); + } + }; + + if (signal) { + if (signal.aborted) cleanup(); + else signal.addEventListener("abort", cleanup, { once: true }); + } + + return sub.iterate(cleanup); + } + + /** Test-only: current subscriber count for a topic. */ + subscriberCount(topic: string): number { + return this.subscribers.get(topic)?.size ?? 0; + } +} + +class Subscriber { + private readonly queue: RealtimeEvent[] = []; + private pending: ((v: RealtimeEvent | null) => void) | null = null; + private closed = false; + /** Number of events dropped because the queue was full. The consumer can + * observe a gap by comparing monotonic event timestamps; exposing the + * count on a side channel is future work. */ + public droppedCount = 0; + + constructor(private readonly bound: number) {} + + push(event: RealtimeEvent): void { + if (this.closed) return; + if (this.pending) { + const p = this.pending; + this.pending = null; + p(event); + return; + } + if (this.queue.length >= this.bound) { + this.queue.shift(); + this.droppedCount += 1; + } + this.queue.push(event); + } + + close(): void { + if (this.closed) return; + this.closed = true; + if (this.pending) { + const p = this.pending; + this.pending = null; + p(null); + } + } + + iterate(cleanup: () => void): AsyncIterable<RealtimeEvent> { + const self = this; + return { + [Symbol.asyncIterator]() { + return { + async next(): Promise<IteratorResult<RealtimeEvent>> { + if (self.queue.length > 0) { + return { value: self.queue.shift()!, done: false }; + } + if (self.closed) return { value: undefined, done: true }; + const event = await new Promise<RealtimeEvent | null>((resolve) => { + self.pending = resolve; + }); + if (event === null) return { value: undefined, done: true }; + return { value: event, done: false }; + }, + async return(): Promise<IteratorResult<RealtimeEvent>> { + cleanup(); + return { value: undefined, done: true }; + }, + }; + }, + }; + } +} diff --git a/packages/contrail-base/src/realtime/merge.ts b/packages/contrail-base/src/realtime/merge.ts new file mode 100644 index 0000000..e0fa7df --- /dev/null +++ b/packages/contrail-base/src/realtime/merge.ts @@ -0,0 +1,77 @@ +/** Merge N AsyncIterables into one, interleaving events as they arrive. + * Terminates when every source terminates, or when `signal` aborts. */ + +export function mergeAsyncIterables<T>( + sources: AsyncIterable<T>[], + signal?: AbortSignal +): AsyncIterable<T> { + if (sources.length === 0) { + return { + async *[Symbol.asyncIterator]() { + /* nothing to yield */ + }, + }; + } + + return { + [Symbol.asyncIterator]() { + const iterators = sources.map((s) => s[Symbol.asyncIterator]()); + // One in-flight next() per source, racing each other. + type Slot = { + idx: number; + promise: Promise<{ idx: number; result: IteratorResult<T> }>; + }; + const pending = new Map<number, Slot>(); + let doneCount = 0; + + const schedule = (idx: number) => { + const slot: Slot = { + idx, + promise: iterators[idx]! + .next() + .then((result) => ({ idx, result })), + }; + pending.set(idx, slot); + }; + + for (let i = 0; i < iterators.length; i++) schedule(i); + + const cleanup = () => { + for (const it of iterators) { + try { + it.return?.(); + } catch { + /* ignore */ + } + } + }; + + if (signal) { + if (signal.aborted) cleanup(); + else signal.addEventListener("abort", cleanup, { once: true }); + } + + return { + async next(): Promise<IteratorResult<T>> { + while (pending.size > 0) { + const slots = [...pending.values()]; + const { idx, result } = await Promise.race(slots.map((s) => s.promise)); + pending.delete(idx); + if (result.done) { + doneCount += 1; + if (doneCount === iterators.length) return { value: undefined, done: true }; + continue; + } + schedule(idx); + return { value: result.value, done: false }; + } + return { value: undefined, done: true }; + }, + async return(): Promise<IteratorResult<T>> { + cleanup(); + return { value: undefined, done: true }; + }, + }; + }, + }; +} diff --git a/packages/contrail-base/src/realtime/query-filter.ts b/packages/contrail-base/src/realtime/query-filter.ts new file mode 100644 index 0000000..1b79eb6 --- /dev/null +++ b/packages/contrail-base/src/realtime/query-filter.ts @@ -0,0 +1,235 @@ +/** Shared query-spec → event-translation logic. + * + * Used by: + * - the Durable Object's WS publish path (per-subscriber filter after hibernation) + * - the Worker's SSE / Worker-terminated WS path (in-process filter) + * + * Given a raw RealtimeEvent and a SubscriberQuerySpec, returns the list of + * `{kind, data}` envelopes to send to the subscriber, or `null` if the + * subscriber has no spec (i.e. raw-firehose mode). */ + +import type { RealtimeEvent } from "./types"; +import type { SubscriberQuerySpec } from "./durable-object"; + +export type TranslatedEnvelope = + | { + kind: "record.created"; + data: { + record: { + uri: string; + did: string; + rkey: string; + collection: string; + cid: string | null | undefined; + record: Record<string, unknown>; + time_us: number; + indexed_at: number; + space: string; + }; + }; + } + | { kind: "record.deleted"; data: { uri: string; did: string; rkey: string } } + | { + kind: "hydration.added"; + data: { + parentUri: string; + relation: string; + child: { + uri: string; + did: string; + rkey: string; + collection: string; + cid: string | null | undefined; + record: Record<string, unknown>; + space: string; + }; + }; + } + | { + kind: "hydration.removed"; + data: { + parentUri: string; + relation: string; + childRkey: string; + childDid?: string; + }; + }; + +export interface SubscriberView { + querySpec?: SubscriberQuerySpec; + parentUris?: Set<string> | string[]; + childToParent?: + | Map<string, { parentUri: string; relName: string }> + | Record<string, { parentUri: string; relName: string }>; +} + +export function translateForQuery( + event: RealtimeEvent, + sub: SubscriberView +): TranslatedEnvelope[] | null { + const spec = sub.querySpec; + if (!spec) return null; + if (event.kind !== "record.created" && event.kind !== "record.deleted") return []; + if (event.payload.space !== spec.spaceUri) return []; + + const primaryUri = `at://${event.payload.did}/${event.payload.collection}/${event.payload.rkey}`; + + if (event.payload.collection === spec.collection) { + if (event.kind === "record.created") { + return [ + { + kind: "record.created", + data: { + record: { + uri: primaryUri, + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + record: event.payload.record, + time_us: event.ts * 1000, + indexed_at: event.ts, + space: spec.spaceUri + } + } + } + ]; + } + return [ + { + kind: "record.deleted", + data: { + uri: primaryUri, + did: event.payload.did, + rkey: event.payload.rkey + } + } + ]; + } + + if (!spec.hydrate) return []; + for (const [relName, rel] of Object.entries(spec.hydrate)) { + if (rel.childCollection !== event.payload.collection) continue; + if (event.kind === "record.created") { + const parentUri = getNestedValue( + event.payload.record as Record<string, unknown>, + rel.matchField + ); + if (typeof parentUri !== "string") continue; + if (!hasParent(sub.parentUris, parentUri)) continue; + return [ + { + kind: "hydration.added", + data: { + parentUri, + relation: relName, + child: { + uri: primaryUri, + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + record: event.payload.record, + space: spec.spaceUri + } + } + } + ]; + } + const info = getChildInfo(sub.childToParent, event.payload.rkey); + if (!info || info.relName !== relName) continue; + return [ + { + kind: "hydration.removed", + data: { + parentUri: info.parentUri, + relation: relName, + childRkey: event.payload.rkey, + childDid: event.payload.did + } + } + ]; + } + return []; +} + +export function applyEnvelopesToSubscriber( + subscriber: SubscriberView, + envs: TranslatedEnvelope[] +): void { + for (const msg of envs) { + if (msg.kind === "record.created") { + ensureParentSet(subscriber).add(msg.data.record.uri); + } else if (msg.kind === "record.deleted") { + const set = subscriber.parentUris; + if (set instanceof Set) set.delete(msg.data.uri); + else if (Array.isArray(set)) { + const idx = set.indexOf(msg.data.uri); + if (idx >= 0) set.splice(idx, 1); + } + } else if (msg.kind === "hydration.added") { + ensureChildMap(subscriber).set(msg.data.child.rkey, { + parentUri: msg.data.parentUri, + relName: msg.data.relation + }); + } else if (msg.kind === "hydration.removed") { + const map = subscriber.childToParent; + if (map instanceof Map) map.delete(msg.data.childRkey); + else if (map && typeof map === "object") { + delete (map as Record<string, unknown>)[msg.data.childRkey]; + } + } + } +} + +function ensureParentSet(sub: SubscriberView): Set<string> { + if (sub.parentUris instanceof Set) return sub.parentUris; + const set = new Set<string>(sub.parentUris ?? []); + sub.parentUris = set; + return set; +} + +function ensureChildMap( + sub: SubscriberView +): Map<string, { parentUri: string; relName: string }> { + if (sub.childToParent instanceof Map) return sub.childToParent; + const map = new Map<string, { parentUri: string; relName: string }>(); + if (sub.childToParent && typeof sub.childToParent === "object") { + for (const [k, v] of Object.entries(sub.childToParent)) map.set(k, v); + } + sub.childToParent = map; + return map; +} + +function hasParent( + parents: Set<string> | string[] | undefined, + uri: string +): boolean { + if (!parents) return false; + if (parents instanceof Set) return parents.has(uri); + return parents.includes(uri); +} + +function getChildInfo( + map: + | Map<string, { parentUri: string; relName: string }> + | Record<string, { parentUri: string; relName: string }> + | undefined, + rkey: string +): { parentUri: string; relName: string } | undefined { + if (!map) return undefined; + if (map instanceof Map) return map.get(rkey); + return map[rkey]; +} + +function getNestedValue( + obj: Record<string, unknown>, + path: string +): unknown { + let cur: unknown = obj; + for (const key of path.split(".")) { + if (cur == null || typeof cur !== "object") return undefined; + cur = (cur as Record<string, unknown>)[key]; + } + return cur; +} diff --git a/packages/contrail-base/src/realtime/sse.ts b/packages/contrail-base/src/realtime/sse.ts new file mode 100644 index 0000000..65d89c1 --- /dev/null +++ b/packages/contrail-base/src/realtime/sse.ts @@ -0,0 +1,98 @@ +/** Server-Sent Events transport. + * + * Wraps an AsyncIterable<RealtimeEvent> as a streaming Response. The caller + * (the router) has already done auth and has an AbortSignal it can use to + * tear the stream down (e.g. on `member.removed` for the subscriber's DID). */ + +import type { RealtimeEvent } from "./types"; +import { DEFAULT_KEEPALIVE_MS } from "./types"; + +export interface SseOptions { + keepaliveMs?: number; + /** Called before the stream closes. Useful for cleanup that the caller + * can't do via the signal (e.g. removing a subscriber from a set). */ + onClose?: () => void; +} + +export function sseResponse( + iter: AsyncIterable<RealtimeEvent>, + signal: AbortSignal, + opts: SseOptions = {} +): Response { + const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + const encoder = new TextEncoder(); + + const stream = new ReadableStream<Uint8Array>({ + start(controller) { + let closed = false; + let keepalive: ReturnType<typeof setInterval> | null = null; + + const close = () => { + if (closed) return; + closed = true; + if (keepalive) clearInterval(keepalive); + try { + controller.close(); + } catch { + /* already closed */ + } + opts.onClose?.(); + }; + + signal.addEventListener("abort", close, { once: true }); + + keepalive = setInterval(() => { + if (closed) return; + try { + controller.enqueue(encoder.encode(`: keepalive\n\n`)); + } catch { + close(); + } + }, keepaliveMs); + + (async () => { + // Opening comment — helps some clients / proxies initialize promptly. + controller.enqueue(encoder.encode(`: open\n\n`)); + try { + for await (const event of iter) { + if (closed) break; + controller.enqueue(encoder.encode(frameEvent(event))); + } + } catch (err) { + if (!closed) { + try { + controller.enqueue( + encoder.encode( + `event: error\ndata: ${JSON.stringify({ + message: err instanceof Error ? err.message : String(err), + })}\n\n` + ) + ); + } catch { + /* stream already torn down */ + } + } + } finally { + close(); + } + })(); + }, + cancel() { + opts.onClose?.(); + }, + }); + + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + }, + }); +} + +function frameEvent(event: RealtimeEvent): string { + return `event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`; +} diff --git a/packages/contrail-base/src/realtime/ticket.ts b/packages/contrail-base/src/realtime/ticket.ts new file mode 100644 index 0000000..1c6e3de --- /dev/null +++ b/packages/contrail-base/src/realtime/ticket.ts @@ -0,0 +1,179 @@ +/** Subscription tickets — HMAC-signed short-lived `{topics, did, exp}` blobs. + * + * Wire format: `<payload>.<sig>` where + * payload = base64url(JSON({ topics, did, exp, iat })) + * sig = base64url(HMAC-SHA256(key, payload)) + * + * Tickets are integrity-only (not encrypted). Browsers use them because + * EventSource / WebSocket can't send Authorization headers; server-side + * consumers skip the ticket dance and send their JWT directly. */ + +export interface TicketPayload { + /** Concrete delivery topics this ticket authorizes. `community:<did>` is + * expanded to the caller's visible spaces before signing — a ticket never + * carries a community alias. */ + topics: string[]; + did: string; + /** Unix ms. */ + exp: number; + /** Unix ms — useful for debugging; ignored on verify. */ + iat: number; + /** Optional: query-scoped watchRecords spec this ticket authorizes. Present + * when the ticket was minted from a watchRecords handshake. The server + * trusts the signed spec on upgrade and forwards it to the DO. */ + querySpec?: TicketQuerySpec; +} + +export interface TicketQuerySpec { + collection: string; + /** Exactly one of `spaceUri` or `actor` is set. `spaceUri` = per-space + * watch; `actor` = cross-space watch for records authored by this DID + * (the ticket's `topics` list carries the expanded delivery topics). */ + spaceUri?: string; + actor?: string; + hydrate?: Record<string, { childCollection: string; matchField: string }>; +} + +function normalizeSecret(secret: Uint8Array | string): Uint8Array { + if (typeof secret !== "string") { + if (secret.length !== 32) { + throw new Error(`realtime ticketSecret must be 32 bytes, got ${secret.length}`); + } + return secret; + } + // 64 hex chars would also round-trip as base64 (to 48 bytes). Prefer hex + // when the input matches the hex alphabet exactly; fall back to base64. + const hex = tryHex(secret); + if (hex && hex.length === 32) return hex; + const b64 = tryBase64(secret); + if (b64 && b64.length === 32) return b64; + if (hex || b64) { + const got = (hex ?? b64)!.length; + throw new Error(`realtime ticketSecret must decode to 32 bytes, got ${got}`); + } + throw new Error("realtime ticketSecret must be a 32-byte Uint8Array or base64/hex string"); +} + +function tryBase64(s: string): Uint8Array | null { + try { + const normal = s.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); + if (!/^[A-Za-z0-9+/]*=*$/.test(padded)) return null; + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; + } catch { + return null; + } +} + +function tryHex(s: string): Uint8Array | null { + if (!/^[0-9a-fA-F]+$/.test(s) || s.length % 2 !== 0) return null; + const out = new Uint8Array(s.length / 2); + for (let i = 0; i < out.length; i++) { + out[i] = parseInt(s.slice(i * 2, i * 2 + 2), 16); + } + return out; +} + +function b64urlFromBytes(bytes: Uint8Array): string { + let bin = ""; + for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); + return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +function b64urlToBytes(s: string): Uint8Array { + const normal = s.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function b64urlFromString(s: string): string { + return b64urlFromBytes(new TextEncoder().encode(s)); +} + +function stringFromB64url(s: string): string { + return new TextDecoder().decode(b64urlToBytes(s)); +} + +function constantTimeEq(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + let diff = 0; + for (let i = 0; i < a.length; i++) diff |= a[i]! ^ b[i]!; + return diff === 0; +} + +export class TicketSigner { + private readonly keyPromise: Promise<CryptoKey>; + + constructor(secret: Uint8Array | string) { + const raw = normalizeSecret(secret); + this.keyPromise = crypto.subtle.importKey( + "raw", + raw as BufferSource, + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign", "verify"] + ); + } + + async sign(input: { + topics: string[]; + did: string; + ttlMs: number; + querySpec?: TicketQuerySpec; + }): Promise<string> { + const now = Date.now(); + const payload: TicketPayload = { + topics: input.topics, + did: input.did, + exp: now + input.ttlMs, + iat: now, + ...(input.querySpec ? { querySpec: input.querySpec } : {}), + }; + const payloadPart = b64urlFromString(JSON.stringify(payload)); + const sig = await crypto.subtle.sign( + "HMAC", + await this.keyPromise, + new TextEncoder().encode(payloadPart) as BufferSource + ); + const sigPart = b64urlFromBytes(new Uint8Array(sig)); + return `${payloadPart}.${sigPart}`; + } + + /** Returns the decoded payload if the ticket is valid + unexpired, else null. */ + async verify(ticket: string): Promise<TicketPayload | null> { + const dot = ticket.indexOf("."); + if (dot < 0) return null; + const payloadPart = ticket.slice(0, dot); + const sigPart = ticket.slice(dot + 1); + let expectedSig: Uint8Array; + try { + expectedSig = b64urlToBytes(sigPart); + } catch { + return null; + } + const computedRaw = await crypto.subtle.sign( + "HMAC", + await this.keyPromise, + new TextEncoder().encode(payloadPart) as BufferSource + ); + const computed = new Uint8Array(computedRaw); + if (!constantTimeEq(expectedSig, computed)) return null; + let parsed: TicketPayload; + try { + parsed = JSON.parse(stringFromB64url(payloadPart)); + } catch { + return null; + } + if (!parsed || !Array.isArray(parsed.topics) || typeof parsed.did !== "string") { + return null; + } + if (typeof parsed.exp !== "number" || parsed.exp <= Date.now()) return null; + return parsed; + } +} diff --git a/packages/contrail-base/src/realtime/types.ts b/packages/contrail-base/src/realtime/types.ts new file mode 100644 index 0000000..05a246e --- /dev/null +++ b/packages/contrail-base/src/realtime/types.ts @@ -0,0 +1,127 @@ +/** Realtime module — canonical types + interfaces. See docs/realtime.md. */ + +/** Discriminated union of every event kind that flows through the PubSub. + * + * `record.created` carries the full record body so a subscriber can apply an + * insert or upsert without a follow-up `getRecord` call. Writing a new record + * to the same `(did, collection, rkey)` publishes another `record.created` — + * treat it as upsert. + * + * **Payload shape mirrors `listRecords` output** (`uri`, `did`, `space?`, + * `time_us`), so a subscriber can render a live row the same way it renders + * a fetched row. + * + * **Publisher/topic matrix (intentional trust split):** + * - `collection:<nsid>` and `actor:<did>` carry *public* record events only + * (from jetstream ingestion) — no `space`. + * - `space:<uri>` and `community:<did>` carry *space* events — `space` is + * always set. Never cross-published to public topics (privacy). */ +export type RealtimeEvent = + | { + topic: string; + kind: "record.created"; + payload: { + uri: string; + did: string; + collection: string; + rkey: string; + cid: string | null; + record: Record<string, unknown>; + time_us: number; + /** Present only for space records; absent for public records. */ + space?: string; + }; + ts: number; + } + | { + topic: string; + kind: "record.deleted"; + payload: { + uri: string; + did: string; + collection: string; + rkey: string; + /** Present only for space records; absent for public records. */ + space?: string; + }; + ts: number; + } + | { + topic: string; + kind: "member.added"; + payload: { space: string; did: string }; + ts: number; + } + | { + topic: string; + kind: "member.removed"; + payload: { space: string; did: string }; + ts: number; + }; + +export type RealtimeEventKind = RealtimeEvent["kind"]; + +/** Core pubsub abstraction. Implementations: InMemoryPubSub, DurableObjectPubSub. */ +export interface PubSub { + publish(event: RealtimeEvent): Promise<void>; + /** Stream events on the topic until the signal aborts (or the iterator is + * returned/broken out of). Implementations use a bounded per-subscriber + * queue with drop-oldest semantics — a slow subscriber can't stall publishers. */ + subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent>; +} + +// ---- Canonical topic strings ----------------------------------------------- +// `community:<did>` is an alias resolved at ticket-mint time to the concrete +// set of `space:<uri>` topics the caller can see; it is never a real delivery +// topic. The other three are real. + +export function spaceTopic(uri: string): string { + return `space:${uri}`; +} + +export function communityTopic(did: string): string { + return `community:${did}`; +} + +export function collectionTopic(nsid: string): string { + return `collection:${nsid}`; +} + +export function actorTopic(did: string): string { + return `actor:${did}`; +} + +export function isCommunityTopic(topic: string): boolean { + return topic.startsWith("community:"); +} + +export function parseCommunityTopic(topic: string): string | null { + return isCommunityTopic(topic) ? topic.slice("community:".length) : null; +} + +export function parseSpaceTopic(topic: string): string | null { + return topic.startsWith("space:") ? topic.slice("space:".length) : null; +} + +// ---- Config ----------------------------------------------------------------- + +export interface RealtimeConfig { + /** Backing pubsub. Default: new InMemoryPubSub() (single-process only). On + * Workers, pass `new DurableObjectPubSub(env.REALTIME)`. */ + pubsub?: PubSub; + /** HMAC secret used to sign subscription tickets. 32 bytes. Accepts raw + * Uint8Array or base64 / hex string. Envelope-encrypts nothing — tickets + * are integrity-only, not confidential. */ + ticketSecret: Uint8Array | string; + /** Ticket lifetime in ms. Default 120_000 (2 minutes). */ + ticketTtlMs?: number; + /** SSE/WS keepalive interval in ms. Default 15_000. */ + keepaliveMs?: number; + /** Per-subscriber queue bound. Default 1024. Events beyond this are dropped + * oldest-first and the subscriber receives a `lag` signal (out of band). */ + queueBound?: number; +} + +export const DEFAULT_TICKET_TTL_MS = 120_000; +export const DEFAULT_KEEPALIVE_MS = 15_000; +export const DEFAULT_QUEUE_BOUND = 1024; diff --git a/packages/contrail-base/src/realtime/websocket.ts b/packages/contrail-base/src/realtime/websocket.ts new file mode 100644 index 0000000..833483d --- /dev/null +++ b/packages/contrail-base/src/realtime/websocket.ts @@ -0,0 +1,84 @@ +/** WebSocket transport. + * + * Accepts a new WebSocket connection (either via `WebSocketPair` on Workers + * or a platform-provided server-side socket) and pumps events to it from an + * AsyncIterable<RealtimeEvent>. Messages are UTF-8 JSON, one event per frame. + * + * Close codes (subset, RFC 6455 + app-custom): + * - 4001: server error pumping + * - 4003: membership revoked + * - 4008: ticket/auth invalid (used by the router, not here) + */ + +import type { RealtimeEvent } from "./types"; +import { DEFAULT_KEEPALIVE_MS } from "./types"; + +export interface WebSocketLike { + send(data: string): void; + close(code?: number, reason?: string): void; + addEventListener(type: "message" | "close" | "error", listener: (ev: any) => void): void; +} + +export interface WebSocketPumpOptions { + keepaliveMs?: number; + onClose?: () => void; +} + +/** Pump events from `iter` to `ws` until the signal aborts or the iter ends. + * Caller is responsible for having already accept()ed the socket. */ +export async function pumpWebSocket( + ws: WebSocketLike, + iter: AsyncIterable<RealtimeEvent>, + signal: AbortSignal, + opts: WebSocketPumpOptions = {} +): Promise<void> { + const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + let closed = false; + + const close = (code?: number, reason?: string) => { + if (closed) return; + closed = true; + try { + ws.close(code, reason); + } catch { + /* already closed */ + } + opts.onClose?.(); + }; + + ws.addEventListener("close", () => { + closed = true; + opts.onClose?.(); + }); + ws.addEventListener("error", () => { + closed = true; + opts.onClose?.(); + }); + signal.addEventListener("abort", () => close(1000, "aborted"), { once: true }); + + const keepalive = setInterval(() => { + if (closed) return; + try { + ws.send(JSON.stringify({ kind: "$keepalive" })); + } catch { + close(); + } + }, keepaliveMs); + + try { + for await (const event of iter) { + if (closed) break; + try { + ws.send(JSON.stringify(event)); + } catch { + close(4001, "send-failed"); + break; + } + } + } catch { + close(4001, "pump-error"); + } finally { + clearInterval(keepalive); + close(); + } +} diff --git a/packages/contrail-base/src/spaces/acl.ts b/packages/contrail-base/src/spaces/acl.ts new file mode 100644 index 0000000..6e0da0a --- /dev/null +++ b/packages/contrail-base/src/spaces/acl.ts @@ -0,0 +1,69 @@ +import type { AppPolicy, SpaceMemberRow, SpaceRow } from "./types"; + +export type AclOp = "read" | "write" | "delete"; + +export interface AclInput { + op: AclOp; + space: SpaceRow; + callerDid: string; + /** Membership row for the caller (or null). Owner does not require a row. */ + member: SpaceMemberRow | null; + /** OAuth client_id of the app calling on caller's behalf, for app policy checks. */ + clientId?: string; + /** For per-record ops (get/delete), the record's author DID. */ + targetAuthorDid?: string; +} + +export type AclResult = + | { allow: true } + | { allow: false; reason: AclDenyReason }; + +export type AclDenyReason = + | "not-member" + | "not-own-record" + | "app-not-allowed" + | "unknown-op"; + +/** Check whether the caller's app is permitted to act in this space. */ +export function checkAppPolicy( + appPolicy: AppPolicy | null, + clientId: string | undefined +): boolean { + if (!appPolicy) return true; // no policy = allow-all + const listed = clientId ? appPolicy.apps.includes(clientId) : false; + if (appPolicy.mode === "allow") return !listed; // apps[] is a denylist + return listed; // mode === "deny": apps[] is an allowlist +} + +const isOwner = (space: SpaceRow, did: string) => space.ownerDid === did; +const hasMember = (space: SpaceRow, member: SpaceMemberRow | null, did: string) => + isOwner(space, did) || member != null; + +/** Space-level access check. + * Membership = access. Any member (including owner) can read and write. + * Delete is scoped to the caller's own records — owners don't get a bypass. + * A random member can't nuke other people's records, and neither can the + * owner. To remove a non-author record, delete the whole space. */ +export function checkAccess(input: AclInput): AclResult { + if (!checkAppPolicy(input.space.appPolicy, input.clientId)) { + return { allow: false, reason: "app-not-allowed" }; + } + + if (input.op === "read" || input.op === "write") { + return hasMember(input.space, input.member, input.callerDid) + ? { allow: true } + : { allow: false, reason: "not-member" }; + } + + if (input.op === "delete") { + if (!hasMember(input.space, input.member, input.callerDid)) { + return { allow: false, reason: "not-member" }; + } + if (input.targetAuthorDid && input.targetAuthorDid !== input.callerDid) { + return { allow: false, reason: "not-own-record" }; + } + return { allow: true }; + } + + return { allow: false, reason: "unknown-op" }; +} diff --git a/packages/contrail-base/src/spaces/auth.ts b/packages/contrail-base/src/spaces/auth.ts new file mode 100644 index 0000000..4c1e817 --- /dev/null +++ b/packages/contrail-base/src/spaces/auth.ts @@ -0,0 +1,177 @@ +import type { Context, MiddlewareHandler } from "hono"; +import { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, + type DidDocumentResolver, +} from "@atcute/identity-resolver"; +import type { Did, Nsid } from "@atcute/lexicons"; +import type { AuthorityConfig } from "./types"; +import { readInProcess } from "./in-process"; + +export { ServiceJwtVerifier }; + +/** Build a ServiceJwtVerifier from an AuthorityConfig, using the configured + * resolver or a default PLC+Web composite. The verifier checks that incoming + * JWTs target this authority's serviceDid (aud claim). */ +export function buildVerifier(authority: AuthorityConfig): ServiceJwtVerifier { + const resolver = + authority.resolver ?? + new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, + }); + return new ServiceJwtVerifier({ + serviceDid: authority.serviceDid as Did, + resolver, + }); +} + +export interface ServiceAuth { + issuer: string; + audience: string; + lxm: string | undefined; + /** OAuth client_id of the caller, if the JWT carries one. */ + clientId?: string; +} + +export interface ServiceAuthOptions { + serviceDid: Did; + resolver: DidDocumentResolver; +} + +/** Hono middleware that authenticates XRPC requests. Order of precedence: + * 1. In-process marker (same-module calls; see `core/spaces/in-process.ts`) + * 2. Authorization: Bearer <JWT> as an atproto service-auth token + * + * On success, attaches the claims to `c.var.serviceAuth`. Expected Nsid is + * taken from the route pattern (last segment after `/xrpc/`). */ +export function createServiceAuthMiddleware( + verifier: ServiceJwtVerifier +): MiddlewareHandler { + return async (c, next) => { + const lxm = extractLxmFromPath(c); + + const inProcess = readInProcess(c.req.raw); + if (inProcess) { + c.set("serviceAuth", { + issuer: inProcess.did, + audience: "", + lxm: lxm ?? undefined, + } satisfies ServiceAuth); + await next(); + return; + } + + const header = c.req.header("Authorization"); + if (!header || !header.startsWith("Bearer ")) { + return c.json({ error: "AuthRequired", message: "Missing bearer token" }, 401); + } + const token = header.slice(7).trim(); + + const result = await verifier.verify(token, { lxm }); + if (!result.ok) { + const err = result.error as { error?: string; description?: string } | undefined; + return c.json( + { + error: "AuthRequired", + message: err?.description ?? err?.error ?? String(result.error), + }, + 401, + ); + } + + c.set("serviceAuth", { + issuer: result.value.issuer, + audience: result.value.audience, + lxm: result.value.lxm, + } satisfies ServiceAuth); + + await next(); + }; +} + +function extractLxmFromPath(c: Context): Nsid | null { + const path = new URL(c.req.url).pathname; + const match = path.match(/\/xrpc\/([a-zA-Z0-9.-]+)/); + return (match?.[1] as Nsid) ?? null; +} + +/** Read the service auth claims set by the middleware. Throws if unset. */ +export function requireServiceAuth(c: Context): ServiceAuth { + const auth = c.get("serviceAuth") as ServiceAuth | undefined; + if (!auth) throw new Error("service auth missing; middleware not attached"); + return auth; +} + +/** Out-of-band auth check for handlers that don't always require auth. + * Returns claims on success, or null if no valid credentials are present. + * Order of precedence: in-process marker → service-auth JWT. */ +export async function verifyServiceAuthRequest( + verifier: ServiceJwtVerifier, + request: Request, + lxm?: Nsid | null +): Promise<ServiceAuth | null> { + const inProcess = readInProcess(request); + if (inProcess) { + return { + issuer: inProcess.did, + audience: "", + lxm: lxm ?? undefined, + }; + } + + const header = request.headers.get("Authorization"); + if (!header || !header.startsWith("Bearer ")) return null; + const token = header.slice(7).trim(); + const result = await verifier.verify(token, { lxm: lxm ?? null }); + if (!result.ok) return null; + return { + issuer: result.value.issuer, + audience: result.value.audience, + lxm: result.value.lxm, + }; +} + +/** Pull a space credential off the request — `X-Space-Credential: <jwt>` + * header. Returns the raw token or null. */ +export function extractSpaceCredential(request: Request): string | null { + const header = request.headers.get("X-Space-Credential"); + return header ? header.trim() : null; +} + +/** Pull a read-grant invite token off the request — query string `?inviteToken=` + * or `Authorization: Bearer atmo-invite:<token>`. Returns the raw token (not + * hashed) or null. Routes hash + look up via the adapter. */ +export function extractInviteToken(request: Request): string | null { + const url = new URL(request.url); + const q = url.searchParams.get("inviteToken"); + if (q) return q.trim(); + const header = request.headers.get("Authorization"); + if (header?.startsWith("Bearer atmo-invite:")) { + return header.slice("Bearer atmo-invite:".length).trim(); + } + return null; +} + +/** Validate a read-grant invite token against a target spaceUri. Returns true + * if the token exists, scopes to this space, has a kind that grants read + * (`read` or `read-join`), and is not expired/revoked. */ +export async function checkInviteReadGrant( + adapter: { getInvite(tokenHash: string): Promise<{ spaceUri: string; kind: string; revokedAt: number | null; expiresAt: number | null } | null> }, + rawToken: string, + spaceUri: string, + hashFn: (token: string) => Promise<string> +): Promise<boolean> { + const tokenHash = await hashFn(rawToken); + const invite = await adapter.getInvite(tokenHash); + if (!invite) return false; + if (invite.spaceUri !== spaceUri) return false; + if (invite.kind !== "read" && invite.kind !== "read-join") return false; + if (invite.revokedAt != null) return false; + if (invite.expiresAt != null && invite.expiresAt <= Date.now()) return false; + return true; +} diff --git a/packages/contrail-base/src/spaces/binding.ts b/packages/contrail-base/src/spaces/binding.ts new file mode 100644 index 0000000..a81b95f --- /dev/null +++ b/packages/contrail-base/src/spaces/binding.ts @@ -0,0 +1,274 @@ +/** Binding resolution: given a space URI, which DID is authorized to sign + * credentials for it, and where do we find that DID's verification key? + * + * Two layers of pluggable resolvers compose into a credential verifier: + * + * BindingResolver — `ats://<owner>/<type>/<key>` → authority DID + * KeyResolver — (DID, kid) → JsonWebKey + * + * The BindingResolver is what makes user-owned-DID-with-PDS-record work: + * given a space URI, we resolve the owner's PDS, fetch the declaration + * record, and read its `authority` field. For provisioned (no-PDS) DIDs we + * fall back to the owner DID's `#atproto_space_authority` service entry. + * And finally for the trivial case (HappyView-style "owner self-issues"), + * we return the owner DID itself. + * + * See conversation history (phase 4 design) for the rationale on why these + * three sources, in this order. */ + +import type { DidDocumentResolver } from "@atcute/identity-resolver"; +import type { Did } from "@atcute/lexicons"; +import { parseSpaceUri } from "./uri"; +import type { RecordHost } from "./types"; + +export interface BindingResolver { + /** Resolve the DID authorized to sign credentials for this space. Returns + * null if no binding could be found via this resolver — the composite + * walks down its list looking for a non-null. */ + resolveAuthority(spaceUri: string): Promise<string | null>; +} + +export interface KeyResolver { + /** Resolve `did`'s verification key for credential signing. `kid` is the + * full header `kid` value (e.g. "did:web:x.com#atproto_space_authority"), + * used to disambiguate when a DID doc lists multiple methods. */ + resolveKey(did: string, kid: string | undefined): Promise<JsonWebKey | null>; +} + +// --------------------------------------------------------------------------- +// Binding resolvers +// --------------------------------------------------------------------------- + +/** Always returns the configured authority DID. Used in-process when the + * authority and record host run in one deployment — no need to walk DID + * docs or PDSes; we know what we are. */ +export function createLocalBindingResolver(args: { + authorityDid: string; +}): BindingResolver { + const { authorityDid } = args; + return { + async resolveAuthority() { + return authorityDid; + }, + }; +} + +/** Reads the record host's local enrollment table. This is the *canonical* + * binding source on a record host: the host owner explicitly consented to a + * given authority for a given space (via the `recordHost.enroll` endpoint + * or auto-enrollment from the authority's createSpace). PDS-record / + * DID-doc resolvers are out-of-band discovery aids; the enrollment is what + * actually gates whether records get stored here. */ +export function createEnrollmentBindingResolver(args: { + recordHost: RecordHost; +}): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const e = await args.recordHost.getEnrollment(spaceUri); + return e?.authorityDid ?? null; + }, + }; +} + +/** Returns the space owner DID as the authority. This is the implicit + * fallback ("HappyView path") — when no PDS record and no DID-doc service + * entry declare an issuer, the owner is taken to be its own. Whether the + * resulting credential actually verifies depends on whether the owner's DID + * doc publishes a usable signing key. */ +export function createOwnerSelfBindingResolver(): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + return parts ? parts.ownerDid : null; + }, + }; +} + +/** Walks the resolver list in order, returns the first non-null. Use this + * to compose [pdsRecord, didDocService, ownerSelf] etc. */ +export function createCompositeBindingResolver( + resolvers: BindingResolver[] +): BindingResolver { + return { + async resolveAuthority(spaceUri) { + for (const r of resolvers) { + const did = await r.resolveAuthority(spaceUri); + if (did) return did; + } + return null; + }, + }; +} + +/** Reads a space-declaration record from the owner's PDS at + * `at://<owner>/<type>/<key>` and returns its `authority` field if present. + * + * This is the user-owned-DID path: the user writes a record to their PDS + * authorizing some service as the space's authority, no DID-doc edits + * required. */ +export function createPdsBindingResolver(args: { + /** DID resolver, used to look up the owner's PDS endpoint. */ + resolver: DidDocumentResolver; + /** Fetch impl. Defaults to `globalThis.fetch`. */ + fetch?: typeof fetch; + /** Per-request timeout in ms. Defaults to 5000. */ + timeoutMs?: number; +}): BindingResolver { + const fetchImpl = args.fetch ?? globalThis.fetch; + const timeoutMs = args.timeoutMs ?? 5000; + + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + const pds = await pdsEndpointFor(args.resolver, parts.ownerDid); + if (!pds) return null; + + const url = new URL(`${pds}/xrpc/com.atproto.repo.getRecord`); + url.searchParams.set("repo", parts.ownerDid); + url.searchParams.set("collection", parts.type); + url.searchParams.set("rkey", parts.key); + + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + let res: Response; + try { + res = await fetchImpl(url.toString(), { signal: ctrl.signal }); + } catch { + return null; + } finally { + clearTimeout(timer); + } + if (!res.ok) return null; + const body = (await res.json().catch(() => null)) as + | { value?: { authority?: unknown } } + | null; + const authority = body?.value?.authority; + return typeof authority === "string" && authority.startsWith("did:") ? authority : null; + }, + }; +} + +/** Reads `service[id="#atproto_space_authority"].serviceEndpoint` from the + * owner's DID doc. This is the no-PDS path — useful for provisioned space + * DIDs that exist as DID docs only. + * + * Note the service endpoint here is a *DID*, not a URL. The DID names the + * authority; the key resolver's job is to then fetch its verification key. + * For DID docs that declare a URL endpoint, we treat the URL as a + * did:web hint — caller can normalize. */ +export function createDidDocBindingResolver(args: { + resolver: DidDocumentResolver; + /** Service id to look up. Defaults to "#atproto_space_authority". */ + serviceId?: string; +}): BindingResolver { + const serviceId = args.serviceId ?? "#atproto_space_authority"; + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + let doc; + try { + doc = await args.resolver.resolve(parts.ownerDid as Did); + } catch { + return null; + } + const entry = doc.service?.find((s: { id?: string }) => s.id === serviceId); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + if (typeof endpoint !== "string") return null; + // Endpoint may be a DID (preferred) or a URL hint. Only DIDs are + // verifiable downstream; URLs require the caller to map URL → DID. + return endpoint.startsWith("did:") ? endpoint : null; + }, + }; +} + +// --------------------------------------------------------------------------- +// Key resolvers +// --------------------------------------------------------------------------- + +/** Knows the local authority's public key directly. Returns null for any + * other DID — composite with a DID-doc resolver if you also accept + * external authorities. */ +export function createLocalKeyResolver(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): KeyResolver { + return { + async resolveKey(did) { + return did === args.authorityDid ? args.publicKey : null; + }, + }; +} + +/** Resolves a DID, finds the verification method matching `kid`, returns + * its `publicKeyJwk`. */ +export function createDidDocKeyResolver(args: { + resolver: DidDocumentResolver; +}): KeyResolver { + return { + async resolveKey(did, kid) { + let doc; + try { + doc = await args.resolver.resolve(did as Did); + } catch { + return null; + } + const methods = (doc as { verificationMethod?: VerificationMethod[] }).verificationMethod; + if (!methods) return null; + // kid is "<did>#<methodId>" — we match against the method.id which DID + // docs spell as "<did>#<methodId>" too. + const method = kid + ? methods.find((m) => m.id === kid) + : methods[0]; + if (!method?.publicKeyJwk) return null; + return method.publicKeyJwk as JsonWebKey; + }, + }; +} + +/** Walks resolvers in order; returns the first non-null. */ +export function createCompositeKeyResolver( + resolvers: KeyResolver[] +): KeyResolver { + return { + async resolveKey(did, kid) { + for (const r of resolvers) { + const k = await r.resolveKey(did, kid); + if (k) return k; + } + return null; + }, + }; +} + +interface VerificationMethod { + id: string; + type?: string; + controller?: string; + publicKeyJwk?: unknown; + publicKeyMultibase?: string; +} + +// --------------------------------------------------------------------------- +// Internal: PDS endpoint lookup +// --------------------------------------------------------------------------- + +async function pdsEndpointFor( + resolver: DidDocumentResolver, + did: string +): Promise<string | null> { + let doc; + try { + doc = await resolver.resolve(did as Did); + } catch { + return null; + } + const entry = doc.service?.find( + (s: { id?: string }) => s.id === "#atproto_pds" + ); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + return typeof endpoint === "string" ? endpoint : null; +} diff --git a/packages/contrail-base/src/spaces/blob-adapter.ts b/packages/contrail-base/src/spaces/blob-adapter.ts new file mode 100644 index 0000000..632b566 --- /dev/null +++ b/packages/contrail-base/src/spaces/blob-adapter.ts @@ -0,0 +1,94 @@ +/** + * Bytes-only storage adapter for space blobs. Metadata (CID, mime, size, + * author, space) lives in the `spaces_blobs` table on the main StorageAdapter; + * this interface only moves bytes in and out of a backend (R2, S3, fs, …). + * + * Keys are opaque strings formed by the router as `blobKey(spaceUri, cid)`. + */ + +export interface BlobUploadMeta { + mimeType: string; + size: number; +} + +export interface BlobAdapter { + put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void>; + get(key: string): Promise<Uint8Array | null>; + /** Bulk delete. Adapters that don't support batch can implement serially. */ + delete(keys: string[]): Promise<void>; +} + +/** In-memory adapter. Useful for tests and local development. */ +export class MemoryBlobAdapter implements BlobAdapter { + private readonly store = new Map<string, Uint8Array>(); + + async put(key: string, bytes: Uint8Array): Promise<void> { + this.store.set(key, bytes.slice()); + } + + async get(key: string): Promise<Uint8Array | null> { + const v = this.store.get(key); + return v ? v.slice() : null; + } + + async delete(keys: string[]): Promise<void> { + for (const k of keys) this.store.delete(k); + } + + /** Test helper. */ + size(): number { + return this.store.size; + } +} + +/** Minimal Cloudflare R2 bucket shape — matches @cloudflare/workers-types' R2Bucket + * without forcing a types dependency here. */ +export interface R2BucketLike { + put( + key: string, + value: ArrayBuffer | ArrayBufferView | ReadableStream | Blob, + options?: { httpMetadata?: { contentType?: string }; customMetadata?: Record<string, string> } + ): Promise<unknown>; + get(key: string): Promise<{ arrayBuffer(): Promise<ArrayBuffer> } | null>; + delete(keys: string | string[]): Promise<void>; +} + +/** Cloudflare R2 adapter. Pass the `env.BLOBS` binding from your Worker. */ +export class R2BlobAdapter implements BlobAdapter { + constructor(private readonly bucket: R2BucketLike) {} + + async put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void> { + await this.bucket.put(key, bytes, { + httpMetadata: { contentType: meta.mimeType }, + }); + } + + async get(key: string): Promise<Uint8Array | null> { + const obj = await this.bucket.get(key); + if (!obj) return null; + const buf = await obj.arrayBuffer(); + return new Uint8Array(buf); + } + + async delete(keys: string[]): Promise<void> { + if (keys.length === 0) return; + await this.bucket.delete(keys); + } +} + +/** Hash a space URI to a short, filesystem/R2-safe key segment. + * Used as the first segment of a blob key so all blobs for one space + * share a common prefix (enables bulk delete on space deletion). */ +export async function spaceKeyPrefix(spaceUri: string): Promise<string> { + const bytes = new TextEncoder().encode(spaceUri); + const digest = await crypto.subtle.digest("SHA-256", bytes); + const hex = Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join(""); + return hex.slice(0, 16); +} + +/** Compose an adapter key from a space URI and CID. + * Shape: `<16-hex-chars-of-sha256(spaceUri)>/<cid>`. */ +export async function blobKey(spaceUri: string, cid: string): Promise<string> { + const prefix = await spaceKeyPrefix(spaceUri); + return `${prefix}/${cid}`; +} diff --git a/packages/contrail-base/src/spaces/credentials.ts b/packages/contrail-base/src/spaces/credentials.ts new file mode 100644 index 0000000..741b82d --- /dev/null +++ b/packages/contrail-base/src/spaces/credentials.ts @@ -0,0 +1,284 @@ +/** Space-credential primitives: ES256 (P-256) JWTs minted by the authority, + * verified by the record host (or any third party that can resolve the + * authority's DID document). + * + * Format is a compact JWS: + * header = { alg: "ES256", typ: "JWT", kid: "<authorityDid>#<keyId>" } + * payload = { iss, sub, space, scope, iat, exp } + * + * - `iss` is the authority DID (the signer; for phase 3 this is the local + * authority's serviceDid; phase 4 adds a binding-resolution layer that + * lets the issuer be a *different* DID from the space owner). + * - `sub` is the caller DID — the credential bearer. + * - `space` is the full `ats://<owner>/<type>/<key>` URI. + * - `scope` is "rw" or "read". + * + * We don't use a JWT library — Web Crypto's subtle covers everything (P-256 + * generate, sign, verify, JWK import/export) and saves a runtime dep. */ + +const ALG = "ES256"; +const TYP = "JWT"; +const DEFAULT_KEY_ID = "atproto_space_authority"; + +export type CredentialScope = "rw" | "read"; + +export interface CredentialClaims { + iss: string; + sub: string; + space: string; + scope: CredentialScope; + iat: number; // seconds since epoch + exp: number; // seconds since epoch +} + +export interface CredentialKeyMaterial { + /** Private key in JWK form. P-256 / ES256. */ + privateKey: JsonWebKey; + /** Public key in JWK form. Must match privateKey. */ + publicKey: JsonWebKey; + /** DID-doc verification method id. The full JWT `kid` becomes + * `<authorityDid>#<keyId>`. Defaults to "atproto_space_authority". */ + keyId?: string; +} + +/** Generate a fresh P-256 keypair as JWKs. Useful for local dev / tests; in + * production the operator generates once and stores out-of-band. */ +export async function generateAuthoritySigningKey(): Promise<CredentialKeyMaterial> { + const pair = (await crypto.subtle.generateKey( + { name: "ECDSA", namedCurve: "P-256" }, + true, + ["sign", "verify"] + )) as CryptoKeyPair; + const privateKey = (await crypto.subtle.exportKey("jwk", pair.privateKey)) as JsonWebKey; + const publicKey = (await crypto.subtle.exportKey("jwk", pair.publicKey)) as JsonWebKey; + return { privateKey, publicKey }; +} + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function base64urlEncode(bytes: Uint8Array): string { + let s = btoa(String.fromCharCode(...bytes)); + return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +function base64urlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function jsonEncode(value: unknown): string { + return base64urlEncode(enc.encode(JSON.stringify(value))); +} + +function jsonDecode<T>(seg: string): T { + return JSON.parse(dec.decode(base64urlDecode(seg))) as T; +} + +async function importPrivate(jwk: JsonWebKey): Promise<CryptoKey> { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["sign"] + ); +} + +async function importPublic(jwk: JsonWebKey): Promise<CryptoKey> { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"] + ); +} + +/** Sign a credential payload with the authority's private key. + * `iat` and `exp` are filled in by the caller (so tests can mint expired + * tokens deterministically). */ +export async function signCredential( + payload: CredentialClaims, + key: CredentialKeyMaterial +): Promise<string> { + const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; + const header = { alg: ALG, typ: TYP, kid }; + const head = jsonEncode(header); + const body = jsonEncode(payload); + const signingInput = `${head}.${body}`; + const privateKey = await importPrivate(key.privateKey); + const sig = await crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + privateKey, + enc.encode(signingInput) + ); + return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; +} + +/** Issue a credential using the current wall-clock for iat/exp. */ +export async function issueCredential( + args: Omit<CredentialClaims, "iat" | "exp"> & { ttlMs: number }, + key: CredentialKeyMaterial +): Promise<{ credential: string; expiresAt: number }> { + const now = Math.floor(Date.now() / 1000); + const expSec = now + Math.floor(args.ttlMs / 1000); + const claims: CredentialClaims = { + iss: args.iss, + sub: args.sub, + space: args.space, + scope: args.scope, + iat: now, + exp: expSec, + }; + const credential = await signCredential(claims, key); + return { credential, expiresAt: expSec * 1000 }; +} + +export type VerifyOk = { ok: true; claims: CredentialClaims }; +export type VerifyErr = { + ok: false; + reason: + | "malformed" + | "bad-alg" + | "bad-signature" + | "expired" + | "not-yet-valid" + | "wrong-space" + | "wrong-scope" + | "unknown-issuer"; +}; + +export interface VerifyOptions { + /** Optional: when set, rejects credentials whose `space` claim differs. + * Omit when verifying in middleware where the target space isn't known + * yet — handlers can do the match themselves against the verified + * claims. */ + expectedSpace?: string; + /** Optional: required scope (e.g. "rw" rejects read-only credentials on writes). */ + requiredScope?: CredentialScope; + /** Resolve a verification key for `iss`. If null, verification fails with + * unknown-issuer. */ + resolveKey: (iss: string, kid: string | undefined) => Promise<JsonWebKey | null>; + /** Time provider for tests. Returns ms since epoch. */ + now?: () => number; +} + +export async function verifyCredential( + jwt: string, + opts: VerifyOptions +): Promise<VerifyOk | VerifyErr> { + const parts = jwt.split("."); + if (parts.length !== 3) return { ok: false, reason: "malformed" }; + const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; + + let header: { alg?: string; typ?: string; kid?: string }; + let claims: CredentialClaims; + try { + header = jsonDecode(headSeg); + claims = jsonDecode(bodySeg); + } catch { + return { ok: false, reason: "malformed" }; + } + if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; + if (opts.expectedSpace !== undefined && claims.space !== opts.expectedSpace) { + return { ok: false, reason: "wrong-space" }; + } + if (opts.requiredScope === "rw" && claims.scope !== "rw") { + return { ok: false, reason: "wrong-scope" }; + } + + const nowMs = (opts.now ?? Date.now)(); + const nowSec = Math.floor(nowMs / 1000); + if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; + if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; + + const jwk = await opts.resolveKey(claims.iss, header.kid); + if (!jwk) return { ok: false, reason: "unknown-issuer" }; + + const publicKey = await importPublic(jwk); + const sigBytes = base64urlDecode(sigSeg); + const signingInput = `${headSeg}.${bodySeg}`; + const valid = await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + sigBytes as BufferSource, + enc.encode(signingInput) + ); + if (!valid) return { ok: false, reason: "bad-signature" }; + return { ok: true, claims }; +} + +/** Header reader for handlers that want to peek at `iss` before resolving the + * key (e.g. to short-circuit DID-doc fetches for the local authority). */ +export function decodeUnverifiedClaims(jwt: string): CredentialClaims | null { + const parts = jwt.split("."); + if (parts.length !== 3) return null; + try { + return jsonDecode<CredentialClaims>(parts[1]!); + } catch { + return null; + } +} + +/** Verifier interface consumed by the record host. The record host doesn't + * care HOW credentials get verified — it only cares whether a given JWT is + * valid. Phase 3 ships an in-process verifier that knows the local + * authority's public key; phase 4 adds a binding-resolving verifier that + * consults PDS records / DID docs. */ +export interface CredentialVerifier { + /** Verify a credential's signature, expiry, and `not-before` window. Does + * NOT enforce a space match — handlers do that against the request URI. */ + verify(jwt: string): Promise<VerifyOk | VerifyErr>; +} + +/** In-process verifier for the simple deployment: the authority and record + * host run in one process and the record host has direct access to the + * authority's public key. Rejects any credential whose `iss` isn't the + * configured authority. Phase 4 has a more general + * {@link createBindingCredentialVerifier} that does proper binding lookup. */ +export function createInProcessVerifier(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): CredentialVerifier { + return { + verify(jwt) { + return verifyCredential(jwt, { + resolveKey: async (iss) => (iss === args.authorityDid ? args.publicKey : null), + }); + }, + }; +} + +/** Verifier composed of a {@link BindingResolver} (which DID is authorized + * to issue for this space?) and a {@link KeyResolver} (what's that DID's + * public key?). This is the production-shape verifier — phase 4's main + * contribution. + * + * Verification flow: + * 1. Decode the JWT's claims (no signature check yet). + * 2. Ask the binding resolver: who's authorized for `claims.space`? + * 3. Confirm `claims.iss === authorizedDid`. + * 4. Ask the key resolver for that DID's verification key. + * 5. Verify signature + expiry + scope match. + */ +export function createBindingCredentialVerifier(args: { + bindings: import("./binding").BindingResolver; + keys: import("./binding").KeyResolver; +}): CredentialVerifier { + return { + async verify(jwt) { + const peek = decodeUnverifiedClaims(jwt); + if (!peek) return { ok: false, reason: "malformed" }; + const authorizedDid = await args.bindings.resolveAuthority(peek.space); + if (!authorizedDid) return { ok: false, reason: "unknown-issuer" }; + if (peek.iss !== authorizedDid) return { ok: false, reason: "unknown-issuer" }; + return verifyCredential(jwt, { + resolveKey: (iss, kid) => args.keys.resolveKey(iss, kid), + }); + }, + }; +} diff --git a/packages/contrail-base/src/spaces/in-process.ts b/packages/contrail-base/src/spaces/in-process.ts new file mode 100644 index 0000000..ec1d7a1 --- /dev/null +++ b/packages/contrail-base/src/spaces/in-process.ts @@ -0,0 +1,34 @@ +/** In-process auth marker. + * + * For same-module callers (e.g. a SvelteKit worker that imports contrail and + * dispatches requests directly to the handler), service-auth JWTs are pure + * overhead: no network boundary is crossed, so there's nothing for the JWT to + * protect against. Instead, the caller tags the `Request` with a principal via + * a module-private WeakMap, and the auth middleware reads it back. + * + * Security note: this is unforgeable from outside the module because + * - WeakMap keys are `Request` object identities, not serialized data; + * - no HTTP request crossing a network boundary can reach into this map; + * - exploiting it requires code execution inside the same isolate, at + * which point auth is already game over. + * + * This is the strongest auth adapter contrail offers — it has no secret to + * leak. */ + +export interface InProcessPrincipal { + did: string; +} + +const PRINCIPALS = new WeakMap<Request, InProcessPrincipal>(); + +/** Tag a Request with an in-process principal. The returned Request is the + * same reference; the return value is for ergonomics. */ +export function markInProcess(req: Request, did: string): Request { + PRINCIPALS.set(req, { did }); + return req; +} + +/** Read the in-process principal for a Request, or null if unmarked. */ +export function readInProcess(req: Request): InProcessPrincipal | null { + return PRINCIPALS.get(req) ?? null; +} diff --git a/packages/contrail-base/src/spaces/tid.ts b/packages/contrail-base/src/spaces/tid.ts new file mode 100644 index 0000000..c419090 --- /dev/null +++ b/packages/contrail-base/src/spaces/tid.ts @@ -0,0 +1,20 @@ +const B32_CHARSET = "234567abcdefghijklmnopqrstuvwxyz"; + +let lastTimestamp = 0; +let clockId = Math.floor(Math.random() * 1024); + +/** Generate an atproto TID: 13-char base32-sortable (timestamp-ordered). */ +export function nextTid(): string { + let now = Date.now() * 1000; + if (now <= lastTimestamp) now = lastTimestamp + 1; + lastTimestamp = now; + + const n = BigInt(now) * 1024n + BigInt(clockId); + let s = ""; + let v = n; + for (let i = 0; i < 13; i++) { + s = B32_CHARSET[Number(v & 31n)] + s; + v >>= 5n; + } + return s; +} diff --git a/packages/contrail-base/src/spaces/types.ts b/packages/contrail-base/src/spaces/types.ts new file mode 100644 index 0000000..6561773 --- /dev/null +++ b/packages/contrail-base/src/spaces/types.ts @@ -0,0 +1,273 @@ +import type { Database } from "../types"; +import type { DidDocumentResolver } from "@atcute/identity-resolver"; +import type { BlobAdapter } from "./blob-adapter"; +import type { CredentialKeyMaterial } from "./credentials"; + +export type AppPolicyMode = "allow" | "deny"; + +export interface AppPolicy { + mode: AppPolicyMode; + apps: string[]; +} + +export interface SpacesBlobsConfig { + /** Bytes backend (R2, S3, in-memory, …). */ + adapter: BlobAdapter; + /** Max blob size in bytes. Defaults to 2 MiB. */ + maxSize?: number; + /** MIME allowlist. If set, only these content types are accepted. */ + accept?: string[]; + /** Orphan blobs (those with no referencing record) are kept this long before + * GC can delete them, to allow upload-then-putRecord flows. + * Defaults to 24 hours. */ + gcOrphanAfterMs?: number; +} + +export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; +export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; + +/** Default credential lifetime. The rough spec calls for 2–4h; we pick the + * lower bound so revocation (kicked-from-space) is observable within 2h. */ +export const DEFAULT_CREDENTIAL_TTL_MS = 2 * 60 * 60 * 1000; + +/** Configuration for the **space authority** role: holds the member list, + * signs credentials, and gates space-management operations. In a fully-split + * deployment, the authority can run in a different process (or even a + * different operator) than the record host. */ +export interface AuthorityConfig { + /** NSID that identifies the kind of space this authority hosts, + * e.g. "tools.atmo.event.space". */ + type: string; + /** Service DID that service-auth tokens must target (aud claim) AND that + * signs credentials it issues (`iss` claim on emitted JWTs). */ + serviceDid: string; + /** Default app policy applied to new spaces. */ + defaultAppPolicy?: AppPolicy; + /** DID document resolver for service-auth JWT verification. + * Defaults to a composite PLC + did:web resolver if omitted. */ + resolver?: DidDocumentResolver; + /** Signing key material for issuing space credentials. When omitted, + * `<ns>.space.getCredential` returns 501 NotImplemented and the record + * host's credential-verifying middleware can't be wired up. */ + signing?: CredentialKeyMaterial; + /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ + credentialTtlMs?: number; +} + +/** Configuration for the **record host** role: stores per-space records and + * blobs and serves reads. Verifies space credentials (later phases) on + * incoming traffic. */ +export interface RecordHostConfig { + /** Blob-upload backend. When omitted, blob XRPCs are not exposed. */ + blobs?: SpacesBlobsConfig; +} + +/** Spaces config — host an authority, a record host, or both. + * Today both run in one process and most deployments will set both; the + * shape is split now so phase 5 can run them independently without churning + * every consumer's config. */ +export interface SpacesConfig { + /** Space-authority config — member list, credentials (later), space + * management. Required for any space to exist. */ + authority?: AuthorityConfig; + /** Record-host config — record + blob storage. Required for records to be + * written/read on this deployment. */ + recordHost?: RecordHostConfig; +} + +export interface SpaceRow { + uri: string; + ownerDid: string; + type: string; + key: string; + serviceDid: string; + appPolicyRef: string | null; + appPolicy: AppPolicy | null; + createdAt: number; + deletedAt: number | null; +} + +export interface SpaceMemberRow { + spaceUri: string; + did: string; + addedAt: number; + addedBy: string | null; +} + +export interface StoredRecord { + spaceUri: string; + collection: string; + authorDid: string; + rkey: string; + cid: string | null; + record: Record<string, unknown>; + createdAt: number; +} + +export interface ListOptions { + byUser?: string; + cursor?: string; + limit?: number; +} + +export interface ListResult { + records: StoredRecord[]; + cursor?: string; +} + +export interface ListSpacesOptions { + type?: string; + ownerDid?: string; + memberDid?: string; + limit?: number; + cursor?: string; +} + +export interface CollectionCount { + collection: string; + count: number; +} + +/** What a token holder can do with this invite. + * - `'join'`: must be redeemed while signed in; redeemer becomes a member. + * - `'read'`: bearer-only — token itself grants read access to the space; cannot be redeemed. + * - `'read-join'`: both — anonymous holders read; signed-in holders may also redeem to join. */ +export type InviteKind = "join" | "read" | "read-join"; + +export interface InviteRow { + tokenHash: string; + spaceUri: string; + kind: InviteKind; + expiresAt: number | null; + maxUses: number | null; + usedCount: number; + createdBy: string; + createdAt: number; + revokedAt: number | null; + note: string | null; +} + +export interface CreateInviteInput { + spaceUri: string; + tokenHash: string; + kind: InviteKind; + expiresAt: number | null; + maxUses: number | null; + createdBy: string; + note: string | null; +} + +export interface RedeemInviteResult { + spaceUri: string; +} + +export interface BlobMetaRow { + spaceUri: string; + cid: string; + mimeType: string; + size: number; + authorDid: string; + createdAt: number; +} + +/** Local cache on the record host: this space is accepted here, and `authority_did` + * is the DID authorized to sign credentials for it. Populated via the + * `recordHost.enroll` endpoint or auto-populated by the authority's + * createSpace when both roles run in one process. */ +export interface EnrollmentRow { + spaceUri: string; + authorityDid: string; + enrolledAt: number; + enrolledBy: string; +} + +export interface ListBlobsOptions { + byUser?: string; + cursor?: string; + limit?: number; +} + +export interface ListBlobsResult { + blobs: BlobMetaRow[]; + cursor?: string; +} + +/** **Space authority** interface — owner of the space's ACL state and + * (eventually) credential issuer. Holds the member list, manages invites, + * governs space lifecycle and app policy. Does NOT touch records or blobs. + * + * In a fully-split deployment this is a separate service; today the + * HostedAdapter implements both this and {@link RecordHost} against one DB. */ +export interface SpaceAuthority { + // Space lifecycle + createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow>; + getSpace(spaceUri: string): Promise<SpaceRow | null>; + listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }>; + deleteSpace(spaceUri: string): Promise<void>; + updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void>; + + // Members + addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void>; + removeMember(spaceUri: string, did: string): Promise<void>; + getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null>; + listMembers(spaceUri: string): Promise<SpaceMemberRow[]>; + /** Bulk-apply a membership diff. Used only by the community module's reconciler; + * not exposed as an XRPC endpoint. */ + applyMembershipDiff( + spaceUri: string, + adds: string[], + removes: string[], + addedBy: string | null + ): Promise<void>; + + // Invites (token primitive — issued by the authority, scoped to a space) + createInvite(input: CreateInviteInput): Promise<InviteRow>; + listInvites(spaceUri: string, options?: { includeRevoked?: boolean }): Promise<InviteRow[]>; + revokeInvite(tokenHash: string): Promise<boolean>; + /** Look up an invite without consuming it. Used to validate read-token bearer access. */ + getInvite(tokenHash: string): Promise<InviteRow | null>; + /** Atomically mark a join-capable invite as used. Returns the row if usable + * (kind allows join, not expired/revoked/exhausted), null otherwise. */ + redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null>; +} + +/** **Record host** interface — stores records and blobs for a space, plus + * the local enrollment table that decides which spaces this host accepts. + * + * Trust model: the host trusts whatever credential the authority signs, so + * long as the authority is the one named in the local enrollment for this + * space. Enrollment is the consent step — the host owner agrees to spend + * storage on a given space, scoped to a specific authority. */ +export interface RecordHost { + // Enrollment + enroll(input: EnrollmentRow): Promise<void>; + getEnrollment(spaceUri: string): Promise<EnrollmentRow | null>; + listEnrollments(options?: { authorityDid?: string; limit?: number }): Promise<EnrollmentRow[]>; + removeEnrollment(spaceUri: string): Promise<void>; + + // Records + putRecord(record: StoredRecord): Promise<void>; + getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<StoredRecord | null>; + listRecords(spaceUri: string, collection: string, options?: ListOptions): Promise<ListResult>; + deleteRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<void>; + listCollections(spaceUri: string, options?: { byUser?: string }): Promise<CollectionCount[]>; + + // Blobs (metadata only; bytes live on BlobAdapter) + putBlobMeta(row: BlobMetaRow): Promise<void>; + getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null>; + listBlobMeta(spaceUri: string, options?: ListBlobsOptions): Promise<ListBlobsResult>; + deleteBlobMeta(spaceUri: string, cid: string): Promise<void>; + /** Find blob rows older than `cutoff` whose CIDs are not referenced in any + * record JSON in this space. Capped at `limit` to bound a single GC pass. */ + findOrphanBlobs(spaceUri: string, cutoff: number, limit: number): Promise<BlobMetaRow[]>; +} + +/** Combined adapter. Used internally where a single object satisfies both + * roles (today's HostedAdapter, the community reconciler, the realtime + * publishing wrapper). Phases 5+ replace consumers of this with two + * injected interfaces. */ +export type StorageAdapter = SpaceAuthority & RecordHost; + +export interface AdapterContext { + db: Database; +} diff --git a/packages/contrail-base/src/spaces/uri.ts b/packages/contrail-base/src/spaces/uri.ts new file mode 100644 index 0000000..e73ec21 --- /dev/null +++ b/packages/contrail-base/src/spaces/uri.ts @@ -0,0 +1,37 @@ +/** Centralized space URI construction / parsing. + * + * Permissioned spaces are addressed by (ownerDid, type, key) and use the + * `ats://` scheme — distinct from atproto record URIs (`at://`) so the two + * can't be confused at any layer (logs, params, dispatch). Tracks the rough + * spec at https://dholms.leaflet.pub/3mhj6bcqats2o. + * + * Record URIs inside a space are minted by authorDid for index purposes + * (`at://<authorDid>/<collection>/<rkey>`); the spec is explicitly undecided + * about authority (user vs space owner), so we don't expose those as a + * canonical record address — they're storage-internal. */ + +export interface SpaceUriParts { + ownerDid: string; + type: string; + key: string; +} + +/** Build a space URI from its three addressing components. */ +export function buildSpaceUri(parts: SpaceUriParts): string { + return `ats://${parts.ownerDid}/${parts.type}/${parts.key}`; +} + +/** Parse a space URI into its components, or null if malformed. */ +export function parseSpaceUri(uri: string): SpaceUriParts | null { + if (!uri.startsWith("ats://")) return null; + const rest = uri.slice("ats://".length); + const [ownerDid, type, key, ...extra] = rest.split("/"); + if (!ownerDid || !type || !key || extra.length > 0) return null; + return { ownerDid, type, key }; +} + +/** Build a record URI under a given author. Used only as a secondary index key + * inside storage — not a canonical address for permissioned records. */ +export function buildRecordUri(authorDid: string, collection: string, rkey: string): string { + return `at://${authorDid}/${collection}/${rkey}`; +} diff --git a/packages/contrail-base/src/types.ts b/packages/contrail-base/src/types.ts new file mode 100644 index 0000000..3b9dc48 --- /dev/null +++ b/packages/contrail-base/src/types.ts @@ -0,0 +1,513 @@ +import type { SqlDialect } from "./dialect"; + +// Database interface — D1 implements this natively +export interface Database { + prepare(sql: string): Statement; + batch(stmts: Statement[]): Promise<any[]>; + dialect?: SqlDialect; +} + +export interface Statement { + bind(...values: any[]): Statement; + run(): Promise<any>; + all<T = any>(): Promise<{ results: T[] }>; + first<T = any>(): Promise<T | null>; +} + +// Config types + +export interface QueryableField { + type?: "range"; +} + +export interface RelationConfig { + /** Short name of the child collection (a key in `collections`). */ + collection: string; + field?: string; + match?: "uri" | "did"; + groupBy?: string; + /** Enable materialized count columns on the parent. Defaults to true. */ + count?: boolean; + /** Count distinct values of a field (e.g. "did" for unique users) instead of total records. */ + countDistinct?: string; + /** Pre-resolved group mappings: shortName → full token (e.g. { going: "community.lexicon.calendar.rsvp#going" }). Auto-computed from groupBy if omitted. */ + groups?: Record<string, string>; +} + +/** A forward reference: this collection's records point at another collection. */ +export interface ReferenceConfig { + /** Short name of the target collection. */ + collection: string; + /** Field on this collection's records containing the target URI. */ + field: string; +} + +export type CustomQueryHandler = ( + db: Database, + params: URLSearchParams, + config: ContrailConfig +) => Promise<Response>; + +export interface RecordSource { + joins?: string; + conditions?: string[]; + params?: (string | number)[]; +} + +export type PipelineQueryHandler = ( + db: Database, + params: URLSearchParams, + config: ContrailConfig +) => Promise<RecordSource>; + +export interface FeedConfig { + /** Short name of the follow collection. */ + follow: string; + /** Short names of target collections to fan out to. */ + targets: string[]; + /** Max feed items per user (default: 200). Oldest items are pruned after backfill. */ + maxItems?: number; +} + +export const DEFAULT_FEED_MAX_ITEMS = 200; + +export type CollectionMethod = "listRecords" | "getRecord"; +export const DEFAULT_COLLECTION_METHODS: CollectionMethod[] = [ + "listRecords", + "getRecord", +]; + +export interface CollectionConfig { + /** Full NSID of the record type this collection indexes. */ + collection: string; + /** Include this collection in Jetstream ingest / discovery (default true). + * Set false for dependent collections (auto-fetched on demand). */ + discover?: boolean; + queryable?: Record<string, QueryableField>; + relations?: Record<string, RelationConfig>; + /** Forward references: fields on this collection's records that point at another collection. */ + references?: Record<string, ReferenceConfig>; + queries?: Record<string, CustomQueryHandler>; + pipelineQueries?: Record<string, PipelineQueryHandler>; + /** FTS5 search fields. Provide an array of field names to enable full-text search. Omit or set to false to disable. */ + searchable?: string[] | false; + /** XRPC methods to emit. Defaults to ['listRecords', 'getRecord']. */ + methods?: CollectionMethod[]; + /** When spaces are enabled globally, emit a parallel spaces_records_<short> table + * so this collection can also live inside spaces. Defaults to true. */ + allowInSpaces?: boolean; +} + +export interface ProfileConfig { + /** Full NSID of the profile record type. */ + collection: string; + /** Short name used for table/endpoint naming. Defaults to the NSID's last segment. */ + shortName?: string; + rkey?: string; // defaults to "self" +} + +export const DEFAULT_PROFILES: ProfileConfig[] = [ + { collection: "app.bsky.actor.profile", shortName: "profile" }, +]; + +/** Normalize a profiles config entry (string or object) into ProfileConfig. */ +export function normalizeProfileConfig( + p: string | ProfileConfig +): ProfileConfig { + if (typeof p === "string") { + return { collection: p, shortName: deriveShortName(p) }; + } + return { ...p, shortName: p.shortName ?? deriveShortName(p.collection) }; +} + +/** Last NSID segment, used as fallback short name. */ +export function deriveShortName(nsid: string): string { + const parts = nsid.split("."); + return parts[parts.length - 1] ?? nsid; +} + +export const DEFAULT_JETSTREAMS = [ + "wss://jetstream1.us-east.bsky.network", +]; + +export const DEFAULT_RELAYS = [ + "https://relay1.us-east.bsky.network" +]; + +export interface Logger { + log(...args: any[]): void; + warn(...args: any[]): void; + error(...args: any[]): void; +} + +export interface ContrailConfig { + namespace: string; + /** Collections to index, keyed by short name. Short names become endpoint URL segments + * (`<namespace>.<short>.listRecords`) and table suffixes (`records_<short>`). */ + collections: Record<string, CollectionConfig>; + profiles?: (string | ProfileConfig)[]; + relays?: string[]; + jetstreams?: string[]; + feeds?: Record<string, FeedConfig>; + logger?: Logger; + /** Expose the notifyOfUpdate HTTP endpoint. Off by default. + * Set to `true` for open access, or a string to require `Authorization: Bearer <secret>`. */ + notify?: boolean | string; + /** Permissioned spaces configuration. When set, the service exposes space XRPCs. */ + spaces?: import("./spaces/types").SpacesConfig; + /** Community module configuration. Typed by the community package via + * declaration merging — contrail core only knows it's "something the + * community package consumes." Set when wiring community via + * `createCommunityIntegration({ ... })`. Requires `spaces.authority`. */ + community?: unknown; + /** Realtime module configuration. When set, the service exposes ticket + SSE/WS + * subscribe XRPCs, and wraps the spaces adapter to publish events after writes. */ + realtime?: import("./realtime/types").RealtimeConfig; + /** Labels module configuration. When set, contrail subscribes to the + * configured labelers, indexes their labels into a single `labels` table, + * and hydrates `record.labels` onto `listRecords` / `getRecord` / profile + * responses gated by the caller's `atproto-accept-labelers` header. */ + labels?: import("./labels/types").LabelsConfig; + /** Customize the auto-generated `<namespace>.authFull` lexicon. */ + permissionSet?: PermissionSetConfig; +} + +/** Single entry in an atproto permission-set's `permissions` array. + * See https://atproto.com/guides/permission-sets for the full schema. */ +export type PermissionEntry = + | { type: "permission"; resource: "rpc"; lxm?: string[]; aud?: string; inheritAud?: boolean } + | { type: "permission"; resource: "repo"; collection?: string[] } + | { type: "permission"; resource: "blob"; accept?: string[]; maxSize?: number } + | { type: "permission"; resource: "account"; attr?: string[] } + | { type: "permission"; resource: string; [key: string]: unknown }; + +export interface PermissionSetConfig { + /** Shown on the OAuth consent screen. Defaults to the namespace. */ + title?: string; + /** Shown on the OAuth consent screen. Defaults to a generated description. */ + description?: string; + /** Extra permission entries appended after the auto-generated rpc entry — + * e.g. repo writes for collections your app needs the user to create, or + * blob permissions for uploads. */ + additional?: PermissionEntry[]; +} + +export interface ResolvedRelation { + /** Short name of the child collection. */ + collection: string; + groupBy: string; + groups: Record<string, string>; // shortName → full token value +} + +export interface ResolvedMaps { + queryable: Record<string, Record<string, QueryableField>>; + relations: Record<string, Record<string, ResolvedRelation>>; + /** Reverse map: full record NSID → short name. */ + nsidToShort: Record<string, string>; +} + +/** Config after resolveConfig() — has computed queryable/relation maps attached. */ +export interface ResolvedContrailConfig extends ContrailConfig { + _resolved: ResolvedMaps; +} + +/** + * Resolve config: apply defaults, auto-add profile collections, compute queryable maps. + */ +export function resolveConfig(config: ContrailConfig): ResolvedContrailConfig { + const profiles = (config.profiles ?? DEFAULT_PROFILES).map( + normalizeProfileConfig + ); + const collections = { ...config.collections }; + for (const p of profiles) { + const short = p.shortName!; + if (!collections[short]) { + collections[short] = { collection: p.collection, discover: false }; + } + } + + // Auto-add follow collections from feed configs as dependent collections if they're + // not already listed. Feed config already uses short names so nothing to resolve — + // but if the user forgot to declare the follow collection, we can't auto-add it without + // knowing its NSID. In that case we warn later via validateConfig. + + const base = { + ...config, + collections, + profiles, + jetstreams: config.jetstreams ?? DEFAULT_JETSTREAMS, + relays: config.relays ?? DEFAULT_RELAYS, + logger: config.logger ?? console, + }; + + return { + ...base, + _resolved: _resolveQueryableMaps(base), + }; +} + +function _resolveQueryableMaps(config: ContrailConfig): ResolvedMaps { + const queryable: Record<string, Record<string, QueryableField>> = {}; + const relations: Record<string, Record<string, ResolvedRelation>> = {}; + const nsidToShort: Record<string, string> = {}; + + for (const [short, colConfig] of Object.entries(config.collections)) { + nsidToShort[colConfig.collection] = short; + + if (colConfig.queryable) { + queryable[short] = colConfig.queryable; + } + + if (colConfig.relations) { + for (const [relName, rel] of Object.entries(colConfig.relations)) { + if (!rel.groupBy) continue; + const groups: Record<string, string> = rel.groups ? { ...rel.groups } : {}; + if (Object.keys(groups).length > 0) { + if (!relations[short]) relations[short] = {}; + relations[short][relName] = { + collection: rel.collection, + groupBy: rel.groupBy, + groups, + }; + } + } + } + } + + return { queryable, relations, nsidToShort }; +} + +export function getFeedFollowShortNames(config: ContrailConfig): string[] { + if (!config.feeds) return []; + return [...new Set(Object.values(config.feeds).map((f) => f.follow))]; +} + +/** Alias for getFeedFollowShortNames. */ +export const getFeedFollowCollections = getFeedFollowShortNames; + +// Record types + +export interface RecordRow { + uri: string; + did: string; + collection: string; // full NSID + rkey: string; + cid: string | null; + record: string | null; + time_us: number; + indexed_at: number; + /** Set when the row originates from a per-space table. Used by the + * pipeline/hydration/response layers to route child queries to the same + * space and tag the output. */ + space?: string; +} + +export interface IngestEvent { + uri: string; + did: string; + collection: string; // full NSID + rkey: string; + operation: "create" | "update" | "delete"; + cid: string | null; + record: string | null; + time_us: number; + indexed_at: number; +} + +// Validation + +const SAFE_FIELD_NAME = /^[a-zA-Z0-9_.]+$/; +const SAFE_SHORT_NAME = /^[a-zA-Z][a-zA-Z0-9]*$/; + +export function validateFieldName(field: string): string { + if (!SAFE_FIELD_NAME.test(field)) { + throw new Error(`Invalid field name: ${field}`); + } + return field; +} + +function validateShortName(short: string): void { + if (!SAFE_SHORT_NAME.test(short)) { + throw new Error( + `Invalid collection short name: "${short}". Must be alphanumeric, starting with a letter.` + ); + } +} + +export function validateConfig(config: ContrailConfig): void { + const shortNames = new Set<string>(); + for (const [short, colConfig] of Object.entries(config.collections)) { + validateShortName(short); + if (shortNames.has(short)) { + throw new Error(`Duplicate collection short name: ${short}`); + } + shortNames.add(short); + + if (!colConfig.collection) { + throw new Error(`Collection "${short}" is missing required 'collection' field (NSID)`); + } + + for (const field of Object.keys(colConfig.queryable ?? {})) { + validateFieldName(field); + } + for (const [, rel] of Object.entries(colConfig.relations ?? {})) { + if (rel.field) validateFieldName(rel.field); + if (rel.groupBy) validateFieldName(rel.groupBy); + if (rel.countDistinct) validateFieldName(rel.countDistinct); + if (!config.collections[rel.collection]) { + throw new Error( + `Relation in "${short}" references unknown collection short name "${rel.collection}"` + ); + } + } + for (const [, ref] of Object.entries(colConfig.references ?? {})) { + validateFieldName(ref.field); + if (!config.collections[ref.collection]) { + throw new Error( + `Reference in "${short}" references unknown collection short name "${ref.collection}"` + ); + } + } + if (Array.isArray(colConfig.searchable)) { + for (const field of colConfig.searchable) { + validateFieldName(field); + } + } + } + + if (config.feeds) { + for (const [feedName, feed] of Object.entries(config.feeds)) { + if (!config.collections[feed.follow]) { + throw new Error( + `Feed "${feedName}" references unknown follow collection "${feed.follow}"` + ); + } + for (const target of feed.targets) { + if (!config.collections[target]) { + throw new Error( + `Feed "${feedName}" references unknown target collection "${target}"` + ); + } + } + } + } + + if (config.community && !config.spaces?.authority) { + throw new Error( + "Invalid config: `community` requires `spaces.authority`. Community-owned spaces reuse the spaces storage adapter." + ); + } +} + +// Helpers + +export function getNestedValue(obj: any, path: string): any { + let current = obj; + for (const key of path.split(".")) { + if (current == null) return undefined; + current = current[key]; + } + return current; +} + +const DEFAULT_RELATION_FIELD = "subject.uri"; + +export function getRelationField(rel: RelationConfig): string { + return rel.field ?? DEFAULT_RELATION_FIELD; +} + +/** Sanitize a short name for use in SQL identifiers (already-validated; kept for paranoia). */ +function sanitizeIdentifier(name: string): string { + return name.replace(/[^a-zA-Z0-9]/g, "_"); +} + +/** Total-count column name for a relation targeting the given short name. */ +export function countColumnName(childShortName: string): string { + return "count_" + sanitizeIdentifier(childShortName); +} + +/** Grouped-count column name: `count_<child-short>_<groupKey>`. */ +export function groupedCountColumnName( + childShortName: string, + groupKey: string +): string { + return `count_${sanitizeIdentifier(childShortName)}_${sanitizeIdentifier(groupKey)}`; +} + +/** Table name for a collection's records. */ +export function recordsTableName(shortName: string): string { + return "records_" + sanitizeIdentifier(shortName); +} + +/** Table name for a collection's records inside spaces. */ +export function spacesRecordsTableName(shortName: string): string { + return "spaces_records_" + sanitizeIdentifier(shortName); +} + +/** All collection short names. */ +export function getCollectionShortNames(config: ContrailConfig): string[] { + return Object.keys(config.collections); +} + +/** Alias: collection short names (same as getCollectionShortNames). */ +export const getCollectionNames = getCollectionShortNames; + +/** All indexed record NSIDs (what Jetstream filters on). */ +export function getCollectionNsids(config: ContrailConfig): string[] { + return Object.values(config.collections).map((c) => c.collection); +} + +export function getDependentShortNames(config: ContrailConfig): string[] { + return Object.entries(config.collections) + .filter(([, c]) => c.discover === false) + .map(([name]) => name); +} + +export function getDiscoverableShortNames(config: ContrailConfig): string[] { + return Object.entries(config.collections) + .filter(([, c]) => c.discover !== false) + .map(([name]) => name); +} + +/** Aliases for readability elsewhere. These return short names (new semantic). */ +export const getDependentCollections = getDependentShortNames; +export const getDiscoverableCollections = getDiscoverableShortNames; + +/** Short names of collections the user declared with `discover !== false`, mapped to NSIDs. */ +export function getDiscoverableNsids(config: ContrailConfig): string[] { + return Object.values(config.collections) + .filter((c) => c.discover !== false) + .map((c) => c.collection); +} + +export function getDependentNsids(config: ContrailConfig): string[] { + return Object.values(config.collections) + .filter((c) => c.discover === false) + .map((c) => c.collection); +} + +/** Short name for a record NSID, if known. */ +export function shortNameForNsid( + config: ContrailConfig, + nsid: string +): string | undefined { + const resolved = (config as ResolvedContrailConfig)._resolved; + if (resolved?.nsidToShort) return resolved.nsidToShort[nsid]; + for (const [short, c] of Object.entries(config.collections)) { + if (c.collection === nsid) return short; + } + return undefined; +} + +/** Full NSID for a collection short name. */ +export function nsidForShortName( + config: ContrailConfig, + short: string +): string | undefined { + return config.collections[short]?.collection; +} + +/** The methods a collection should expose via XRPC. */ +export function getCollectionMethods(cfg: CollectionConfig): CollectionMethod[] { + return cfg.methods ?? DEFAULT_COLLECTION_METHODS; +} diff --git a/packages/contrail-base/tsconfig.build.json b/packages/contrail-base/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-base/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-base/tsconfig.json b/packages/contrail-base/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-base/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-base/tsup.config.ts b/packages/contrail-base/tsup.config.ts new file mode 100644 index 0000000..3ad3082 --- /dev/null +++ b/packages/contrail-base/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts", "src/adapters/sqlite.ts", "src/adapters/postgres.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["pg", "node:sqlite"], +}); diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index 43fe2e1..c6abcb2 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -41,6 +41,7 @@ "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", "@atmo-dev/contrail": "workspace:*", + "@atmo-dev/contrail-base": "workspace:*", "hono": "^4.12.8" }, "devDependencies": { diff --git a/packages/contrail-community/vitest.config.ts b/packages/contrail-community/vitest.config.ts index 56720e0..a440f50 100644 --- a/packages/contrail-community/vitest.config.ts +++ b/packages/contrail-community/vitest.config.ts @@ -2,13 +2,16 @@ import { defineConfig } from "vitest/config"; import path from "node:path"; const contrailSrc = path.resolve(__dirname, "../contrail/src"); +const baseSrc = path.resolve(__dirname, "../contrail-base/src"); -// Alias `@atmo-dev/contrail` and its subpaths to the source so tests don't -// run through the built dist. Mirrors the in-tree-source-resolution pattern -// the contrail package's own tests use (they import via ../src/...). +// Resolve workspace-internal imports to source so tests don't run through +// dists (where tsup mangles `node:sqlite` → `sqlite`). export default defineConfig({ resolve: { alias: { + "@atmo-dev/contrail-base/sqlite": path.join(baseSrc, "adapters/sqlite.ts"), + "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), + "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), "@atmo-dev/contrail/sqlite": path.join(contrailSrc, "adapters/sqlite.ts"), "@atmo-dev/contrail/postgres": path.join(contrailSrc, "adapters/postgres.ts"), "@atmo-dev/contrail": path.join(contrailSrc, "index.ts"), diff --git a/packages/contrail/package.json b/packages/contrail/package.json index a64484d..8409809 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,6 +73,7 @@ "@atcute/jetstream": "^1.0.2", "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-base": "workspace:*", "cac": "^7.0.0", "hono": "^4.12.8", "jiti": "^2.4.0" diff --git a/packages/contrail/src/adapters/postgres.ts b/packages/contrail/src/adapters/postgres.ts index ae75c9f..6ab2902 100644 --- a/packages/contrail/src/adapters/postgres.ts +++ b/packages/contrail/src/adapters/postgres.ts @@ -1,93 +1 @@ -import pg from "pg"; -import type { Database, Statement } from "../core/types"; -import { postgresDialect } from "../core/dialect"; - -/** Internal interface for statements that can run on a specific client */ -interface PgStatement extends Statement { - /** Execute on a specific client (used by batch for transaction isolation) */ - _runOn(client: pg.PoolClient): Promise<any>; -} - -/** Column names known to be BIGINT — PostgreSQL returns these as strings */ -const BIGINT_COLUMNS = new Set(["time_us", "indexed_at", "resolved_at"]); - -function normalizeRow(row: any): any { - if (!row) return row; - if (typeof row.record === "object" && row.record !== null) { - row.record = JSON.stringify(row.record); - } - for (const col of BIGINT_COLUMNS) { - if (typeof row[col] === "string") row[col] = Number(row[col]); - } - return row; -} - -export function createPostgresDatabase(pool: pg.Pool): Database { - function rewritePlaceholders(sql: string): string { - let idx = 0; - let inString = false; - let result = ""; - for (let i = 0; i < sql.length; i++) { - const ch = sql[i]; - if (ch === "'" && sql[i - 1] !== "\\") { - inString = !inString; - result += ch; - } else if (ch === "?" && !inString) { - result += `$${++idx}`; - } else { - result += ch; - } - } - return result; - } - - function wrapStatement(sql: string, boundValues: any[] = []): PgStatement { - const pgSql = rewritePlaceholders(sql); - - return { - bind(...values: any[]): PgStatement { - return wrapStatement(sql, values); - }, - async run() { - const result = await pool.query(pgSql, boundValues); - return { changes: result.rowCount }; - }, - async _runOn(client: pg.PoolClient) { - const result = await client.query(pgSql, boundValues); - return { changes: result.rowCount }; - }, - async all<T>() { - const result = await pool.query(pgSql, boundValues); - return { results: result.rows.map(normalizeRow) as T[] }; - }, - async first<T>() { - const result = await pool.query(pgSql, boundValues); - return result.rows[0] ? (normalizeRow(result.rows[0]) as T) : null; - }, - }; - } - - return { - prepare(sql: string): Statement { - return wrapStatement(sql); - }, - async batch(stmts: Statement[]): Promise<any[]> { - const client = await pool.connect(); - try { - await client.query("BEGIN"); - const results: any[] = []; - for (const stmt of stmts) { - results.push(await (stmt as PgStatement)._runOn(client)); - } - await client.query("COMMIT"); - return results; - } catch (e) { - await client.query("ROLLBACK"); - throw e; - } finally { - client.release(); - } - }, - dialect: postgresDialect, - }; -} +export * from "@atmo-dev/contrail-base/postgres"; diff --git a/packages/contrail/src/adapters/sqlite.ts b/packages/contrail/src/adapters/sqlite.ts index 2f46dce..42032b3 100644 --- a/packages/contrail/src/adapters/sqlite.ts +++ b/packages/contrail/src/adapters/sqlite.ts @@ -1,39 +1 @@ -import { DatabaseSync } from "node:sqlite"; -import type { Database, Statement } from "../core/types"; -import { sqliteDialect } from "../core/dialect"; - -export function createSqliteDatabase(path: string): Database { - const raw = new DatabaseSync(path); - raw.exec("PRAGMA journal_mode = WAL"); - - function wrapStatement(sql: string, boundValues: any[] = []): Statement { - return { - bind(...values: any[]): Statement { - return wrapStatement(sql, values); - }, - async run() { - return raw.prepare(sql).run(...boundValues); - }, - async all<T>() { - return { results: raw.prepare(sql).all(...boundValues) as T[] }; - }, - async first<T>() { - return (raw.prepare(sql).get(...boundValues) as T) ?? null; - }, - }; - } - - return { - prepare(sql: string): Statement { - return wrapStatement(sql); - }, - async batch(stmts: Statement[]): Promise<any[]> { - const results: any[] = []; - for (const stmt of stmts) { - results.push(await stmt.run()); - } - return results; - }, - dialect: sqliteDialect, - }; -} +export * from "@atmo-dev/contrail-base/sqlite"; diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index 3d23dc1..713e23e 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -1,3 +1,4 @@ +import type {} from "@atcute/atproto"; import { type Did } from "@atcute/lexicons"; import { isDid, isNsid } from "@atcute/lexicons/syntax"; diff --git a/packages/contrail/src/core/client.ts b/packages/contrail/src/core/client.ts index f37bea3..1129419 100644 --- a/packages/contrail/src/core/client.ts +++ b/packages/contrail/src/core/client.ts @@ -1,194 +1 @@ -import { - CompositeDidDocumentResolver, - PlcDidDocumentResolver, - WebDidDocumentResolver, -} from "@atcute/identity-resolver"; -import { type Did } from "@atcute/lexicons"; -import { Client, simpleFetchHandler } from "@atcute/client"; -import type {} from "@atcute/atproto"; -import type { Database } from "./types"; - -// Slingshot-first PDS resolution with fallback to DID document resolution -const SLINGSHOT_URL = - "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; - -export interface ResolvedIdentity { - did: string; - handle: string | null; - pds: string | null; -} - -/** Reject PDS URLs that point to private/internal addresses or non-HTTPS */ -function validatePdsUrl(url: string): boolean { - try { - const parsed = new URL(url); - if (parsed.protocol !== "https:") return false; - const host = parsed.hostname; - // Block private/internal IP ranges - if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; - if (host.startsWith("10.")) return false; - if (host.startsWith("192.168.")) return false; - if (host.startsWith("169.254.")) return false; - if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; - return true; - } catch { - return false; - } -} - -async function resolveViaSlingshot( - identifier: string -): Promise<ResolvedIdentity | undefined> { - const url = new URL(SLINGSHOT_URL); - url.searchParams.set("identifier", identifier); - - try { - const response = await fetch(url.toString()); - if (!response.ok) return undefined; - const data = (await response.json()) as { - did?: string; - handle?: string; - pds?: string; - }; - if (!data.did && !data.pds) return undefined; - return { - did: data.did ?? identifier, - handle: data.handle ?? null, - pds: data.pds ?? null, - }; - } catch { - return undefined; - } -} - -const didResolver = new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, -}); - -async function getPDSViaDidDoc(did: Did): Promise<string | undefined> { - const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); - return doc.service - ?.find((s) => s.id === "#atproto_pds") - ?.serviceEndpoint.toString(); -} - -/** - * Resolve identity info (did, handle, pds) for a DID or handle. - * Uses slingshot first, falls back to DID doc for PDS. - */ -export async function resolvePDS( - identifier: string -): Promise<ResolvedIdentity | undefined> { - const result = await resolveViaSlingshot(identifier); - if (result?.pds) { - if (!validatePdsUrl(result.pds)) return { ...result, pds: null }; - return result; - } - - // Fall back to DID doc resolution (only works for DIDs, not handles) - if (identifier.startsWith("did:")) { - try { - const pds = await getPDSViaDidDoc(identifier as Did); - if (pds && validatePdsUrl(pds)) { - return { - did: identifier, - handle: result?.handle ?? null, - pds, - }; - } - } catch { - // ignore - } - } - - return result; -} - -// In-memory PDS cache with TTL + size limit, plus in-flight deduplication -const PDS_CACHE_TTL = 60 * 60 * 1000; // 1 hour -const PDS_CACHE_MAX = 10_000; -const pdsCache = new Map<string, { pds: string; at: number }>(); -const pdsInflight = new Map<string, Promise<string | undefined>>(); - -function pdsCacheGet(did: string): string | undefined { - const entry = pdsCache.get(did); - if (!entry) return undefined; - if (Date.now() - entry.at > PDS_CACHE_TTL) { - pdsCache.delete(did); - return undefined; - } - return entry.pds; -} - -function pdsCacheSet(did: string, pds: string): void { - // Evict oldest entries if over limit - if (pdsCache.size >= PDS_CACHE_MAX) { - const first = pdsCache.keys().next().value; - if (first) pdsCache.delete(first); - } - pdsCache.set(did, { pds, at: Date.now() }); -} - -export async function getPDS( - did: Did, - db?: Database -): Promise<string | undefined> { - const mem = pdsCacheGet(did); - if (mem) return mem; - - // Deduplicate concurrent calls for the same DID - const inflight = pdsInflight.get(did); - if (inflight) return inflight; - - const promise = resolvePDSCached(did, db); - pdsInflight.set(did, promise); - try { - return await promise; - } finally { - pdsInflight.delete(did); - } -} - -async function resolvePDSCached( - did: Did, - db?: Database -): Promise<string | undefined> { - if (db) { - const cached = await db - .prepare("SELECT pds FROM identities WHERE did = ? AND pds IS NOT NULL") - .bind(did) - .first<{ pds: string }>(); - if (cached?.pds) { - pdsCacheSet(did, cached.pds); - return cached.pds; - } - } - - const resolved = await resolvePDS(did); - if (!resolved?.pds) return undefined; - - pdsCacheSet(did, resolved.pds); - - // Persist to DB for future runs - if (db) { - await db - .prepare( - "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET pds = excluded.pds, handle = COALESCE(excluded.handle, identities.handle), resolved_at = excluded.resolved_at" - ) - .bind(did, resolved.handle, resolved.pds, Date.now()) - .run(); - } - - return resolved.pds; -} - -export async function getClient(did: Did, db?: Database): Promise<Client> { - const pds = await getPDS(did, db); - if (!pds) throw new Error(`PDS not found for ${did}`); - return new Client({ - handler: simpleFetchHandler({ service: pds }), - }); -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/community-integration.ts b/packages/contrail/src/core/community-integration.ts index d91b3a4..1129419 100644 --- a/packages/contrail/src/core/community-integration.ts +++ b/packages/contrail/src/core/community-integration.ts @@ -1,54 +1 @@ -/** Pluggable integration surface for the community module. - * - * Phase 6 extracted community to its own package (`@atmo-dev/contrail-community`). - * The contrail core package never imports from it — couplings only flow - * through these interfaces. The community package's - * `createCommunityIntegration({ ... })` returns a {@link CommunityIntegration}, - * which the consumer hands to `createApp` via `options.community`. - * - * Two layers: - * - {@link CommunityProbe}: minimal "is this a community DID" / "what - * spaces does this caller reach" surface used by realtime + collection - * routes for community-aware dispatch. - * - {@link CommunityIntegration}: the umbrella bundle — probe, whoami - * extension, invite handler, plus route + schema wiring that the - * umbrella router calls during setup. */ - -import type { Hono, MiddlewareHandler } from "hono"; -import type { Database } from "./types"; -import type { CommunityInviteHandler } from "./invite/community-handler"; -import type { WhoamiExtension } from "./spaces/router"; - -/** Narrow interface for the deep callers (realtime/resolve, router/collection) - * that just need to ask "is this a community DID?" or "what spaces does this - * caller reach via community membership?" */ -export interface CommunityProbe { - /** Look up a community row by DID. Returns null for non-community DIDs. - * Callers usually only check truthiness — community-specific fields stay - * inside the community package. */ - getCommunity(did: string): Promise<{ did: string } | null>; - - /** Resolve the set of space URIs reachable by `callerDid` through community - * membership (direct grants + delegations). Used by realtime to expand - * community: topics into the caller's concrete space: topics. */ - resolveReachableSpaces(callerDid: string): Promise<Set<string>>; -} - -/** Umbrella integration the consumer constructs once and hands to createApp. - * contrail core treats this as an opaque bundle — it doesn't introspect - * community state, just calls these methods at the right wiring points. */ -export interface CommunityIntegration { - /** Probe used by realtime + collection cross-cutting concerns. */ - probe: CommunityProbe; - /** Whoami extension that returns `accessLevel` for community-owned spaces. */ - whoamiExtension: WhoamiExtension; - /** Handler for the community-grant path of the unified invite surface. */ - inviteHandler: CommunityInviteHandler; - /** Register `<ns>.community.*` routes onto the Hono app. */ - registerRoutes( - app: Hono, - options?: { authMiddleware?: MiddlewareHandler } - ): void; - /** Apply community schema (DDL) to the database. Called by initSchema. */ - applySchema(db: Database): Promise<void>; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/dialect.ts b/packages/contrail/src/core/dialect.ts index e3bb790..1129419 100644 --- a/packages/contrail/src/core/dialect.ts +++ b/packages/contrail/src/core/dialect.ts @@ -1,127 +1 @@ -/** Get the dialect from a Database, defaulting to SQLite (for D1 compatibility) */ -export function getDialect(db: { dialect?: SqlDialect }): SqlDialect { - return db.dialect ?? sqliteDialect; -} - -const SAFE_FIELD = /^[a-zA-Z0-9_.]+$/; - -function assertSafeField(field: string): void { - if (!SAFE_FIELD.test(field)) { - throw new Error(`Invalid field name: ${field}`); - } -} - -export interface SqlDialect { - /** json_extract(col, '$.field') or col->>'field' */ - jsonExtract(column: string, field: string): string; - - /** Convert INSERT INTO to ignore-duplicates form. - * SQLite: INSERT INTO → INSERT OR IGNORE INTO - * PG: appends ON CONFLICT DO NOTHING - * Accepts full SQL starting with "INSERT INTO" (works with both VALUES and SELECT). */ - insertOrIgnore(sql: string): string; - - /** Column type for the record column: TEXT (SQLite) or JSONB (PostgreSQL) */ - readonly recordColumnType: string; - - /** FTS strategy: 'virtual-table' (SQLite FTS5) or 'generated-column' (PG tsvector) */ - readonly ftsStrategy: "virtual-table" | "generated-column"; - - /** INTEGER type name — same on both, but PostgreSQL may want BIGINT for time_us */ - readonly integerType: string; - - /** BIGINT type name for timestamps */ - readonly bigintType: string; - - /** Wrap an expression for use in CREATE INDEX — PostgreSQL requires parens around expressions */ - indexExpression(expr: string): string; -} - -export const sqliteDialect: SqlDialect = { - jsonExtract(column: string, field: string): string { - assertSafeField(field); - return `json_extract(${column}, '$.${field}')`; - }, - - insertOrIgnore(sql: string): string { - return sql.replace(/^INSERT INTO/, "INSERT OR IGNORE INTO"); - }, - - recordColumnType: "TEXT", - ftsStrategy: "virtual-table", - integerType: "INTEGER", - bigintType: "INTEGER", - - indexExpression(expr: string): string { - return expr; - }, -}; - -export const postgresDialect: SqlDialect = { - jsonExtract(column: string, field: string): string { - assertSafeField(field); - const parts = field.split("."); - if (parts.length === 1) { - return `${column}->>'${parts[0]}'`; - } - // a.b.c → col->'a'->'b'->>'c' - const intermediate = parts.slice(0, -1).map((p) => `->'${p}'`).join(""); - return `${column}${intermediate}->>'${parts[parts.length - 1]}'`; - }, - - insertOrIgnore(sql: string): string { - return `${sql} ON CONFLICT DO NOTHING`; - }, - - recordColumnType: "JSONB", - ftsStrategy: "generated-column", - integerType: "INTEGER", - bigintType: "BIGINT", - - indexExpression(expr: string): string { - return `(${expr})`; - }, -}; - -/** Generate FTS schema statements based on dialect */ -export function buildFtsSchema( - dialect: SqlDialect, - recordsTable: string, - fields: string[] -): string[] { - if (dialect.ftsStrategy === "virtual-table") { - const ftsTable = recordsTable.replace("records_", "fts_"); - return [ - `CREATE VIRTUAL TABLE IF NOT EXISTS ${ftsTable} USING fts5(uri UNINDEXED, content)` - ]; - } else { - const concatExpr = fields - .map((f) => `COALESCE(${dialect.jsonExtract("record", f)}, '')`) - .join(" || ' ' || "); - return [ - `ALTER TABLE ${recordsTable} ADD COLUMN IF NOT EXISTS search_vector TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', ${concatExpr})) STORED`, - `CREATE INDEX IF NOT EXISTS idx_${recordsTable}_search ON ${recordsTable} USING GIN (search_vector)`, - ]; - } -} - -/** Generate FTS query clause based on dialect */ -export function ftsQueryClause( - dialect: SqlDialect, - recordsTable: string -): { join: string; condition: string; orderExpr: string } { - if (dialect.ftsStrategy === "virtual-table") { - const ftsTable = recordsTable.replace("records_", "fts_"); - return { - join: `JOIN ${ftsTable} fts ON fts.uri = r.uri`, - condition: "fts.content MATCH ?", - orderExpr: "fts.rank", - }; - } else { - return { - join: "", - condition: "r.search_vector @@ plainto_tsquery('english', ?)", - orderExpr: "ts_rank(r.search_vector, plainto_tsquery('english', ?))", - }; - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/identity.ts b/packages/contrail/src/core/identity.ts index 05040fd..1129419 100644 --- a/packages/contrail/src/core/identity.ts +++ b/packages/contrail/src/core/identity.ts @@ -1,159 +1 @@ -import type { Did } from "@atcute/lexicons"; -import type { Database, Logger } from "./types"; -import { isDid, isHandle } from "@atcute/lexicons/syntax"; -import { resolvePDS } from "./client"; - -const STALE_MS = 24 * 60 * 60 * 1000; // 24 hours - -export interface Identity { - did: string; - handle: string | null; - pds: string | null; - resolved_at: number; -} - -async function saveIdentity(db: Database, identity: Identity): Promise<void> { - await db - .prepare( - "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET handle = excluded.handle, pds = excluded.pds, resolved_at = excluded.resolved_at" - ) - .bind(identity.did, identity.handle, identity.pds, identity.resolved_at) - .run(); -} - -function isStale(resolvedAt: number): boolean { - return Date.now() - resolvedAt >= STALE_MS; -} - -async function fetchAndSave( - db: Database, - identifier: string, - cached?: Identity | null -): Promise<Identity> { - const resolved = await resolvePDS(identifier); - const identity: Identity = { - did: resolved?.did ?? identifier, - handle: resolved?.handle ?? cached?.handle ?? null, - pds: resolved?.pds ?? cached?.pds ?? null, - resolved_at: Date.now(), - }; - await saveIdentity(db, identity); - return identity; -} - -export async function resolveIdentity( - db: Database, - did: Did -): Promise<Identity> { - const cached = await db - .prepare("SELECT did, handle, pds, resolved_at FROM identities WHERE did = ?") - .bind(did) - .first<Identity>(); - - if (cached && !isStale(cached.resolved_at)) return cached; - - return fetchAndSave(db, did, cached); -} - -export async function resolveIdentities( - db: Database, - dids: string[] -): Promise<Map<string, Identity>> { - const map = new Map<string, Identity>(); - if (dids.length === 0) return map; - - // Batch lookup from DB - const BATCH = 50; - for (let i = 0; i < dids.length; i += BATCH) { - const chunk = dids.slice(i, i + BATCH); - const placeholders = chunk.map(() => "?").join(","); - const rows = await db - .prepare(`SELECT did, handle, pds, resolved_at FROM identities WHERE did IN (${placeholders})`) - .bind(...chunk) - .all<Identity>(); - for (const row of rows.results ?? []) { - map.set(row.did, row); - } - } - - // Resolve missing via slingshot directly (no redundant DB lookup) - for (const did of dids) { - if (map.has(did) || !isDid(did)) continue; - try { - const identity = await fetchAndSave(db, did); - map.set(did, identity); - } catch { - // Silently skip unresolvable identities - } - } - - return map; -} - -export async function resolveActor( - db: Database, - actor: string -): Promise<string | null> { - if (isDid(actor)) return actor; - if (!isHandle(actor)) return null; - - // Look up handle in identities table - const cached = await db - .prepare("SELECT did, resolved_at FROM identities WHERE handle = ?") - .bind(actor) - .first<{ did: string; resolved_at: number }>(); - - if (cached && !isStale(cached.resolved_at)) return cached.did; - - // Resolve via slingshot - const resolved = await resolvePDS(actor); - if (!resolved?.did || !isDid(resolved.did)) return null; - - await saveIdentity(db, { - did: resolved.did, - handle: resolved.handle ?? actor, - pds: resolved.pds ?? null, - resolved_at: Date.now(), - }); - - return resolved.did; -} - -export async function refreshStaleIdentities( - db: Database, - dids: string[] -): Promise<void> { - if (dids.length === 0) return; - - const unique = [...new Set(dids)].filter(isDid); - if (unique.length === 0) return; - - const staleThreshold = Date.now() - STALE_MS; - const toRefresh: string[] = []; - - const BATCH = 50; - for (let i = 0; i < unique.length; i += BATCH) { - const chunk = unique.slice(i, i + BATCH); - const placeholders = chunk.map(() => "?").join(","); - const rows = await db - .prepare(`SELECT did, resolved_at FROM identities WHERE did IN (${placeholders})`) - .bind(...chunk) - .all<{ did: string; resolved_at: number }>(); - - const found = new Map((rows.results ?? []).map((r) => [r.did, r.resolved_at])); - for (const did of chunk) { - const resolvedAt = found.get(did); - if (resolvedAt === undefined || resolvedAt < staleThreshold) { - toRefresh.push(did); - } - } - } - - for (const did of toRefresh) { - try { - await fetchAndSave(db, did); - } catch { - // Silently skip unresolvable identities - } - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/invite/community-handler.ts b/packages/contrail/src/core/invite/community-handler.ts index 1043525..1129419 100644 --- a/packages/contrail/src/core/invite/community-handler.ts +++ b/packages/contrail/src/core/invite/community-handler.ts @@ -1,67 +1 @@ -/** Pluggable handler for community-grant invites within the unified invite - * surface. The invite router calls into this when the target space is - * community-owned, or "tries" it on the redeem / revoke-without-spaceUri - * paths. Community module provides the impl; invite/router doesn't import - * from community at all. - * - * Each method returns a `HandlerResponse`: a `{status, body}` envelope that - * the router relays as JSON, or `null` (only on the "try" methods) meaning - * "not applicable, fall through to the user-owned path." */ - -export type HandlerResponse = { - status: number; - body: Record<string, unknown>; -}; - -export interface CommunityInviteHandler { - /** True iff this space is owned by a community (vs. a regular user DID). - * Used by the invite router to choose the dispatch path on - * create / list / revoke-with-spaceUri. */ - isCommunityOwned(spaceUri: string): Promise<boolean>; - - /** Create a community-grant invite. Caller is validated upstream for - * having a JWT; this method handles the access-level checks. */ - create(input: { - spaceUri: string; - callerDid: string; - /** Raw caller-supplied access level — implementation validates. */ - accessLevel?: string; - /** Caller-supplied `kind` field — community spaces don't accept this; the - * handler returns an InvalidRequest if set. */ - kind?: string; - expiresAt: number | null; - maxUses: number | null; - note: string | null; - }): Promise<HandlerResponse>; - - /** List invites for a community-owned space. */ - list(input: { - spaceUri: string; - callerDid: string; - includeRevoked: boolean; - }): Promise<HandlerResponse>; - - /** Revoke a known community-owned invite (caller already passed spaceUri - * and the router classified it as community-owned). */ - revoke(input: { - spaceUri: string; - tokenHash: string; - callerDid: string; - }): Promise<HandlerResponse>; - - /** Revoke without a spaceUri — try to find the invite in the community - * table; return null if not a community invite (router falls through). */ - tryRevokeByToken(input: { - tokenHash: string; - callerDid: string; - }): Promise<HandlerResponse | null>; - - /** Try to redeem a token as a community invite. Returns null if the token - * is not a community invite, in which case the router falls through to - * the user-owned redeem path. */ - tryRedeem(input: { - tokenHash: string; - callerDid: string; - now: number; - }): Promise<HandlerResponse | null>; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/invite/token.ts b/packages/contrail/src/core/invite/token.ts index 76f4880..1129419 100644 --- a/packages/contrail/src/core/invite/token.ts +++ b/packages/contrail/src/core/invite/token.ts @@ -1,43 +1 @@ -const B64U_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - -function bytesToB64Url(bytes: Uint8Array): string { - let out = ""; - for (let i = 0; i < bytes.length; i += 3) { - const b0 = bytes[i]; - const b1 = bytes[i + 1] ?? 0; - const b2 = bytes[i + 2] ?? 0; - out += B64U_ALPHABET[b0 >> 2]; - out += B64U_ALPHABET[((b0 & 3) << 4) | (b1 >> 4)]; - if (i + 1 < bytes.length) out += B64U_ALPHABET[((b1 & 15) << 2) | (b2 >> 6)]; - if (i + 2 < bytes.length) out += B64U_ALPHABET[b2 & 63]; - } - return out; -} - -function bytesToHex(bytes: Uint8Array): string { - let out = ""; - for (let i = 0; i < bytes.length; i++) out += bytes[i].toString(16).padStart(2, "0"); - return out; -} - -/** Generate a fresh invite token (cryptographically random, 32 bytes base64url-encoded). */ -export function generateInviteToken(): string { - const bytes = new Uint8Array(32); - crypto.getRandomValues(bytes); - return bytesToB64Url(bytes); -} - -/** SHA-256 hash of a token, hex-encoded. Used as the PK in storage so raw tokens are never persisted. */ -export async function hashInviteToken(token: string): Promise<string> { - const encoded = new TextEncoder().encode(token); - const digest = await crypto.subtle.digest("SHA-256", encoded); - return bytesToHex(new Uint8Array(digest)); -} - -/** Convenience: generate a token and return both the raw form (returned to - * the creator once) and its hash (persisted as the stable ID). */ -export async function mintInviteToken(): Promise<{ token: string; tokenHash: string }> { - const token = generateInviteToken(); - const tokenHash = await hashInviteToken(token); - return { token, tokenHash }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/labels/types.ts b/packages/contrail/src/core/labels/types.ts index f8bc2e6..1129419 100644 --- a/packages/contrail/src/core/labels/types.ts +++ b/packages/contrail/src/core/labels/types.ts @@ -1,59 +1 @@ -import type { Database } from "../types"; - -/** A labeler the operator wants contrail to track. */ -export interface LabelerSource { - /** Labeler DID — `did:plc:...` or `did:web:...`. */ - did: string; - /** Override the service endpoint resolution. Otherwise resolved from the - * DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ - endpoint?: string; - /** Backfill from `cursor=0` on first sight. Defaults to true. Set false - * for "start from now" — useful for very chatty labelers. */ - backfill?: boolean; -} - -export interface LabelsConfig { - /** Labelers to subscribe to and index. */ - sources: LabelerSource[]; - /** DIDs honored when the caller sends no `atproto-accept-labelers` / - * `?labelers=`. Defaults to every entry in `sources`. Set `[]` for - * opt-in-only — clients see no labels unless they ask. */ - defaults?: string[]; - /** Per-request cap. Default: 20 (matches Bluesky). */ - maxPerRequest?: number; -} - -export const DEFAULT_LABELS_MAX_PER_REQUEST = 20; - -/** A single label as stored. Matches `com.atproto.label.defs#label`. */ -export interface LabelRow { - /** Issuing labeler DID. */ - src: string; - /** Subject — at-URI for record labels, plain DID for account labels. */ - uri: string; - /** Label value — kebab-case, ≤128 bytes per spec. */ - val: string; - /** Optional CID pin to a specific record version. */ - cid: string | null; - /** When true, retracts a previously-emitted label for the same (src, uri, val). */ - neg: boolean; - /** Expiry, unix seconds. Past this, hydration drops the row. */ - exp: number | null; - /** Creation timestamp, unix seconds — what we collapse on. */ - cts: number; - /** Raw signature bytes. Stored when present so we can re-emit later; - * not verified in v1. */ - sig: Uint8Array | null; -} - -/** Per-labeler state row — endpoint cache and last-seen seq cursor. */ -export interface LabelerCursorRow { - did: string; - cursor: number; - endpoint: string | null; - resolved_at: number | null; -} - -export interface AdapterContext { - db: Database; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/durable-object.ts b/packages/contrail/src/core/realtime/durable-object.ts index 966ab53..1129419 100644 --- a/packages/contrail/src/core/realtime/durable-object.ts +++ b/packages/contrail/src/core/realtime/durable-object.ts @@ -1,567 +1 @@ -/** Durable Object backend for realtime PubSub. - * - * Two pieces live here: - * 1. `RealtimePubSubDO` — the DO class. Ship it from your Worker via - * `export { RealtimePubSubDO } from "@atmo-dev/contrail";` and bind it in - * your `wrangler.toml`. One DO = one topic; addressed by name. - * 2. `DurableObjectPubSub` — client-side adapter implementing the PubSub - * interface against a DO namespace binding. - * - * Wire format between Worker and DO (internal, not a stable public contract): - * POST /publish — body = RealtimeEvent JSON - * GET /subscribe — server-sent events stream, optionally with - * `Upgrade: websocket` for WS connections. - * Auth/ACL is already checked at the Worker edge; - * the DO trusts anything that reaches it. */ - -import type { PubSub, RealtimeEvent } from "./types"; -import { translateForQuery, type TranslatedEnvelope } from "./query-filter"; -type TranslatedEvent = TranslatedEnvelope; - -/** Query spec attached to a WS subscriber, used to filter events before - * delivery. Shape matches what the Worker's `watchRecords` handler builds; - * forwarded to the DO via trusted internal headers on the WS upgrade. */ -export interface SubscriberQuerySpec { - /** NSID of the primary collection the client is watching. */ - collection: string; - /** Space URI this subscription is scoped to. Events outside are dropped. */ - spaceUri: string; - /** Hydrated relations. Keyed by relName — value is the child collection - * NSID and the field on the child record that references the parent. */ - hydrate?: Record<string, { childCollection: string; matchField: string }>; -} - -// ---- Minimal structural typings so we don't depend on @cloudflare/workers-types -// at the library level. Callers on Workers will have proper types. -// ---------------------------------------------------------------------------- - -export interface DurableObjectId { - toString(): string; -} - -export interface DurableObjectStub { - fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>; -} - -export interface DurableObjectNamespace { - idFromName(name: string): DurableObjectId; - get(id: DurableObjectId): DurableObjectStub; -} - -export interface DurableObjectState { - acceptWebSocket(ws: any, tags?: string[]): void; - getWebSockets(tag?: string): any[]; -} - -// ---------------------------------------------------------------------------- -// Client adapter -// ---------------------------------------------------------------------------- - -export class DurableObjectPubSub implements PubSub { - constructor(private readonly namespace: DurableObjectNamespace) {} - - private stub(topic: string): DurableObjectStub { - return this.namespace.get(this.namespace.idFromName(topic)); - } - - async publish(event: RealtimeEvent): Promise<void> { - const res = await this.stub(event.topic).fetch("https://do/publish", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(event), - }); - if (!res.ok) { - // Consume the body so the edge doesn't hold a dangling response. - await res.text().catch(() => ""); - throw new Error(`DO publish failed (${res.status})`); - } - } - - /** In-Worker server-side subscribe. Browsers should hit the SSE endpoint - * directly; this is the path for an in-process consumer that wants an - * AsyncIterable (tests, bots embedded in the Worker). */ - subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { - const stub = this.stub(topic); - return { - [Symbol.asyncIterator]() { - return pullIterator(stub, signal); - }, - }; - } - - /** Forward an incoming browser WS upgrade (or SSE GET) through to the DO - * that owns this topic, attaching a query-filter spec that the DO will use - * to decide what to deliver. The Worker must verify auth + spec validity - * before calling this — the DO trusts the headers. */ - async forwardSubscribe( - topic: string, - request: Request, - opts: { - did?: string; - querySpec?: SubscriberQuerySpec; - /** Unix ms. DO replays any buffered event with ts > sinceTs before - * going live — closes the snapshot→WS race window on the client. */ - sinceTs?: number; - } = {} - ): Promise<Response> { - const headers = new Headers(request.headers); - if (opts.querySpec) { - headers.set("X-Contrail-Query-Spec", JSON.stringify(opts.querySpec)); - } - const url = new URL("https://do/subscribe"); - if (opts.did) url.searchParams.set("did", opts.did); - if (opts.sinceTs && opts.sinceTs > 0) { - url.searchParams.set("sinceTs", String(opts.sinceTs)); - } - return this.stub(topic).fetch(url.toString(), { - method: "GET", - headers - }); - } -} - -function pullIterator( - stub: DurableObjectStub, - signal?: AbortSignal -): AsyncIterator<RealtimeEvent> { - let reader: ReadableStreamDefaultReader<Uint8Array> | null = null; - let buf = ""; - const decoder = new TextDecoder(); - const ac = new AbortController(); - if (signal) { - if (signal.aborted) ac.abort(); - else signal.addEventListener("abort", () => ac.abort(), { once: true }); - } - - const open = async () => { - const res = await stub.fetch("https://do/subscribe", { - method: "GET", - headers: { accept: "text/event-stream" }, - signal: ac.signal, - }); - if (!res.ok || !res.body) throw new Error(`DO subscribe failed (${res.status})`); - reader = res.body.getReader(); - }; - - return { - async next(): Promise<IteratorResult<RealtimeEvent>> { - if (!reader) await open(); - while (true) { - // Drain buffered frames. - while (true) { - const sep = buf.indexOf("\n\n"); - if (sep < 0) break; - const frame = buf.slice(0, sep); - buf = buf.slice(sep + 2); - let data: string | null = null; - for (const line of frame.split("\n")) { - if (line.startsWith(":")) continue; - if (line.startsWith("data:")) data = line.slice(5).trim(); - } - if (data) { - try { - return { value: JSON.parse(data) as RealtimeEvent, done: false }; - } catch { - /* skip malformed */ - } - } - } - if (ac.signal.aborted) return { value: undefined, done: true }; - const r = await reader!.read(); - if (r.done) return { value: undefined, done: true }; - buf += decoder.decode(r.value, { stream: true }); - } - }, - async return(): Promise<IteratorResult<RealtimeEvent>> { - ac.abort(); - try { - await reader?.cancel(); - } catch { - /* ignore */ - } - return { value: undefined, done: true }; - }, - }; -} - -// ---------------------------------------------------------------------------- -// Durable Object class -// ---------------------------------------------------------------------------- - -/** The Durable Object implementation. Each DO instance owns the fan-out for - * exactly one topic. WebSocket connections are stored via the Hibernation - * API (`state.acceptWebSocket`) so idle rooms cost near-zero. - * - * This class intentionally avoids the `DurableObject` base class so we don't - * have to depend on @cloudflare/workers-types at the library level — users - * wire it up directly in their Worker entry. */ -/** Rolling buffer of recent events, used to close the snapshot→WS race: - * when a new subscriber connects with `?sinceTs=X`, replay any buffered - * event with `event.ts > X` before going live. Bounded by count + age so - * memory stays small. */ -const RECENT_BUFFER_MS = 15_000; -const RECENT_BUFFER_MAX = 500; - -export class RealtimePubSubDO { - private readonly recentEvents: RealtimeEvent[] = []; - - constructor( - protected readonly state: DurableObjectState, - _env?: unknown - ) {} - - private pushRecent(event: RealtimeEvent): void { - this.recentEvents.push(event); - const cutoff = Date.now() - RECENT_BUFFER_MS; - while ( - this.recentEvents.length > RECENT_BUFFER_MAX || - (this.recentEvents.length > 0 && this.recentEvents[0]!.ts < cutoff) - ) { - this.recentEvents.shift(); - } - } - - /** Worker entry delegates `fetch` to this method. */ - async fetch(request: Request): Promise<Response> { - const url = new URL(request.url); - if (request.method === "POST" && url.pathname === "/publish") { - let event: RealtimeEvent; - try { - event = (await request.json()) as RealtimeEvent; - } catch { - return new Response(JSON.stringify({ error: "InvalidRequest" }), { status: 400 }); - } - this.publishEvent(event); - return new Response("{}", { status: 200 }); - } - if (request.method === "GET" && url.pathname === "/subscribe") { - const did = url.searchParams.get("did") ?? undefined; - const sinceTsRaw = url.searchParams.get("sinceTs"); - const sinceTs = sinceTsRaw ? Number(sinceTsRaw) : 0; - // Optional query-filter spec, forwarded by the Worker after it has - // verified the caller's auth + access. Parsed once here; the parsed - // object is serialized into the WS attachment so the DO can filter - // events on publish without re-parsing. - let querySpec: SubscriberQuerySpec | undefined; - const rawSpec = request.headers.get("X-Contrail-Query-Spec"); - if (rawSpec) { - try { - querySpec = JSON.parse(rawSpec) as SubscriberQuerySpec; - } catch { - return new Response( - JSON.stringify({ error: "InvalidRequest", message: "bad X-Contrail-Query-Spec" }), - { status: 400 } - ); - } - } - - if (request.headers.get("Upgrade")?.toLowerCase() === "websocket") { - const Pair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; - if (!Pair) return new Response("websockets require Workers", { status: 426 }); - const pair = new Pair(); - this.acceptWebSocketSubscriber(pair[1], did, querySpec); - if (sinceTs > 0) this.replayRecentTo(pair[1], sinceTs); - return new Response(null, { - status: 101, - // Workers-specific init field - webSocket: pair[0], - } as ResponseInit & { webSocket: unknown }); - } - return this.openSseResponse(did, querySpec, sinceTs); - } - return new Response("not found", { status: 404 }); - } - - /** Fan-out an event to every connected subscriber (WS + SSE). - * Public so tests + advanced callers can skip the HTTP layer. - * - * If a subscriber has attached a `querySpec`, we translate the raw event - * into 0–1 watchRecords-shaped events (record.created, record.deleted, - * hydration.added, hydration.removed) and deliver only those. Otherwise - * the raw event is delivered as-is (topic-firehose behaviour for the - * `realtime.subscribe` endpoint). */ - publishEvent(event: RealtimeEvent): void { - // Buffer first so a subscriber connecting mid-publish (race-window - // replay) can pick up this event too once they provide their sinceTs. - this.pushRecent(event); - - const rawPayload = JSON.stringify(event); - const rawFrame = `event: ${event.kind}\ndata: ${rawPayload}\n\n`; - - for (const ws of this.state.getWebSockets()) { - const attachment = getAttachment(ws); - - if (attachment?.querySpec) { - const translated = translateForQuery(event, attachment); - if (translated) this.writeSubscriberState(ws, attachment, translated); - for (const msg of translated ?? []) { - try { - ws.send(JSON.stringify(msg)); - } catch { - /* ignore */ - } - } - } else { - try { - ws.send(rawPayload); - } catch { - /* ignore */ - } - } - - if ( - event.kind === "member.removed" && - attachment?.did && - event.payload.did === attachment.did - ) { - try { - ws.close(4003, "membership-revoked"); - } catch { - /* ignore */ - } - } - } - - for (const entry of this.sseControllers) { - if (entry.querySpec) { - const translated = translateForQuery(event, entry); - if (translated) this.writeSubscriberStateForSse(entry, translated); - for (const msg of translated ?? []) { - try { - entry.controller.enqueue( - this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) - ); - } catch { - /* drop */ - } - } - } else { - try { - entry.controller.enqueue(this.encoder.encode(rawFrame)); - } catch { - /* drop; cleanup happens on the subscribe-side */ - } - } - - if ( - event.kind === "member.removed" && - entry.did && - event.payload.did === entry.did - ) { - try { - entry.controller.close(); - } catch { - /* ignore */ - } - } - } - } - - /** Register a server-side WebSocket as a subscriber. Wires the DID + - * optional query spec into the hibernation attachment so this DO can - * filter and route events after going to sleep. */ - acceptWebSocketSubscriber( - serverWs: any, - did?: string, - querySpec?: SubscriberQuerySpec - ): void { - this.state.acceptWebSocket(serverWs, did ? [did] : undefined); - if (did || querySpec) { - setAttachment(serverWs, { - did, - querySpec, - parentUris: [], - childToParent: {} - }); - } - } - - /** Open an SSE subscriber; returns the streaming Response. */ - openSseResponse( - did?: string, - querySpec?: SubscriberQuerySpec, - sinceTs = 0 - ): Response { - let entry: SseEntry; - const stream = new ReadableStream<Uint8Array>({ - start: (controller) => { - entry = { - controller, - did, - querySpec, - parentUris: new Set(), - childToParent: new Map() - }; - this.sseControllers.add(entry); - controller.enqueue(this.encoder.encode(`: open\n\n`)); - if (sinceTs > 0) this.replayRecentToSse(entry, sinceTs); - }, - cancel: () => { - this.sseControllers.delete(entry); - }, - }); - return new Response(stream, { - status: 200, - headers: { - "content-type": "text/event-stream", - "cache-control": "no-cache, no-transform", - connection: "keep-alive", - }, - }); - } - - /** Replay buffered events with ts > sinceTs through this subscriber's - * query-spec filter. Called once, synchronously, on WS connect. */ - private replayRecentTo(ws: any, sinceTs: number): void { - const attachment = getAttachment(ws); - for (const event of this.recentEvents) { - if (event.ts <= sinceTs) continue; - if (attachment?.querySpec) { - const translated = translateForQuery(event, attachment); - if (translated) this.writeSubscriberState(ws, attachment, translated); - for (const msg of translated ?? []) { - try { - ws.send(JSON.stringify(msg)); - } catch { - /* ignore */ - } - } - } else { - try { - ws.send(JSON.stringify(event)); - } catch { - /* ignore */ - } - } - } - } - - private replayRecentToSse(entry: SseEntry, sinceTs: number): void { - for (const event of this.recentEvents) { - if (event.ts <= sinceTs) continue; - if (entry.querySpec) { - const translated = translateForQuery(event, entry); - if (translated) this.writeSubscriberStateForSse(entry, translated); - for (const msg of translated ?? []) { - try { - entry.controller.enqueue( - this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) - ); - } catch { - /* drop */ - } - } - } else { - try { - entry.controller.enqueue( - this.encoder.encode(`event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`) - ); - } catch { - /* drop */ - } - } - } - } - - /** Update the persisted WS attachment state after we've decided which - * events to forward. Keeps the parent/child tracking tables warm across - * hibernation. */ - private writeSubscriberState( - ws: any, - attachment: WsAttachment, - translated: TranslatedEvent[] - ): void { - let dirty = false; - for (const msg of translated) { - if (msg.kind === "record.created" && msg.data.record?.uri) { - attachment.parentUris = Array.from( - new Set([...(attachment.parentUris ?? []), msg.data.record.uri]) - ); - dirty = true; - } else if (msg.kind === "record.deleted" && msg.data.uri) { - const before = attachment.parentUris ?? []; - attachment.parentUris = before.filter((u) => u !== msg.data.uri); - if (attachment.parentUris.length !== before.length) dirty = true; - } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { - attachment.childToParent = { - ...(attachment.childToParent ?? {}), - [msg.data.child.rkey]: { - parentUri: msg.data.parentUri, - relName: msg.data.relation - } - }; - dirty = true; - } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { - const next = { ...(attachment.childToParent ?? {}) }; - if (next[msg.data.childRkey]) { - delete next[msg.data.childRkey]; - attachment.childToParent = next; - dirty = true; - } - } - } - if (dirty) setAttachment(ws, attachment); - } - - private writeSubscriberStateForSse( - entry: SseEntry, - translated: TranslatedEvent[] - ): void { - for (const msg of translated) { - if (msg.kind === "record.created" && msg.data.record?.uri) { - entry.parentUris?.add(msg.data.record.uri); - } else if (msg.kind === "record.deleted" && msg.data.uri) { - entry.parentUris?.delete(msg.data.uri); - } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { - entry.childToParent?.set(msg.data.child.rkey, { - parentUri: msg.data.parentUri, - relName: msg.data.relation - }); - } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { - entry.childToParent?.delete(msg.data.childRkey); - } - } - } - - private readonly sseControllers = new Set<SseEntry>(); - private readonly encoder = new TextEncoder(); -} - -interface SseEntry { - controller: ReadableStreamDefaultController<Uint8Array>; - did: string | undefined; - querySpec?: SubscriberQuerySpec; - parentUris?: Set<string>; - childToParent?: Map<string, { parentUri: string; relName: string }>; -} - -interface WsAttachment { - did?: string; - querySpec?: SubscriberQuerySpec; - /** URIs of primary records currently in this subscriber's result set. */ - parentUris?: string[]; - /** childRkey → parent info, for routing child delete events. */ - childToParent?: Record<string, { parentUri: string; relName: string }>; -} - -function setAttachment(ws: any, attachment: WsAttachment): void { - try { - ws.serializeAttachment?.(attachment); - } catch { - /* non-hibernating socket — fall back to a direct property */ - ws.__attachment = attachment; - } -} - -function getAttachment(ws: any): WsAttachment | null { - try { - const a = ws.deserializeAttachment?.(); - if (a) return a as WsAttachment; - } catch { - /* ignore */ - } - return (ws.__attachment as WsAttachment | undefined) ?? null; -} - -// Query-spec filtering lives in ./query-filter so the Worker can reuse it for -// non-DO (InMemoryPubSub) watchRecords paths without bundling the whole DO. +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/in-memory.ts b/packages/contrail/src/core/realtime/in-memory.ts index 311c0a5..1129419 100644 --- a/packages/contrail/src/core/realtime/in-memory.ts +++ b/packages/contrail/src/core/realtime/in-memory.ts @@ -1,116 +1 @@ -import type { PubSub, RealtimeEvent } from "./types"; -import { DEFAULT_QUEUE_BOUND } from "./types"; - -/** Single-process PubSub backed by in-memory subscriber sets. - * - * Each subscriber owns a bounded queue; when full, oldest events are dropped. - * `publish` returns once every subscriber has been offered the event — it - * never awaits a subscriber's consumption, so a slow consumer can't block - * producers. The cost of that guarantee is the drop-oldest policy. */ -export class InMemoryPubSub implements PubSub { - private readonly subscribers = new Map<string, Set<Subscriber>>(); - private readonly queueBound: number; - - constructor(opts: { queueBound?: number } = {}) { - this.queueBound = opts.queueBound ?? DEFAULT_QUEUE_BOUND; - } - - async publish(event: RealtimeEvent): Promise<void> { - const set = this.subscribers.get(event.topic); - if (!set) return; - for (const sub of set) sub.push(event); - } - - subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { - const sub = new Subscriber(this.queueBound); - let set = this.subscribers.get(topic); - if (!set) { - set = new Set(); - this.subscribers.set(topic, set); - } - set.add(sub); - - const cleanup = () => { - sub.close(); - const s = this.subscribers.get(topic); - if (s) { - s.delete(sub); - if (s.size === 0) this.subscribers.delete(topic); - } - }; - - if (signal) { - if (signal.aborted) cleanup(); - else signal.addEventListener("abort", cleanup, { once: true }); - } - - return sub.iterate(cleanup); - } - - /** Test-only: current subscriber count for a topic. */ - subscriberCount(topic: string): number { - return this.subscribers.get(topic)?.size ?? 0; - } -} - -class Subscriber { - private readonly queue: RealtimeEvent[] = []; - private pending: ((v: RealtimeEvent | null) => void) | null = null; - private closed = false; - /** Number of events dropped because the queue was full. The consumer can - * observe a gap by comparing monotonic event timestamps; exposing the - * count on a side channel is future work. */ - public droppedCount = 0; - - constructor(private readonly bound: number) {} - - push(event: RealtimeEvent): void { - if (this.closed) return; - if (this.pending) { - const p = this.pending; - this.pending = null; - p(event); - return; - } - if (this.queue.length >= this.bound) { - this.queue.shift(); - this.droppedCount += 1; - } - this.queue.push(event); - } - - close(): void { - if (this.closed) return; - this.closed = true; - if (this.pending) { - const p = this.pending; - this.pending = null; - p(null); - } - } - - iterate(cleanup: () => void): AsyncIterable<RealtimeEvent> { - const self = this; - return { - [Symbol.asyncIterator]() { - return { - async next(): Promise<IteratorResult<RealtimeEvent>> { - if (self.queue.length > 0) { - return { value: self.queue.shift()!, done: false }; - } - if (self.closed) return { value: undefined, done: true }; - const event = await new Promise<RealtimeEvent | null>((resolve) => { - self.pending = resolve; - }); - if (event === null) return { value: undefined, done: true }; - return { value: event, done: false }; - }, - async return(): Promise<IteratorResult<RealtimeEvent>> { - cleanup(); - return { value: undefined, done: true }; - }, - }; - }, - }; - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/merge.ts b/packages/contrail/src/core/realtime/merge.ts index e0fa7df..1129419 100644 --- a/packages/contrail/src/core/realtime/merge.ts +++ b/packages/contrail/src/core/realtime/merge.ts @@ -1,77 +1 @@ -/** Merge N AsyncIterables into one, interleaving events as they arrive. - * Terminates when every source terminates, or when `signal` aborts. */ - -export function mergeAsyncIterables<T>( - sources: AsyncIterable<T>[], - signal?: AbortSignal -): AsyncIterable<T> { - if (sources.length === 0) { - return { - async *[Symbol.asyncIterator]() { - /* nothing to yield */ - }, - }; - } - - return { - [Symbol.asyncIterator]() { - const iterators = sources.map((s) => s[Symbol.asyncIterator]()); - // One in-flight next() per source, racing each other. - type Slot = { - idx: number; - promise: Promise<{ idx: number; result: IteratorResult<T> }>; - }; - const pending = new Map<number, Slot>(); - let doneCount = 0; - - const schedule = (idx: number) => { - const slot: Slot = { - idx, - promise: iterators[idx]! - .next() - .then((result) => ({ idx, result })), - }; - pending.set(idx, slot); - }; - - for (let i = 0; i < iterators.length; i++) schedule(i); - - const cleanup = () => { - for (const it of iterators) { - try { - it.return?.(); - } catch { - /* ignore */ - } - } - }; - - if (signal) { - if (signal.aborted) cleanup(); - else signal.addEventListener("abort", cleanup, { once: true }); - } - - return { - async next(): Promise<IteratorResult<T>> { - while (pending.size > 0) { - const slots = [...pending.values()]; - const { idx, result } = await Promise.race(slots.map((s) => s.promise)); - pending.delete(idx); - if (result.done) { - doneCount += 1; - if (doneCount === iterators.length) return { value: undefined, done: true }; - continue; - } - schedule(idx); - return { value: result.value, done: false }; - } - return { value: undefined, done: true }; - }, - async return(): Promise<IteratorResult<T>> { - cleanup(); - return { value: undefined, done: true }; - }, - }; - }, - }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/query-filter.ts b/packages/contrail/src/core/realtime/query-filter.ts index 1b79eb6..1129419 100644 --- a/packages/contrail/src/core/realtime/query-filter.ts +++ b/packages/contrail/src/core/realtime/query-filter.ts @@ -1,235 +1 @@ -/** Shared query-spec → event-translation logic. - * - * Used by: - * - the Durable Object's WS publish path (per-subscriber filter after hibernation) - * - the Worker's SSE / Worker-terminated WS path (in-process filter) - * - * Given a raw RealtimeEvent and a SubscriberQuerySpec, returns the list of - * `{kind, data}` envelopes to send to the subscriber, or `null` if the - * subscriber has no spec (i.e. raw-firehose mode). */ - -import type { RealtimeEvent } from "./types"; -import type { SubscriberQuerySpec } from "./durable-object"; - -export type TranslatedEnvelope = - | { - kind: "record.created"; - data: { - record: { - uri: string; - did: string; - rkey: string; - collection: string; - cid: string | null | undefined; - record: Record<string, unknown>; - time_us: number; - indexed_at: number; - space: string; - }; - }; - } - | { kind: "record.deleted"; data: { uri: string; did: string; rkey: string } } - | { - kind: "hydration.added"; - data: { - parentUri: string; - relation: string; - child: { - uri: string; - did: string; - rkey: string; - collection: string; - cid: string | null | undefined; - record: Record<string, unknown>; - space: string; - }; - }; - } - | { - kind: "hydration.removed"; - data: { - parentUri: string; - relation: string; - childRkey: string; - childDid?: string; - }; - }; - -export interface SubscriberView { - querySpec?: SubscriberQuerySpec; - parentUris?: Set<string> | string[]; - childToParent?: - | Map<string, { parentUri: string; relName: string }> - | Record<string, { parentUri: string; relName: string }>; -} - -export function translateForQuery( - event: RealtimeEvent, - sub: SubscriberView -): TranslatedEnvelope[] | null { - const spec = sub.querySpec; - if (!spec) return null; - if (event.kind !== "record.created" && event.kind !== "record.deleted") return []; - if (event.payload.space !== spec.spaceUri) return []; - - const primaryUri = `at://${event.payload.did}/${event.payload.collection}/${event.payload.rkey}`; - - if (event.payload.collection === spec.collection) { - if (event.kind === "record.created") { - return [ - { - kind: "record.created", - data: { - record: { - uri: primaryUri, - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - record: event.payload.record, - time_us: event.ts * 1000, - indexed_at: event.ts, - space: spec.spaceUri - } - } - } - ]; - } - return [ - { - kind: "record.deleted", - data: { - uri: primaryUri, - did: event.payload.did, - rkey: event.payload.rkey - } - } - ]; - } - - if (!spec.hydrate) return []; - for (const [relName, rel] of Object.entries(spec.hydrate)) { - if (rel.childCollection !== event.payload.collection) continue; - if (event.kind === "record.created") { - const parentUri = getNestedValue( - event.payload.record as Record<string, unknown>, - rel.matchField - ); - if (typeof parentUri !== "string") continue; - if (!hasParent(sub.parentUris, parentUri)) continue; - return [ - { - kind: "hydration.added", - data: { - parentUri, - relation: relName, - child: { - uri: primaryUri, - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - record: event.payload.record, - space: spec.spaceUri - } - } - } - ]; - } - const info = getChildInfo(sub.childToParent, event.payload.rkey); - if (!info || info.relName !== relName) continue; - return [ - { - kind: "hydration.removed", - data: { - parentUri: info.parentUri, - relation: relName, - childRkey: event.payload.rkey, - childDid: event.payload.did - } - } - ]; - } - return []; -} - -export function applyEnvelopesToSubscriber( - subscriber: SubscriberView, - envs: TranslatedEnvelope[] -): void { - for (const msg of envs) { - if (msg.kind === "record.created") { - ensureParentSet(subscriber).add(msg.data.record.uri); - } else if (msg.kind === "record.deleted") { - const set = subscriber.parentUris; - if (set instanceof Set) set.delete(msg.data.uri); - else if (Array.isArray(set)) { - const idx = set.indexOf(msg.data.uri); - if (idx >= 0) set.splice(idx, 1); - } - } else if (msg.kind === "hydration.added") { - ensureChildMap(subscriber).set(msg.data.child.rkey, { - parentUri: msg.data.parentUri, - relName: msg.data.relation - }); - } else if (msg.kind === "hydration.removed") { - const map = subscriber.childToParent; - if (map instanceof Map) map.delete(msg.data.childRkey); - else if (map && typeof map === "object") { - delete (map as Record<string, unknown>)[msg.data.childRkey]; - } - } - } -} - -function ensureParentSet(sub: SubscriberView): Set<string> { - if (sub.parentUris instanceof Set) return sub.parentUris; - const set = new Set<string>(sub.parentUris ?? []); - sub.parentUris = set; - return set; -} - -function ensureChildMap( - sub: SubscriberView -): Map<string, { parentUri: string; relName: string }> { - if (sub.childToParent instanceof Map) return sub.childToParent; - const map = new Map<string, { parentUri: string; relName: string }>(); - if (sub.childToParent && typeof sub.childToParent === "object") { - for (const [k, v] of Object.entries(sub.childToParent)) map.set(k, v); - } - sub.childToParent = map; - return map; -} - -function hasParent( - parents: Set<string> | string[] | undefined, - uri: string -): boolean { - if (!parents) return false; - if (parents instanceof Set) return parents.has(uri); - return parents.includes(uri); -} - -function getChildInfo( - map: - | Map<string, { parentUri: string; relName: string }> - | Record<string, { parentUri: string; relName: string }> - | undefined, - rkey: string -): { parentUri: string; relName: string } | undefined { - if (!map) return undefined; - if (map instanceof Map) return map.get(rkey); - return map[rkey]; -} - -function getNestedValue( - obj: Record<string, unknown>, - path: string -): unknown { - let cur: unknown = obj; - for (const key of path.split(".")) { - if (cur == null || typeof cur !== "object") return undefined; - cur = (cur as Record<string, unknown>)[key]; - } - return cur; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/sse.ts b/packages/contrail/src/core/realtime/sse.ts index 65d89c1..1129419 100644 --- a/packages/contrail/src/core/realtime/sse.ts +++ b/packages/contrail/src/core/realtime/sse.ts @@ -1,98 +1 @@ -/** Server-Sent Events transport. - * - * Wraps an AsyncIterable<RealtimeEvent> as a streaming Response. The caller - * (the router) has already done auth and has an AbortSignal it can use to - * tear the stream down (e.g. on `member.removed` for the subscriber's DID). */ - -import type { RealtimeEvent } from "./types"; -import { DEFAULT_KEEPALIVE_MS } from "./types"; - -export interface SseOptions { - keepaliveMs?: number; - /** Called before the stream closes. Useful for cleanup that the caller - * can't do via the signal (e.g. removing a subscriber from a set). */ - onClose?: () => void; -} - -export function sseResponse( - iter: AsyncIterable<RealtimeEvent>, - signal: AbortSignal, - opts: SseOptions = {} -): Response { - const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; - const encoder = new TextEncoder(); - - const stream = new ReadableStream<Uint8Array>({ - start(controller) { - let closed = false; - let keepalive: ReturnType<typeof setInterval> | null = null; - - const close = () => { - if (closed) return; - closed = true; - if (keepalive) clearInterval(keepalive); - try { - controller.close(); - } catch { - /* already closed */ - } - opts.onClose?.(); - }; - - signal.addEventListener("abort", close, { once: true }); - - keepalive = setInterval(() => { - if (closed) return; - try { - controller.enqueue(encoder.encode(`: keepalive\n\n`)); - } catch { - close(); - } - }, keepaliveMs); - - (async () => { - // Opening comment — helps some clients / proxies initialize promptly. - controller.enqueue(encoder.encode(`: open\n\n`)); - try { - for await (const event of iter) { - if (closed) break; - controller.enqueue(encoder.encode(frameEvent(event))); - } - } catch (err) { - if (!closed) { - try { - controller.enqueue( - encoder.encode( - `event: error\ndata: ${JSON.stringify({ - message: err instanceof Error ? err.message : String(err), - })}\n\n` - ) - ); - } catch { - /* stream already torn down */ - } - } - } finally { - close(); - } - })(); - }, - cancel() { - opts.onClose?.(); - }, - }); - - return new Response(stream, { - status: 200, - headers: { - "content-type": "text/event-stream", - "cache-control": "no-cache, no-transform", - connection: "keep-alive", - "x-accel-buffering": "no", - }, - }); -} - -function frameEvent(event: RealtimeEvent): string { - return `event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/ticket.ts b/packages/contrail/src/core/realtime/ticket.ts index 1c6e3de..1129419 100644 --- a/packages/contrail/src/core/realtime/ticket.ts +++ b/packages/contrail/src/core/realtime/ticket.ts @@ -1,179 +1 @@ -/** Subscription tickets — HMAC-signed short-lived `{topics, did, exp}` blobs. - * - * Wire format: `<payload>.<sig>` where - * payload = base64url(JSON({ topics, did, exp, iat })) - * sig = base64url(HMAC-SHA256(key, payload)) - * - * Tickets are integrity-only (not encrypted). Browsers use them because - * EventSource / WebSocket can't send Authorization headers; server-side - * consumers skip the ticket dance and send their JWT directly. */ - -export interface TicketPayload { - /** Concrete delivery topics this ticket authorizes. `community:<did>` is - * expanded to the caller's visible spaces before signing — a ticket never - * carries a community alias. */ - topics: string[]; - did: string; - /** Unix ms. */ - exp: number; - /** Unix ms — useful for debugging; ignored on verify. */ - iat: number; - /** Optional: query-scoped watchRecords spec this ticket authorizes. Present - * when the ticket was minted from a watchRecords handshake. The server - * trusts the signed spec on upgrade and forwards it to the DO. */ - querySpec?: TicketQuerySpec; -} - -export interface TicketQuerySpec { - collection: string; - /** Exactly one of `spaceUri` or `actor` is set. `spaceUri` = per-space - * watch; `actor` = cross-space watch for records authored by this DID - * (the ticket's `topics` list carries the expanded delivery topics). */ - spaceUri?: string; - actor?: string; - hydrate?: Record<string, { childCollection: string; matchField: string }>; -} - -function normalizeSecret(secret: Uint8Array | string): Uint8Array { - if (typeof secret !== "string") { - if (secret.length !== 32) { - throw new Error(`realtime ticketSecret must be 32 bytes, got ${secret.length}`); - } - return secret; - } - // 64 hex chars would also round-trip as base64 (to 48 bytes). Prefer hex - // when the input matches the hex alphabet exactly; fall back to base64. - const hex = tryHex(secret); - if (hex && hex.length === 32) return hex; - const b64 = tryBase64(secret); - if (b64 && b64.length === 32) return b64; - if (hex || b64) { - const got = (hex ?? b64)!.length; - throw new Error(`realtime ticketSecret must decode to 32 bytes, got ${got}`); - } - throw new Error("realtime ticketSecret must be a 32-byte Uint8Array or base64/hex string"); -} - -function tryBase64(s: string): Uint8Array | null { - try { - const normal = s.replace(/-/g, "+").replace(/_/g, "/"); - const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); - if (!/^[A-Za-z0-9+/]*=*$/.test(padded)) return null; - const bin = atob(padded); - const out = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); - return out; - } catch { - return null; - } -} - -function tryHex(s: string): Uint8Array | null { - if (!/^[0-9a-fA-F]+$/.test(s) || s.length % 2 !== 0) return null; - const out = new Uint8Array(s.length / 2); - for (let i = 0; i < out.length; i++) { - out[i] = parseInt(s.slice(i * 2, i * 2 + 2), 16); - } - return out; -} - -function b64urlFromBytes(bytes: Uint8Array): string { - let bin = ""; - for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); - return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); -} - -function b64urlToBytes(s: string): Uint8Array { - const normal = s.replace(/-/g, "+").replace(/_/g, "/"); - const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); - const bin = atob(padded); - const out = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); - return out; -} - -function b64urlFromString(s: string): string { - return b64urlFromBytes(new TextEncoder().encode(s)); -} - -function stringFromB64url(s: string): string { - return new TextDecoder().decode(b64urlToBytes(s)); -} - -function constantTimeEq(a: Uint8Array, b: Uint8Array): boolean { - if (a.length !== b.length) return false; - let diff = 0; - for (let i = 0; i < a.length; i++) diff |= a[i]! ^ b[i]!; - return diff === 0; -} - -export class TicketSigner { - private readonly keyPromise: Promise<CryptoKey>; - - constructor(secret: Uint8Array | string) { - const raw = normalizeSecret(secret); - this.keyPromise = crypto.subtle.importKey( - "raw", - raw as BufferSource, - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign", "verify"] - ); - } - - async sign(input: { - topics: string[]; - did: string; - ttlMs: number; - querySpec?: TicketQuerySpec; - }): Promise<string> { - const now = Date.now(); - const payload: TicketPayload = { - topics: input.topics, - did: input.did, - exp: now + input.ttlMs, - iat: now, - ...(input.querySpec ? { querySpec: input.querySpec } : {}), - }; - const payloadPart = b64urlFromString(JSON.stringify(payload)); - const sig = await crypto.subtle.sign( - "HMAC", - await this.keyPromise, - new TextEncoder().encode(payloadPart) as BufferSource - ); - const sigPart = b64urlFromBytes(new Uint8Array(sig)); - return `${payloadPart}.${sigPart}`; - } - - /** Returns the decoded payload if the ticket is valid + unexpired, else null. */ - async verify(ticket: string): Promise<TicketPayload | null> { - const dot = ticket.indexOf("."); - if (dot < 0) return null; - const payloadPart = ticket.slice(0, dot); - const sigPart = ticket.slice(dot + 1); - let expectedSig: Uint8Array; - try { - expectedSig = b64urlToBytes(sigPart); - } catch { - return null; - } - const computedRaw = await crypto.subtle.sign( - "HMAC", - await this.keyPromise, - new TextEncoder().encode(payloadPart) as BufferSource - ); - const computed = new Uint8Array(computedRaw); - if (!constantTimeEq(expectedSig, computed)) return null; - let parsed: TicketPayload; - try { - parsed = JSON.parse(stringFromB64url(payloadPart)); - } catch { - return null; - } - if (!parsed || !Array.isArray(parsed.topics) || typeof parsed.did !== "string") { - return null; - } - if (typeof parsed.exp !== "number" || parsed.exp <= Date.now()) return null; - return parsed; - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/types.ts b/packages/contrail/src/core/realtime/types.ts index 05a246e..1129419 100644 --- a/packages/contrail/src/core/realtime/types.ts +++ b/packages/contrail/src/core/realtime/types.ts @@ -1,127 +1 @@ -/** Realtime module — canonical types + interfaces. See docs/realtime.md. */ - -/** Discriminated union of every event kind that flows through the PubSub. - * - * `record.created` carries the full record body so a subscriber can apply an - * insert or upsert without a follow-up `getRecord` call. Writing a new record - * to the same `(did, collection, rkey)` publishes another `record.created` — - * treat it as upsert. - * - * **Payload shape mirrors `listRecords` output** (`uri`, `did`, `space?`, - * `time_us`), so a subscriber can render a live row the same way it renders - * a fetched row. - * - * **Publisher/topic matrix (intentional trust split):** - * - `collection:<nsid>` and `actor:<did>` carry *public* record events only - * (from jetstream ingestion) — no `space`. - * - `space:<uri>` and `community:<did>` carry *space* events — `space` is - * always set. Never cross-published to public topics (privacy). */ -export type RealtimeEvent = - | { - topic: string; - kind: "record.created"; - payload: { - uri: string; - did: string; - collection: string; - rkey: string; - cid: string | null; - record: Record<string, unknown>; - time_us: number; - /** Present only for space records; absent for public records. */ - space?: string; - }; - ts: number; - } - | { - topic: string; - kind: "record.deleted"; - payload: { - uri: string; - did: string; - collection: string; - rkey: string; - /** Present only for space records; absent for public records. */ - space?: string; - }; - ts: number; - } - | { - topic: string; - kind: "member.added"; - payload: { space: string; did: string }; - ts: number; - } - | { - topic: string; - kind: "member.removed"; - payload: { space: string; did: string }; - ts: number; - }; - -export type RealtimeEventKind = RealtimeEvent["kind"]; - -/** Core pubsub abstraction. Implementations: InMemoryPubSub, DurableObjectPubSub. */ -export interface PubSub { - publish(event: RealtimeEvent): Promise<void>; - /** Stream events on the topic until the signal aborts (or the iterator is - * returned/broken out of). Implementations use a bounded per-subscriber - * queue with drop-oldest semantics — a slow subscriber can't stall publishers. */ - subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent>; -} - -// ---- Canonical topic strings ----------------------------------------------- -// `community:<did>` is an alias resolved at ticket-mint time to the concrete -// set of `space:<uri>` topics the caller can see; it is never a real delivery -// topic. The other three are real. - -export function spaceTopic(uri: string): string { - return `space:${uri}`; -} - -export function communityTopic(did: string): string { - return `community:${did}`; -} - -export function collectionTopic(nsid: string): string { - return `collection:${nsid}`; -} - -export function actorTopic(did: string): string { - return `actor:${did}`; -} - -export function isCommunityTopic(topic: string): boolean { - return topic.startsWith("community:"); -} - -export function parseCommunityTopic(topic: string): string | null { - return isCommunityTopic(topic) ? topic.slice("community:".length) : null; -} - -export function parseSpaceTopic(topic: string): string | null { - return topic.startsWith("space:") ? topic.slice("space:".length) : null; -} - -// ---- Config ----------------------------------------------------------------- - -export interface RealtimeConfig { - /** Backing pubsub. Default: new InMemoryPubSub() (single-process only). On - * Workers, pass `new DurableObjectPubSub(env.REALTIME)`. */ - pubsub?: PubSub; - /** HMAC secret used to sign subscription tickets. 32 bytes. Accepts raw - * Uint8Array or base64 / hex string. Envelope-encrypts nothing — tickets - * are integrity-only, not confidential. */ - ticketSecret: Uint8Array | string; - /** Ticket lifetime in ms. Default 120_000 (2 minutes). */ - ticketTtlMs?: number; - /** SSE/WS keepalive interval in ms. Default 15_000. */ - keepaliveMs?: number; - /** Per-subscriber queue bound. Default 1024. Events beyond this are dropped - * oldest-first and the subscriber receives a `lag` signal (out of band). */ - queueBound?: number; -} - -export const DEFAULT_TICKET_TTL_MS = 120_000; -export const DEFAULT_KEEPALIVE_MS = 15_000; -export const DEFAULT_QUEUE_BOUND = 1024; +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/websocket.ts b/packages/contrail/src/core/realtime/websocket.ts index 833483d..1129419 100644 --- a/packages/contrail/src/core/realtime/websocket.ts +++ b/packages/contrail/src/core/realtime/websocket.ts @@ -1,84 +1 @@ -/** WebSocket transport. - * - * Accepts a new WebSocket connection (either via `WebSocketPair` on Workers - * or a platform-provided server-side socket) and pumps events to it from an - * AsyncIterable<RealtimeEvent>. Messages are UTF-8 JSON, one event per frame. - * - * Close codes (subset, RFC 6455 + app-custom): - * - 4001: server error pumping - * - 4003: membership revoked - * - 4008: ticket/auth invalid (used by the router, not here) - */ - -import type { RealtimeEvent } from "./types"; -import { DEFAULT_KEEPALIVE_MS } from "./types"; - -export interface WebSocketLike { - send(data: string): void; - close(code?: number, reason?: string): void; - addEventListener(type: "message" | "close" | "error", listener: (ev: any) => void): void; -} - -export interface WebSocketPumpOptions { - keepaliveMs?: number; - onClose?: () => void; -} - -/** Pump events from `iter` to `ws` until the signal aborts or the iter ends. - * Caller is responsible for having already accept()ed the socket. */ -export async function pumpWebSocket( - ws: WebSocketLike, - iter: AsyncIterable<RealtimeEvent>, - signal: AbortSignal, - opts: WebSocketPumpOptions = {} -): Promise<void> { - const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; - let closed = false; - - const close = (code?: number, reason?: string) => { - if (closed) return; - closed = true; - try { - ws.close(code, reason); - } catch { - /* already closed */ - } - opts.onClose?.(); - }; - - ws.addEventListener("close", () => { - closed = true; - opts.onClose?.(); - }); - ws.addEventListener("error", () => { - closed = true; - opts.onClose?.(); - }); - signal.addEventListener("abort", () => close(1000, "aborted"), { once: true }); - - const keepalive = setInterval(() => { - if (closed) return; - try { - ws.send(JSON.stringify({ kind: "$keepalive" })); - } catch { - close(); - } - }, keepaliveMs); - - try { - for await (const event of iter) { - if (closed) break; - try { - ws.send(JSON.stringify(event)); - } catch { - close(4001, "send-failed"); - break; - } - } - } catch { - close(4001, "pump-error"); - } finally { - clearInterval(keepalive); - close(); - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/refresh.ts b/packages/contrail/src/core/refresh.ts index aacedaa..7b2fd21 100644 --- a/packages/contrail/src/core/refresh.ts +++ b/packages/contrail/src/core/refresh.ts @@ -1,3 +1,4 @@ +import type {} from "@atcute/atproto"; /** * Fresh refresh: re-walk every known DID's PDS for every configured collection * and reconcile against what's in our DB. Unlike `backfillPending`, this diff --git a/packages/contrail/src/core/spaces/acl.ts b/packages/contrail/src/core/spaces/acl.ts index 6e0da0a..1129419 100644 --- a/packages/contrail/src/core/spaces/acl.ts +++ b/packages/contrail/src/core/spaces/acl.ts @@ -1,69 +1 @@ -import type { AppPolicy, SpaceMemberRow, SpaceRow } from "./types"; - -export type AclOp = "read" | "write" | "delete"; - -export interface AclInput { - op: AclOp; - space: SpaceRow; - callerDid: string; - /** Membership row for the caller (or null). Owner does not require a row. */ - member: SpaceMemberRow | null; - /** OAuth client_id of the app calling on caller's behalf, for app policy checks. */ - clientId?: string; - /** For per-record ops (get/delete), the record's author DID. */ - targetAuthorDid?: string; -} - -export type AclResult = - | { allow: true } - | { allow: false; reason: AclDenyReason }; - -export type AclDenyReason = - | "not-member" - | "not-own-record" - | "app-not-allowed" - | "unknown-op"; - -/** Check whether the caller's app is permitted to act in this space. */ -export function checkAppPolicy( - appPolicy: AppPolicy | null, - clientId: string | undefined -): boolean { - if (!appPolicy) return true; // no policy = allow-all - const listed = clientId ? appPolicy.apps.includes(clientId) : false; - if (appPolicy.mode === "allow") return !listed; // apps[] is a denylist - return listed; // mode === "deny": apps[] is an allowlist -} - -const isOwner = (space: SpaceRow, did: string) => space.ownerDid === did; -const hasMember = (space: SpaceRow, member: SpaceMemberRow | null, did: string) => - isOwner(space, did) || member != null; - -/** Space-level access check. - * Membership = access. Any member (including owner) can read and write. - * Delete is scoped to the caller's own records — owners don't get a bypass. - * A random member can't nuke other people's records, and neither can the - * owner. To remove a non-author record, delete the whole space. */ -export function checkAccess(input: AclInput): AclResult { - if (!checkAppPolicy(input.space.appPolicy, input.clientId)) { - return { allow: false, reason: "app-not-allowed" }; - } - - if (input.op === "read" || input.op === "write") { - return hasMember(input.space, input.member, input.callerDid) - ? { allow: true } - : { allow: false, reason: "not-member" }; - } - - if (input.op === "delete") { - if (!hasMember(input.space, input.member, input.callerDid)) { - return { allow: false, reason: "not-member" }; - } - if (input.targetAuthorDid && input.targetAuthorDid !== input.callerDid) { - return { allow: false, reason: "not-own-record" }; - } - return { allow: true }; - } - - return { allow: false, reason: "unknown-op" }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 4c1e817..1129419 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -1,177 +1 @@ -import type { Context, MiddlewareHandler } from "hono"; -import { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; -import { - CompositeDidDocumentResolver, - PlcDidDocumentResolver, - WebDidDocumentResolver, - type DidDocumentResolver, -} from "@atcute/identity-resolver"; -import type { Did, Nsid } from "@atcute/lexicons"; -import type { AuthorityConfig } from "./types"; -import { readInProcess } from "./in-process"; - -export { ServiceJwtVerifier }; - -/** Build a ServiceJwtVerifier from an AuthorityConfig, using the configured - * resolver or a default PLC+Web composite. The verifier checks that incoming - * JWTs target this authority's serviceDid (aud claim). */ -export function buildVerifier(authority: AuthorityConfig): ServiceJwtVerifier { - const resolver = - authority.resolver ?? - new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, - }); - return new ServiceJwtVerifier({ - serviceDid: authority.serviceDid as Did, - resolver, - }); -} - -export interface ServiceAuth { - issuer: string; - audience: string; - lxm: string | undefined; - /** OAuth client_id of the caller, if the JWT carries one. */ - clientId?: string; -} - -export interface ServiceAuthOptions { - serviceDid: Did; - resolver: DidDocumentResolver; -} - -/** Hono middleware that authenticates XRPC requests. Order of precedence: - * 1. In-process marker (same-module calls; see `core/spaces/in-process.ts`) - * 2. Authorization: Bearer <JWT> as an atproto service-auth token - * - * On success, attaches the claims to `c.var.serviceAuth`. Expected Nsid is - * taken from the route pattern (last segment after `/xrpc/`). */ -export function createServiceAuthMiddleware( - verifier: ServiceJwtVerifier -): MiddlewareHandler { - return async (c, next) => { - const lxm = extractLxmFromPath(c); - - const inProcess = readInProcess(c.req.raw); - if (inProcess) { - c.set("serviceAuth", { - issuer: inProcess.did, - audience: "", - lxm: lxm ?? undefined, - } satisfies ServiceAuth); - await next(); - return; - } - - const header = c.req.header("Authorization"); - if (!header || !header.startsWith("Bearer ")) { - return c.json({ error: "AuthRequired", message: "Missing bearer token" }, 401); - } - const token = header.slice(7).trim(); - - const result = await verifier.verify(token, { lxm }); - if (!result.ok) { - const err = result.error as { error?: string; description?: string } | undefined; - return c.json( - { - error: "AuthRequired", - message: err?.description ?? err?.error ?? String(result.error), - }, - 401, - ); - } - - c.set("serviceAuth", { - issuer: result.value.issuer, - audience: result.value.audience, - lxm: result.value.lxm, - } satisfies ServiceAuth); - - await next(); - }; -} - -function extractLxmFromPath(c: Context): Nsid | null { - const path = new URL(c.req.url).pathname; - const match = path.match(/\/xrpc\/([a-zA-Z0-9.-]+)/); - return (match?.[1] as Nsid) ?? null; -} - -/** Read the service auth claims set by the middleware. Throws if unset. */ -export function requireServiceAuth(c: Context): ServiceAuth { - const auth = c.get("serviceAuth") as ServiceAuth | undefined; - if (!auth) throw new Error("service auth missing; middleware not attached"); - return auth; -} - -/** Out-of-band auth check for handlers that don't always require auth. - * Returns claims on success, or null if no valid credentials are present. - * Order of precedence: in-process marker → service-auth JWT. */ -export async function verifyServiceAuthRequest( - verifier: ServiceJwtVerifier, - request: Request, - lxm?: Nsid | null -): Promise<ServiceAuth | null> { - const inProcess = readInProcess(request); - if (inProcess) { - return { - issuer: inProcess.did, - audience: "", - lxm: lxm ?? undefined, - }; - } - - const header = request.headers.get("Authorization"); - if (!header || !header.startsWith("Bearer ")) return null; - const token = header.slice(7).trim(); - const result = await verifier.verify(token, { lxm: lxm ?? null }); - if (!result.ok) return null; - return { - issuer: result.value.issuer, - audience: result.value.audience, - lxm: result.value.lxm, - }; -} - -/** Pull a space credential off the request — `X-Space-Credential: <jwt>` - * header. Returns the raw token or null. */ -export function extractSpaceCredential(request: Request): string | null { - const header = request.headers.get("X-Space-Credential"); - return header ? header.trim() : null; -} - -/** Pull a read-grant invite token off the request — query string `?inviteToken=` - * or `Authorization: Bearer atmo-invite:<token>`. Returns the raw token (not - * hashed) or null. Routes hash + look up via the adapter. */ -export function extractInviteToken(request: Request): string | null { - const url = new URL(request.url); - const q = url.searchParams.get("inviteToken"); - if (q) return q.trim(); - const header = request.headers.get("Authorization"); - if (header?.startsWith("Bearer atmo-invite:")) { - return header.slice("Bearer atmo-invite:".length).trim(); - } - return null; -} - -/** Validate a read-grant invite token against a target spaceUri. Returns true - * if the token exists, scopes to this space, has a kind that grants read - * (`read` or `read-join`), and is not expired/revoked. */ -export async function checkInviteReadGrant( - adapter: { getInvite(tokenHash: string): Promise<{ spaceUri: string; kind: string; revokedAt: number | null; expiresAt: number | null } | null> }, - rawToken: string, - spaceUri: string, - hashFn: (token: string) => Promise<string> -): Promise<boolean> { - const tokenHash = await hashFn(rawToken); - const invite = await adapter.getInvite(tokenHash); - if (!invite) return false; - if (invite.spaceUri !== spaceUri) return false; - if (invite.kind !== "read" && invite.kind !== "read-join") return false; - if (invite.revokedAt != null) return false; - if (invite.expiresAt != null && invite.expiresAt <= Date.now()) return false; - return true; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts index a81b95f..1129419 100644 --- a/packages/contrail/src/core/spaces/binding.ts +++ b/packages/contrail/src/core/spaces/binding.ts @@ -1,274 +1 @@ -/** Binding resolution: given a space URI, which DID is authorized to sign - * credentials for it, and where do we find that DID's verification key? - * - * Two layers of pluggable resolvers compose into a credential verifier: - * - * BindingResolver — `ats://<owner>/<type>/<key>` → authority DID - * KeyResolver — (DID, kid) → JsonWebKey - * - * The BindingResolver is what makes user-owned-DID-with-PDS-record work: - * given a space URI, we resolve the owner's PDS, fetch the declaration - * record, and read its `authority` field. For provisioned (no-PDS) DIDs we - * fall back to the owner DID's `#atproto_space_authority` service entry. - * And finally for the trivial case (HappyView-style "owner self-issues"), - * we return the owner DID itself. - * - * See conversation history (phase 4 design) for the rationale on why these - * three sources, in this order. */ - -import type { DidDocumentResolver } from "@atcute/identity-resolver"; -import type { Did } from "@atcute/lexicons"; -import { parseSpaceUri } from "./uri"; -import type { RecordHost } from "./types"; - -export interface BindingResolver { - /** Resolve the DID authorized to sign credentials for this space. Returns - * null if no binding could be found via this resolver — the composite - * walks down its list looking for a non-null. */ - resolveAuthority(spaceUri: string): Promise<string | null>; -} - -export interface KeyResolver { - /** Resolve `did`'s verification key for credential signing. `kid` is the - * full header `kid` value (e.g. "did:web:x.com#atproto_space_authority"), - * used to disambiguate when a DID doc lists multiple methods. */ - resolveKey(did: string, kid: string | undefined): Promise<JsonWebKey | null>; -} - -// --------------------------------------------------------------------------- -// Binding resolvers -// --------------------------------------------------------------------------- - -/** Always returns the configured authority DID. Used in-process when the - * authority and record host run in one deployment — no need to walk DID - * docs or PDSes; we know what we are. */ -export function createLocalBindingResolver(args: { - authorityDid: string; -}): BindingResolver { - const { authorityDid } = args; - return { - async resolveAuthority() { - return authorityDid; - }, - }; -} - -/** Reads the record host's local enrollment table. This is the *canonical* - * binding source on a record host: the host owner explicitly consented to a - * given authority for a given space (via the `recordHost.enroll` endpoint - * or auto-enrollment from the authority's createSpace). PDS-record / - * DID-doc resolvers are out-of-band discovery aids; the enrollment is what - * actually gates whether records get stored here. */ -export function createEnrollmentBindingResolver(args: { - recordHost: RecordHost; -}): BindingResolver { - return { - async resolveAuthority(spaceUri) { - const e = await args.recordHost.getEnrollment(spaceUri); - return e?.authorityDid ?? null; - }, - }; -} - -/** Returns the space owner DID as the authority. This is the implicit - * fallback ("HappyView path") — when no PDS record and no DID-doc service - * entry declare an issuer, the owner is taken to be its own. Whether the - * resulting credential actually verifies depends on whether the owner's DID - * doc publishes a usable signing key. */ -export function createOwnerSelfBindingResolver(): BindingResolver { - return { - async resolveAuthority(spaceUri) { - const parts = parseSpaceUri(spaceUri); - return parts ? parts.ownerDid : null; - }, - }; -} - -/** Walks the resolver list in order, returns the first non-null. Use this - * to compose [pdsRecord, didDocService, ownerSelf] etc. */ -export function createCompositeBindingResolver( - resolvers: BindingResolver[] -): BindingResolver { - return { - async resolveAuthority(spaceUri) { - for (const r of resolvers) { - const did = await r.resolveAuthority(spaceUri); - if (did) return did; - } - return null; - }, - }; -} - -/** Reads a space-declaration record from the owner's PDS at - * `at://<owner>/<type>/<key>` and returns its `authority` field if present. - * - * This is the user-owned-DID path: the user writes a record to their PDS - * authorizing some service as the space's authority, no DID-doc edits - * required. */ -export function createPdsBindingResolver(args: { - /** DID resolver, used to look up the owner's PDS endpoint. */ - resolver: DidDocumentResolver; - /** Fetch impl. Defaults to `globalThis.fetch`. */ - fetch?: typeof fetch; - /** Per-request timeout in ms. Defaults to 5000. */ - timeoutMs?: number; -}): BindingResolver { - const fetchImpl = args.fetch ?? globalThis.fetch; - const timeoutMs = args.timeoutMs ?? 5000; - - return { - async resolveAuthority(spaceUri) { - const parts = parseSpaceUri(spaceUri); - if (!parts) return null; - const pds = await pdsEndpointFor(args.resolver, parts.ownerDid); - if (!pds) return null; - - const url = new URL(`${pds}/xrpc/com.atproto.repo.getRecord`); - url.searchParams.set("repo", parts.ownerDid); - url.searchParams.set("collection", parts.type); - url.searchParams.set("rkey", parts.key); - - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), timeoutMs); - let res: Response; - try { - res = await fetchImpl(url.toString(), { signal: ctrl.signal }); - } catch { - return null; - } finally { - clearTimeout(timer); - } - if (!res.ok) return null; - const body = (await res.json().catch(() => null)) as - | { value?: { authority?: unknown } } - | null; - const authority = body?.value?.authority; - return typeof authority === "string" && authority.startsWith("did:") ? authority : null; - }, - }; -} - -/** Reads `service[id="#atproto_space_authority"].serviceEndpoint` from the - * owner's DID doc. This is the no-PDS path — useful for provisioned space - * DIDs that exist as DID docs only. - * - * Note the service endpoint here is a *DID*, not a URL. The DID names the - * authority; the key resolver's job is to then fetch its verification key. - * For DID docs that declare a URL endpoint, we treat the URL as a - * did:web hint — caller can normalize. */ -export function createDidDocBindingResolver(args: { - resolver: DidDocumentResolver; - /** Service id to look up. Defaults to "#atproto_space_authority". */ - serviceId?: string; -}): BindingResolver { - const serviceId = args.serviceId ?? "#atproto_space_authority"; - return { - async resolveAuthority(spaceUri) { - const parts = parseSpaceUri(spaceUri); - if (!parts) return null; - let doc; - try { - doc = await args.resolver.resolve(parts.ownerDid as Did); - } catch { - return null; - } - const entry = doc.service?.find((s: { id?: string }) => s.id === serviceId); - if (!entry) return null; - const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; - if (typeof endpoint !== "string") return null; - // Endpoint may be a DID (preferred) or a URL hint. Only DIDs are - // verifiable downstream; URLs require the caller to map URL → DID. - return endpoint.startsWith("did:") ? endpoint : null; - }, - }; -} - -// --------------------------------------------------------------------------- -// Key resolvers -// --------------------------------------------------------------------------- - -/** Knows the local authority's public key directly. Returns null for any - * other DID — composite with a DID-doc resolver if you also accept - * external authorities. */ -export function createLocalKeyResolver(args: { - authorityDid: string; - publicKey: JsonWebKey; -}): KeyResolver { - return { - async resolveKey(did) { - return did === args.authorityDid ? args.publicKey : null; - }, - }; -} - -/** Resolves a DID, finds the verification method matching `kid`, returns - * its `publicKeyJwk`. */ -export function createDidDocKeyResolver(args: { - resolver: DidDocumentResolver; -}): KeyResolver { - return { - async resolveKey(did, kid) { - let doc; - try { - doc = await args.resolver.resolve(did as Did); - } catch { - return null; - } - const methods = (doc as { verificationMethod?: VerificationMethod[] }).verificationMethod; - if (!methods) return null; - // kid is "<did>#<methodId>" — we match against the method.id which DID - // docs spell as "<did>#<methodId>" too. - const method = kid - ? methods.find((m) => m.id === kid) - : methods[0]; - if (!method?.publicKeyJwk) return null; - return method.publicKeyJwk as JsonWebKey; - }, - }; -} - -/** Walks resolvers in order; returns the first non-null. */ -export function createCompositeKeyResolver( - resolvers: KeyResolver[] -): KeyResolver { - return { - async resolveKey(did, kid) { - for (const r of resolvers) { - const k = await r.resolveKey(did, kid); - if (k) return k; - } - return null; - }, - }; -} - -interface VerificationMethod { - id: string; - type?: string; - controller?: string; - publicKeyJwk?: unknown; - publicKeyMultibase?: string; -} - -// --------------------------------------------------------------------------- -// Internal: PDS endpoint lookup -// --------------------------------------------------------------------------- - -async function pdsEndpointFor( - resolver: DidDocumentResolver, - did: string -): Promise<string | null> { - let doc; - try { - doc = await resolver.resolve(did as Did); - } catch { - return null; - } - const entry = doc.service?.find( - (s: { id?: string }) => s.id === "#atproto_pds" - ); - if (!entry) return null; - const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; - return typeof endpoint === "string" ? endpoint : null; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/blob-adapter.ts b/packages/contrail/src/core/spaces/blob-adapter.ts index 632b566..1129419 100644 --- a/packages/contrail/src/core/spaces/blob-adapter.ts +++ b/packages/contrail/src/core/spaces/blob-adapter.ts @@ -1,94 +1 @@ -/** - * Bytes-only storage adapter for space blobs. Metadata (CID, mime, size, - * author, space) lives in the `spaces_blobs` table on the main StorageAdapter; - * this interface only moves bytes in and out of a backend (R2, S3, fs, …). - * - * Keys are opaque strings formed by the router as `blobKey(spaceUri, cid)`. - */ - -export interface BlobUploadMeta { - mimeType: string; - size: number; -} - -export interface BlobAdapter { - put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void>; - get(key: string): Promise<Uint8Array | null>; - /** Bulk delete. Adapters that don't support batch can implement serially. */ - delete(keys: string[]): Promise<void>; -} - -/** In-memory adapter. Useful for tests and local development. */ -export class MemoryBlobAdapter implements BlobAdapter { - private readonly store = new Map<string, Uint8Array>(); - - async put(key: string, bytes: Uint8Array): Promise<void> { - this.store.set(key, bytes.slice()); - } - - async get(key: string): Promise<Uint8Array | null> { - const v = this.store.get(key); - return v ? v.slice() : null; - } - - async delete(keys: string[]): Promise<void> { - for (const k of keys) this.store.delete(k); - } - - /** Test helper. */ - size(): number { - return this.store.size; - } -} - -/** Minimal Cloudflare R2 bucket shape — matches @cloudflare/workers-types' R2Bucket - * without forcing a types dependency here. */ -export interface R2BucketLike { - put( - key: string, - value: ArrayBuffer | ArrayBufferView | ReadableStream | Blob, - options?: { httpMetadata?: { contentType?: string }; customMetadata?: Record<string, string> } - ): Promise<unknown>; - get(key: string): Promise<{ arrayBuffer(): Promise<ArrayBuffer> } | null>; - delete(keys: string | string[]): Promise<void>; -} - -/** Cloudflare R2 adapter. Pass the `env.BLOBS` binding from your Worker. */ -export class R2BlobAdapter implements BlobAdapter { - constructor(private readonly bucket: R2BucketLike) {} - - async put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void> { - await this.bucket.put(key, bytes, { - httpMetadata: { contentType: meta.mimeType }, - }); - } - - async get(key: string): Promise<Uint8Array | null> { - const obj = await this.bucket.get(key); - if (!obj) return null; - const buf = await obj.arrayBuffer(); - return new Uint8Array(buf); - } - - async delete(keys: string[]): Promise<void> { - if (keys.length === 0) return; - await this.bucket.delete(keys); - } -} - -/** Hash a space URI to a short, filesystem/R2-safe key segment. - * Used as the first segment of a blob key so all blobs for one space - * share a common prefix (enables bulk delete on space deletion). */ -export async function spaceKeyPrefix(spaceUri: string): Promise<string> { - const bytes = new TextEncoder().encode(spaceUri); - const digest = await crypto.subtle.digest("SHA-256", bytes); - const hex = Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join(""); - return hex.slice(0, 16); -} - -/** Compose an adapter key from a space URI and CID. - * Shape: `<16-hex-chars-of-sha256(spaceUri)>/<cid>`. */ -export async function blobKey(spaceUri: string, cid: string): Promise<string> { - const prefix = await spaceKeyPrefix(spaceUri); - return `${prefix}/${cid}`; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts index d5c3029..1129419 100644 --- a/packages/contrail/src/core/spaces/credentials.ts +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -1,284 +1 @@ -/** Space-credential primitives: ES256 (P-256) JWTs minted by the authority, - * verified by the record host (or any third party that can resolve the - * authority's DID document). - * - * Format is a compact JWS: - * header = { alg: "ES256", typ: "JWT", kid: "<authorityDid>#<keyId>" } - * payload = { iss, sub, space, scope, iat, exp } - * - * - `iss` is the authority DID (the signer; for phase 3 this is the local - * authority's serviceDid; phase 4 adds a binding-resolution layer that - * lets the issuer be a *different* DID from the space owner). - * - `sub` is the caller DID — the credential bearer. - * - `space` is the full `ats://<owner>/<type>/<key>` URI. - * - `scope` is "rw" or "read". - * - * We don't use a JWT library — Web Crypto's subtle covers everything (P-256 - * generate, sign, verify, JWK import/export) and saves a runtime dep. */ - -const ALG = "ES256"; -const TYP = "JWT"; -const DEFAULT_KEY_ID = "atproto_space_authority"; - -export type CredentialScope = "rw" | "read"; - -export interface CredentialClaims { - iss: string; - sub: string; - space: string; - scope: CredentialScope; - iat: number; // seconds since epoch - exp: number; // seconds since epoch -} - -export interface CredentialKeyMaterial { - /** Private key in JWK form. P-256 / ES256. */ - privateKey: JsonWebKey; - /** Public key in JWK form. Must match privateKey. */ - publicKey: JsonWebKey; - /** DID-doc verification method id. The full JWT `kid` becomes - * `<authorityDid>#<keyId>`. Defaults to "atproto_space_authority". */ - keyId?: string; -} - -/** Generate a fresh P-256 keypair as JWKs. Useful for local dev / tests; in - * production the operator generates once and stores out-of-band. */ -export async function generateAuthoritySigningKey(): Promise<CredentialKeyMaterial> { - const pair = (await crypto.subtle.generateKey( - { name: "ECDSA", namedCurve: "P-256" }, - true, - ["sign", "verify"] - )) as CryptoKeyPair; - const privateKey = (await crypto.subtle.exportKey("jwk", pair.privateKey)) as JsonWebKey; - const publicKey = (await crypto.subtle.exportKey("jwk", pair.publicKey)) as JsonWebKey; - return { privateKey, publicKey }; -} - -const enc = new TextEncoder(); -const dec = new TextDecoder(); - -function base64urlEncode(bytes: Uint8Array): string { - let s = btoa(String.fromCharCode(...bytes)); - return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); -} - -function base64urlDecode(s: string): Uint8Array { - const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); - const bin = atob(padded); - const out = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); - return out; -} - -function jsonEncode(value: unknown): string { - return base64urlEncode(enc.encode(JSON.stringify(value))); -} - -function jsonDecode<T>(seg: string): T { - return JSON.parse(dec.decode(base64urlDecode(seg))) as T; -} - -async function importPrivate(jwk: JsonWebKey): Promise<CryptoKey> { - return crypto.subtle.importKey( - "jwk", - jwk, - { name: "ECDSA", namedCurve: "P-256" }, - false, - ["sign"] - ); -} - -async function importPublic(jwk: JsonWebKey): Promise<CryptoKey> { - return crypto.subtle.importKey( - "jwk", - jwk, - { name: "ECDSA", namedCurve: "P-256" }, - false, - ["verify"] - ); -} - -/** Sign a credential payload with the authority's private key. - * `iat` and `exp` are filled in by the caller (so tests can mint expired - * tokens deterministically). */ -export async function signCredential( - payload: CredentialClaims, - key: CredentialKeyMaterial -): Promise<string> { - const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; - const header = { alg: ALG, typ: TYP, kid }; - const head = jsonEncode(header); - const body = jsonEncode(payload); - const signingInput = `${head}.${body}`; - const privateKey = await importPrivate(key.privateKey); - const sig = await crypto.subtle.sign( - { name: "ECDSA", hash: "SHA-256" }, - privateKey, - enc.encode(signingInput) - ); - return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; -} - -/** Issue a credential using the current wall-clock for iat/exp. */ -export async function issueCredential( - args: Omit<CredentialClaims, "iat" | "exp"> & { ttlMs: number }, - key: CredentialKeyMaterial -): Promise<{ credential: string; expiresAt: number }> { - const now = Math.floor(Date.now() / 1000); - const expSec = now + Math.floor(args.ttlMs / 1000); - const claims: CredentialClaims = { - iss: args.iss, - sub: args.sub, - space: args.space, - scope: args.scope, - iat: now, - exp: expSec, - }; - const credential = await signCredential(claims, key); - return { credential, expiresAt: expSec * 1000 }; -} - -export type VerifyOk = { ok: true; claims: CredentialClaims }; -export type VerifyErr = { - ok: false; - reason: - | "malformed" - | "bad-alg" - | "bad-signature" - | "expired" - | "not-yet-valid" - | "wrong-space" - | "wrong-scope" - | "unknown-issuer"; -}; - -export interface VerifyOptions { - /** Optional: when set, rejects credentials whose `space` claim differs. - * Omit when verifying in middleware where the target space isn't known - * yet — handlers can do the match themselves against the verified - * claims. */ - expectedSpace?: string; - /** Optional: required scope (e.g. "rw" rejects read-only credentials on writes). */ - requiredScope?: CredentialScope; - /** Resolve a verification key for `iss`. If null, verification fails with - * unknown-issuer. */ - resolveKey: (iss: string, kid: string | undefined) => Promise<JsonWebKey | null>; - /** Time provider for tests. Returns ms since epoch. */ - now?: () => number; -} - -export async function verifyCredential( - jwt: string, - opts: VerifyOptions -): Promise<VerifyOk | VerifyErr> { - const parts = jwt.split("."); - if (parts.length !== 3) return { ok: false, reason: "malformed" }; - const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; - - let header: { alg?: string; typ?: string; kid?: string }; - let claims: CredentialClaims; - try { - header = jsonDecode(headSeg); - claims = jsonDecode(bodySeg); - } catch { - return { ok: false, reason: "malformed" }; - } - if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; - if (opts.expectedSpace !== undefined && claims.space !== opts.expectedSpace) { - return { ok: false, reason: "wrong-space" }; - } - if (opts.requiredScope === "rw" && claims.scope !== "rw") { - return { ok: false, reason: "wrong-scope" }; - } - - const nowMs = (opts.now ?? Date.now)(); - const nowSec = Math.floor(nowMs / 1000); - if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; - if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; - - const jwk = await opts.resolveKey(claims.iss, header.kid); - if (!jwk) return { ok: false, reason: "unknown-issuer" }; - - const publicKey = await importPublic(jwk); - const sigBytes = base64urlDecode(sigSeg); - const signingInput = `${headSeg}.${bodySeg}`; - const valid = await crypto.subtle.verify( - { name: "ECDSA", hash: "SHA-256" }, - publicKey, - sigBytes, - enc.encode(signingInput) - ); - if (!valid) return { ok: false, reason: "bad-signature" }; - return { ok: true, claims }; -} - -/** Header reader for handlers that want to peek at `iss` before resolving the - * key (e.g. to short-circuit DID-doc fetches for the local authority). */ -export function decodeUnverifiedClaims(jwt: string): CredentialClaims | null { - const parts = jwt.split("."); - if (parts.length !== 3) return null; - try { - return jsonDecode<CredentialClaims>(parts[1]!); - } catch { - return null; - } -} - -/** Verifier interface consumed by the record host. The record host doesn't - * care HOW credentials get verified — it only cares whether a given JWT is - * valid. Phase 3 ships an in-process verifier that knows the local - * authority's public key; phase 4 adds a binding-resolving verifier that - * consults PDS records / DID docs. */ -export interface CredentialVerifier { - /** Verify a credential's signature, expiry, and `not-before` window. Does - * NOT enforce a space match — handlers do that against the request URI. */ - verify(jwt: string): Promise<VerifyOk | VerifyErr>; -} - -/** In-process verifier for the simple deployment: the authority and record - * host run in one process and the record host has direct access to the - * authority's public key. Rejects any credential whose `iss` isn't the - * configured authority. Phase 4 has a more general - * {@link createBindingCredentialVerifier} that does proper binding lookup. */ -export function createInProcessVerifier(args: { - authorityDid: string; - publicKey: JsonWebKey; -}): CredentialVerifier { - return { - verify(jwt) { - return verifyCredential(jwt, { - resolveKey: async (iss) => (iss === args.authorityDid ? args.publicKey : null), - }); - }, - }; -} - -/** Verifier composed of a {@link BindingResolver} (which DID is authorized - * to issue for this space?) and a {@link KeyResolver} (what's that DID's - * public key?). This is the production-shape verifier — phase 4's main - * contribution. - * - * Verification flow: - * 1. Decode the JWT's claims (no signature check yet). - * 2. Ask the binding resolver: who's authorized for `claims.space`? - * 3. Confirm `claims.iss === authorizedDid`. - * 4. Ask the key resolver for that DID's verification key. - * 5. Verify signature + expiry + scope match. - */ -export function createBindingCredentialVerifier(args: { - bindings: import("./binding").BindingResolver; - keys: import("./binding").KeyResolver; -}): CredentialVerifier { - return { - async verify(jwt) { - const peek = decodeUnverifiedClaims(jwt); - if (!peek) return { ok: false, reason: "malformed" }; - const authorizedDid = await args.bindings.resolveAuthority(peek.space); - if (!authorizedDid) return { ok: false, reason: "unknown-issuer" }; - if (peek.iss !== authorizedDid) return { ok: false, reason: "unknown-issuer" }; - return verifyCredential(jwt, { - resolveKey: (iss, kid) => args.keys.resolveKey(iss, kid), - }); - }, - }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/in-process.ts b/packages/contrail/src/core/spaces/in-process.ts index ec1d7a1..1129419 100644 --- a/packages/contrail/src/core/spaces/in-process.ts +++ b/packages/contrail/src/core/spaces/in-process.ts @@ -1,34 +1 @@ -/** In-process auth marker. - * - * For same-module callers (e.g. a SvelteKit worker that imports contrail and - * dispatches requests directly to the handler), service-auth JWTs are pure - * overhead: no network boundary is crossed, so there's nothing for the JWT to - * protect against. Instead, the caller tags the `Request` with a principal via - * a module-private WeakMap, and the auth middleware reads it back. - * - * Security note: this is unforgeable from outside the module because - * - WeakMap keys are `Request` object identities, not serialized data; - * - no HTTP request crossing a network boundary can reach into this map; - * - exploiting it requires code execution inside the same isolate, at - * which point auth is already game over. - * - * This is the strongest auth adapter contrail offers — it has no secret to - * leak. */ - -export interface InProcessPrincipal { - did: string; -} - -const PRINCIPALS = new WeakMap<Request, InProcessPrincipal>(); - -/** Tag a Request with an in-process principal. The returned Request is the - * same reference; the return value is for ergonomics. */ -export function markInProcess(req: Request, did: string): Request { - PRINCIPALS.set(req, { did }); - return req; -} - -/** Read the in-process principal for a Request, or null if unmarked. */ -export function readInProcess(req: Request): InProcessPrincipal | null { - return PRINCIPALS.get(req) ?? null; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/tid.ts b/packages/contrail/src/core/spaces/tid.ts index c419090..1129419 100644 --- a/packages/contrail/src/core/spaces/tid.ts +++ b/packages/contrail/src/core/spaces/tid.ts @@ -1,20 +1 @@ -const B32_CHARSET = "234567abcdefghijklmnopqrstuvwxyz"; - -let lastTimestamp = 0; -let clockId = Math.floor(Math.random() * 1024); - -/** Generate an atproto TID: 13-char base32-sortable (timestamp-ordered). */ -export function nextTid(): string { - let now = Date.now() * 1000; - if (now <= lastTimestamp) now = lastTimestamp + 1; - lastTimestamp = now; - - const n = BigInt(now) * 1024n + BigInt(clockId); - let s = ""; - let v = n; - for (let i = 0; i < 13; i++) { - s = B32_CHARSET[Number(v & 31n)] + s; - v >>= 5n; - } - return s; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 6561773..1129419 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -1,273 +1 @@ -import type { Database } from "../types"; -import type { DidDocumentResolver } from "@atcute/identity-resolver"; -import type { BlobAdapter } from "./blob-adapter"; -import type { CredentialKeyMaterial } from "./credentials"; - -export type AppPolicyMode = "allow" | "deny"; - -export interface AppPolicy { - mode: AppPolicyMode; - apps: string[]; -} - -export interface SpacesBlobsConfig { - /** Bytes backend (R2, S3, in-memory, …). */ - adapter: BlobAdapter; - /** Max blob size in bytes. Defaults to 2 MiB. */ - maxSize?: number; - /** MIME allowlist. If set, only these content types are accepted. */ - accept?: string[]; - /** Orphan blobs (those with no referencing record) are kept this long before - * GC can delete them, to allow upload-then-putRecord flows. - * Defaults to 24 hours. */ - gcOrphanAfterMs?: number; -} - -export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; -export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; - -/** Default credential lifetime. The rough spec calls for 2–4h; we pick the - * lower bound so revocation (kicked-from-space) is observable within 2h. */ -export const DEFAULT_CREDENTIAL_TTL_MS = 2 * 60 * 60 * 1000; - -/** Configuration for the **space authority** role: holds the member list, - * signs credentials, and gates space-management operations. In a fully-split - * deployment, the authority can run in a different process (or even a - * different operator) than the record host. */ -export interface AuthorityConfig { - /** NSID that identifies the kind of space this authority hosts, - * e.g. "tools.atmo.event.space". */ - type: string; - /** Service DID that service-auth tokens must target (aud claim) AND that - * signs credentials it issues (`iss` claim on emitted JWTs). */ - serviceDid: string; - /** Default app policy applied to new spaces. */ - defaultAppPolicy?: AppPolicy; - /** DID document resolver for service-auth JWT verification. - * Defaults to a composite PLC + did:web resolver if omitted. */ - resolver?: DidDocumentResolver; - /** Signing key material for issuing space credentials. When omitted, - * `<ns>.space.getCredential` returns 501 NotImplemented and the record - * host's credential-verifying middleware can't be wired up. */ - signing?: CredentialKeyMaterial; - /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ - credentialTtlMs?: number; -} - -/** Configuration for the **record host** role: stores per-space records and - * blobs and serves reads. Verifies space credentials (later phases) on - * incoming traffic. */ -export interface RecordHostConfig { - /** Blob-upload backend. When omitted, blob XRPCs are not exposed. */ - blobs?: SpacesBlobsConfig; -} - -/** Spaces config — host an authority, a record host, or both. - * Today both run in one process and most deployments will set both; the - * shape is split now so phase 5 can run them independently without churning - * every consumer's config. */ -export interface SpacesConfig { - /** Space-authority config — member list, credentials (later), space - * management. Required for any space to exist. */ - authority?: AuthorityConfig; - /** Record-host config — record + blob storage. Required for records to be - * written/read on this deployment. */ - recordHost?: RecordHostConfig; -} - -export interface SpaceRow { - uri: string; - ownerDid: string; - type: string; - key: string; - serviceDid: string; - appPolicyRef: string | null; - appPolicy: AppPolicy | null; - createdAt: number; - deletedAt: number | null; -} - -export interface SpaceMemberRow { - spaceUri: string; - did: string; - addedAt: number; - addedBy: string | null; -} - -export interface StoredRecord { - spaceUri: string; - collection: string; - authorDid: string; - rkey: string; - cid: string | null; - record: Record<string, unknown>; - createdAt: number; -} - -export interface ListOptions { - byUser?: string; - cursor?: string; - limit?: number; -} - -export interface ListResult { - records: StoredRecord[]; - cursor?: string; -} - -export interface ListSpacesOptions { - type?: string; - ownerDid?: string; - memberDid?: string; - limit?: number; - cursor?: string; -} - -export interface CollectionCount { - collection: string; - count: number; -} - -/** What a token holder can do with this invite. - * - `'join'`: must be redeemed while signed in; redeemer becomes a member. - * - `'read'`: bearer-only — token itself grants read access to the space; cannot be redeemed. - * - `'read-join'`: both — anonymous holders read; signed-in holders may also redeem to join. */ -export type InviteKind = "join" | "read" | "read-join"; - -export interface InviteRow { - tokenHash: string; - spaceUri: string; - kind: InviteKind; - expiresAt: number | null; - maxUses: number | null; - usedCount: number; - createdBy: string; - createdAt: number; - revokedAt: number | null; - note: string | null; -} - -export interface CreateInviteInput { - spaceUri: string; - tokenHash: string; - kind: InviteKind; - expiresAt: number | null; - maxUses: number | null; - createdBy: string; - note: string | null; -} - -export interface RedeemInviteResult { - spaceUri: string; -} - -export interface BlobMetaRow { - spaceUri: string; - cid: string; - mimeType: string; - size: number; - authorDid: string; - createdAt: number; -} - -/** Local cache on the record host: this space is accepted here, and `authority_did` - * is the DID authorized to sign credentials for it. Populated via the - * `recordHost.enroll` endpoint or auto-populated by the authority's - * createSpace when both roles run in one process. */ -export interface EnrollmentRow { - spaceUri: string; - authorityDid: string; - enrolledAt: number; - enrolledBy: string; -} - -export interface ListBlobsOptions { - byUser?: string; - cursor?: string; - limit?: number; -} - -export interface ListBlobsResult { - blobs: BlobMetaRow[]; - cursor?: string; -} - -/** **Space authority** interface — owner of the space's ACL state and - * (eventually) credential issuer. Holds the member list, manages invites, - * governs space lifecycle and app policy. Does NOT touch records or blobs. - * - * In a fully-split deployment this is a separate service; today the - * HostedAdapter implements both this and {@link RecordHost} against one DB. */ -export interface SpaceAuthority { - // Space lifecycle - createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow>; - getSpace(spaceUri: string): Promise<SpaceRow | null>; - listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }>; - deleteSpace(spaceUri: string): Promise<void>; - updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void>; - - // Members - addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void>; - removeMember(spaceUri: string, did: string): Promise<void>; - getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null>; - listMembers(spaceUri: string): Promise<SpaceMemberRow[]>; - /** Bulk-apply a membership diff. Used only by the community module's reconciler; - * not exposed as an XRPC endpoint. */ - applyMembershipDiff( - spaceUri: string, - adds: string[], - removes: string[], - addedBy: string | null - ): Promise<void>; - - // Invites (token primitive — issued by the authority, scoped to a space) - createInvite(input: CreateInviteInput): Promise<InviteRow>; - listInvites(spaceUri: string, options?: { includeRevoked?: boolean }): Promise<InviteRow[]>; - revokeInvite(tokenHash: string): Promise<boolean>; - /** Look up an invite without consuming it. Used to validate read-token bearer access. */ - getInvite(tokenHash: string): Promise<InviteRow | null>; - /** Atomically mark a join-capable invite as used. Returns the row if usable - * (kind allows join, not expired/revoked/exhausted), null otherwise. */ - redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null>; -} - -/** **Record host** interface — stores records and blobs for a space, plus - * the local enrollment table that decides which spaces this host accepts. - * - * Trust model: the host trusts whatever credential the authority signs, so - * long as the authority is the one named in the local enrollment for this - * space. Enrollment is the consent step — the host owner agrees to spend - * storage on a given space, scoped to a specific authority. */ -export interface RecordHost { - // Enrollment - enroll(input: EnrollmentRow): Promise<void>; - getEnrollment(spaceUri: string): Promise<EnrollmentRow | null>; - listEnrollments(options?: { authorityDid?: string; limit?: number }): Promise<EnrollmentRow[]>; - removeEnrollment(spaceUri: string): Promise<void>; - - // Records - putRecord(record: StoredRecord): Promise<void>; - getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<StoredRecord | null>; - listRecords(spaceUri: string, collection: string, options?: ListOptions): Promise<ListResult>; - deleteRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<void>; - listCollections(spaceUri: string, options?: { byUser?: string }): Promise<CollectionCount[]>; - - // Blobs (metadata only; bytes live on BlobAdapter) - putBlobMeta(row: BlobMetaRow): Promise<void>; - getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null>; - listBlobMeta(spaceUri: string, options?: ListBlobsOptions): Promise<ListBlobsResult>; - deleteBlobMeta(spaceUri: string, cid: string): Promise<void>; - /** Find blob rows older than `cutoff` whose CIDs are not referenced in any - * record JSON in this space. Capped at `limit` to bound a single GC pass. */ - findOrphanBlobs(spaceUri: string, cutoff: number, limit: number): Promise<BlobMetaRow[]>; -} - -/** Combined adapter. Used internally where a single object satisfies both - * roles (today's HostedAdapter, the community reconciler, the realtime - * publishing wrapper). Phases 5+ replace consumers of this with two - * injected interfaces. */ -export type StorageAdapter = SpaceAuthority & RecordHost; - -export interface AdapterContext { - db: Database; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/uri.ts b/packages/contrail/src/core/spaces/uri.ts index e73ec21..1129419 100644 --- a/packages/contrail/src/core/spaces/uri.ts +++ b/packages/contrail/src/core/spaces/uri.ts @@ -1,37 +1 @@ -/** Centralized space URI construction / parsing. - * - * Permissioned spaces are addressed by (ownerDid, type, key) and use the - * `ats://` scheme — distinct from atproto record URIs (`at://`) so the two - * can't be confused at any layer (logs, params, dispatch). Tracks the rough - * spec at https://dholms.leaflet.pub/3mhj6bcqats2o. - * - * Record URIs inside a space are minted by authorDid for index purposes - * (`at://<authorDid>/<collection>/<rkey>`); the spec is explicitly undecided - * about authority (user vs space owner), so we don't expose those as a - * canonical record address — they're storage-internal. */ - -export interface SpaceUriParts { - ownerDid: string; - type: string; - key: string; -} - -/** Build a space URI from its three addressing components. */ -export function buildSpaceUri(parts: SpaceUriParts): string { - return `ats://${parts.ownerDid}/${parts.type}/${parts.key}`; -} - -/** Parse a space URI into its components, or null if malformed. */ -export function parseSpaceUri(uri: string): SpaceUriParts | null { - if (!uri.startsWith("ats://")) return null; - const rest = uri.slice("ats://".length); - const [ownerDid, type, key, ...extra] = rest.split("/"); - if (!ownerDid || !type || !key || extra.length > 0) return null; - return { ownerDid, type, key }; -} - -/** Build a record URI under a given author. Used only as a secondary index key - * inside storage — not a canonical address for permissioned records. */ -export function buildRecordUri(authorDid: string, collection: string, rkey: string): string { - return `at://${authorDid}/${collection}/${rkey}`; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index 3b9dc48..1129419 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -1,513 +1 @@ -import type { SqlDialect } from "./dialect"; - -// Database interface — D1 implements this natively -export interface Database { - prepare(sql: string): Statement; - batch(stmts: Statement[]): Promise<any[]>; - dialect?: SqlDialect; -} - -export interface Statement { - bind(...values: any[]): Statement; - run(): Promise<any>; - all<T = any>(): Promise<{ results: T[] }>; - first<T = any>(): Promise<T | null>; -} - -// Config types - -export interface QueryableField { - type?: "range"; -} - -export interface RelationConfig { - /** Short name of the child collection (a key in `collections`). */ - collection: string; - field?: string; - match?: "uri" | "did"; - groupBy?: string; - /** Enable materialized count columns on the parent. Defaults to true. */ - count?: boolean; - /** Count distinct values of a field (e.g. "did" for unique users) instead of total records. */ - countDistinct?: string; - /** Pre-resolved group mappings: shortName → full token (e.g. { going: "community.lexicon.calendar.rsvp#going" }). Auto-computed from groupBy if omitted. */ - groups?: Record<string, string>; -} - -/** A forward reference: this collection's records point at another collection. */ -export interface ReferenceConfig { - /** Short name of the target collection. */ - collection: string; - /** Field on this collection's records containing the target URI. */ - field: string; -} - -export type CustomQueryHandler = ( - db: Database, - params: URLSearchParams, - config: ContrailConfig -) => Promise<Response>; - -export interface RecordSource { - joins?: string; - conditions?: string[]; - params?: (string | number)[]; -} - -export type PipelineQueryHandler = ( - db: Database, - params: URLSearchParams, - config: ContrailConfig -) => Promise<RecordSource>; - -export interface FeedConfig { - /** Short name of the follow collection. */ - follow: string; - /** Short names of target collections to fan out to. */ - targets: string[]; - /** Max feed items per user (default: 200). Oldest items are pruned after backfill. */ - maxItems?: number; -} - -export const DEFAULT_FEED_MAX_ITEMS = 200; - -export type CollectionMethod = "listRecords" | "getRecord"; -export const DEFAULT_COLLECTION_METHODS: CollectionMethod[] = [ - "listRecords", - "getRecord", -]; - -export interface CollectionConfig { - /** Full NSID of the record type this collection indexes. */ - collection: string; - /** Include this collection in Jetstream ingest / discovery (default true). - * Set false for dependent collections (auto-fetched on demand). */ - discover?: boolean; - queryable?: Record<string, QueryableField>; - relations?: Record<string, RelationConfig>; - /** Forward references: fields on this collection's records that point at another collection. */ - references?: Record<string, ReferenceConfig>; - queries?: Record<string, CustomQueryHandler>; - pipelineQueries?: Record<string, PipelineQueryHandler>; - /** FTS5 search fields. Provide an array of field names to enable full-text search. Omit or set to false to disable. */ - searchable?: string[] | false; - /** XRPC methods to emit. Defaults to ['listRecords', 'getRecord']. */ - methods?: CollectionMethod[]; - /** When spaces are enabled globally, emit a parallel spaces_records_<short> table - * so this collection can also live inside spaces. Defaults to true. */ - allowInSpaces?: boolean; -} - -export interface ProfileConfig { - /** Full NSID of the profile record type. */ - collection: string; - /** Short name used for table/endpoint naming. Defaults to the NSID's last segment. */ - shortName?: string; - rkey?: string; // defaults to "self" -} - -export const DEFAULT_PROFILES: ProfileConfig[] = [ - { collection: "app.bsky.actor.profile", shortName: "profile" }, -]; - -/** Normalize a profiles config entry (string or object) into ProfileConfig. */ -export function normalizeProfileConfig( - p: string | ProfileConfig -): ProfileConfig { - if (typeof p === "string") { - return { collection: p, shortName: deriveShortName(p) }; - } - return { ...p, shortName: p.shortName ?? deriveShortName(p.collection) }; -} - -/** Last NSID segment, used as fallback short name. */ -export function deriveShortName(nsid: string): string { - const parts = nsid.split("."); - return parts[parts.length - 1] ?? nsid; -} - -export const DEFAULT_JETSTREAMS = [ - "wss://jetstream1.us-east.bsky.network", -]; - -export const DEFAULT_RELAYS = [ - "https://relay1.us-east.bsky.network" -]; - -export interface Logger { - log(...args: any[]): void; - warn(...args: any[]): void; - error(...args: any[]): void; -} - -export interface ContrailConfig { - namespace: string; - /** Collections to index, keyed by short name. Short names become endpoint URL segments - * (`<namespace>.<short>.listRecords`) and table suffixes (`records_<short>`). */ - collections: Record<string, CollectionConfig>; - profiles?: (string | ProfileConfig)[]; - relays?: string[]; - jetstreams?: string[]; - feeds?: Record<string, FeedConfig>; - logger?: Logger; - /** Expose the notifyOfUpdate HTTP endpoint. Off by default. - * Set to `true` for open access, or a string to require `Authorization: Bearer <secret>`. */ - notify?: boolean | string; - /** Permissioned spaces configuration. When set, the service exposes space XRPCs. */ - spaces?: import("./spaces/types").SpacesConfig; - /** Community module configuration. Typed by the community package via - * declaration merging — contrail core only knows it's "something the - * community package consumes." Set when wiring community via - * `createCommunityIntegration({ ... })`. Requires `spaces.authority`. */ - community?: unknown; - /** Realtime module configuration. When set, the service exposes ticket + SSE/WS - * subscribe XRPCs, and wraps the spaces adapter to publish events after writes. */ - realtime?: import("./realtime/types").RealtimeConfig; - /** Labels module configuration. When set, contrail subscribes to the - * configured labelers, indexes their labels into a single `labels` table, - * and hydrates `record.labels` onto `listRecords` / `getRecord` / profile - * responses gated by the caller's `atproto-accept-labelers` header. */ - labels?: import("./labels/types").LabelsConfig; - /** Customize the auto-generated `<namespace>.authFull` lexicon. */ - permissionSet?: PermissionSetConfig; -} - -/** Single entry in an atproto permission-set's `permissions` array. - * See https://atproto.com/guides/permission-sets for the full schema. */ -export type PermissionEntry = - | { type: "permission"; resource: "rpc"; lxm?: string[]; aud?: string; inheritAud?: boolean } - | { type: "permission"; resource: "repo"; collection?: string[] } - | { type: "permission"; resource: "blob"; accept?: string[]; maxSize?: number } - | { type: "permission"; resource: "account"; attr?: string[] } - | { type: "permission"; resource: string; [key: string]: unknown }; - -export interface PermissionSetConfig { - /** Shown on the OAuth consent screen. Defaults to the namespace. */ - title?: string; - /** Shown on the OAuth consent screen. Defaults to a generated description. */ - description?: string; - /** Extra permission entries appended after the auto-generated rpc entry — - * e.g. repo writes for collections your app needs the user to create, or - * blob permissions for uploads. */ - additional?: PermissionEntry[]; -} - -export interface ResolvedRelation { - /** Short name of the child collection. */ - collection: string; - groupBy: string; - groups: Record<string, string>; // shortName → full token value -} - -export interface ResolvedMaps { - queryable: Record<string, Record<string, QueryableField>>; - relations: Record<string, Record<string, ResolvedRelation>>; - /** Reverse map: full record NSID → short name. */ - nsidToShort: Record<string, string>; -} - -/** Config after resolveConfig() — has computed queryable/relation maps attached. */ -export interface ResolvedContrailConfig extends ContrailConfig { - _resolved: ResolvedMaps; -} - -/** - * Resolve config: apply defaults, auto-add profile collections, compute queryable maps. - */ -export function resolveConfig(config: ContrailConfig): ResolvedContrailConfig { - const profiles = (config.profiles ?? DEFAULT_PROFILES).map( - normalizeProfileConfig - ); - const collections = { ...config.collections }; - for (const p of profiles) { - const short = p.shortName!; - if (!collections[short]) { - collections[short] = { collection: p.collection, discover: false }; - } - } - - // Auto-add follow collections from feed configs as dependent collections if they're - // not already listed. Feed config already uses short names so nothing to resolve — - // but if the user forgot to declare the follow collection, we can't auto-add it without - // knowing its NSID. In that case we warn later via validateConfig. - - const base = { - ...config, - collections, - profiles, - jetstreams: config.jetstreams ?? DEFAULT_JETSTREAMS, - relays: config.relays ?? DEFAULT_RELAYS, - logger: config.logger ?? console, - }; - - return { - ...base, - _resolved: _resolveQueryableMaps(base), - }; -} - -function _resolveQueryableMaps(config: ContrailConfig): ResolvedMaps { - const queryable: Record<string, Record<string, QueryableField>> = {}; - const relations: Record<string, Record<string, ResolvedRelation>> = {}; - const nsidToShort: Record<string, string> = {}; - - for (const [short, colConfig] of Object.entries(config.collections)) { - nsidToShort[colConfig.collection] = short; - - if (colConfig.queryable) { - queryable[short] = colConfig.queryable; - } - - if (colConfig.relations) { - for (const [relName, rel] of Object.entries(colConfig.relations)) { - if (!rel.groupBy) continue; - const groups: Record<string, string> = rel.groups ? { ...rel.groups } : {}; - if (Object.keys(groups).length > 0) { - if (!relations[short]) relations[short] = {}; - relations[short][relName] = { - collection: rel.collection, - groupBy: rel.groupBy, - groups, - }; - } - } - } - } - - return { queryable, relations, nsidToShort }; -} - -export function getFeedFollowShortNames(config: ContrailConfig): string[] { - if (!config.feeds) return []; - return [...new Set(Object.values(config.feeds).map((f) => f.follow))]; -} - -/** Alias for getFeedFollowShortNames. */ -export const getFeedFollowCollections = getFeedFollowShortNames; - -// Record types - -export interface RecordRow { - uri: string; - did: string; - collection: string; // full NSID - rkey: string; - cid: string | null; - record: string | null; - time_us: number; - indexed_at: number; - /** Set when the row originates from a per-space table. Used by the - * pipeline/hydration/response layers to route child queries to the same - * space and tag the output. */ - space?: string; -} - -export interface IngestEvent { - uri: string; - did: string; - collection: string; // full NSID - rkey: string; - operation: "create" | "update" | "delete"; - cid: string | null; - record: string | null; - time_us: number; - indexed_at: number; -} - -// Validation - -const SAFE_FIELD_NAME = /^[a-zA-Z0-9_.]+$/; -const SAFE_SHORT_NAME = /^[a-zA-Z][a-zA-Z0-9]*$/; - -export function validateFieldName(field: string): string { - if (!SAFE_FIELD_NAME.test(field)) { - throw new Error(`Invalid field name: ${field}`); - } - return field; -} - -function validateShortName(short: string): void { - if (!SAFE_SHORT_NAME.test(short)) { - throw new Error( - `Invalid collection short name: "${short}". Must be alphanumeric, starting with a letter.` - ); - } -} - -export function validateConfig(config: ContrailConfig): void { - const shortNames = new Set<string>(); - for (const [short, colConfig] of Object.entries(config.collections)) { - validateShortName(short); - if (shortNames.has(short)) { - throw new Error(`Duplicate collection short name: ${short}`); - } - shortNames.add(short); - - if (!colConfig.collection) { - throw new Error(`Collection "${short}" is missing required 'collection' field (NSID)`); - } - - for (const field of Object.keys(colConfig.queryable ?? {})) { - validateFieldName(field); - } - for (const [, rel] of Object.entries(colConfig.relations ?? {})) { - if (rel.field) validateFieldName(rel.field); - if (rel.groupBy) validateFieldName(rel.groupBy); - if (rel.countDistinct) validateFieldName(rel.countDistinct); - if (!config.collections[rel.collection]) { - throw new Error( - `Relation in "${short}" references unknown collection short name "${rel.collection}"` - ); - } - } - for (const [, ref] of Object.entries(colConfig.references ?? {})) { - validateFieldName(ref.field); - if (!config.collections[ref.collection]) { - throw new Error( - `Reference in "${short}" references unknown collection short name "${ref.collection}"` - ); - } - } - if (Array.isArray(colConfig.searchable)) { - for (const field of colConfig.searchable) { - validateFieldName(field); - } - } - } - - if (config.feeds) { - for (const [feedName, feed] of Object.entries(config.feeds)) { - if (!config.collections[feed.follow]) { - throw new Error( - `Feed "${feedName}" references unknown follow collection "${feed.follow}"` - ); - } - for (const target of feed.targets) { - if (!config.collections[target]) { - throw new Error( - `Feed "${feedName}" references unknown target collection "${target}"` - ); - } - } - } - } - - if (config.community && !config.spaces?.authority) { - throw new Error( - "Invalid config: `community` requires `spaces.authority`. Community-owned spaces reuse the spaces storage adapter." - ); - } -} - -// Helpers - -export function getNestedValue(obj: any, path: string): any { - let current = obj; - for (const key of path.split(".")) { - if (current == null) return undefined; - current = current[key]; - } - return current; -} - -const DEFAULT_RELATION_FIELD = "subject.uri"; - -export function getRelationField(rel: RelationConfig): string { - return rel.field ?? DEFAULT_RELATION_FIELD; -} - -/** Sanitize a short name for use in SQL identifiers (already-validated; kept for paranoia). */ -function sanitizeIdentifier(name: string): string { - return name.replace(/[^a-zA-Z0-9]/g, "_"); -} - -/** Total-count column name for a relation targeting the given short name. */ -export function countColumnName(childShortName: string): string { - return "count_" + sanitizeIdentifier(childShortName); -} - -/** Grouped-count column name: `count_<child-short>_<groupKey>`. */ -export function groupedCountColumnName( - childShortName: string, - groupKey: string -): string { - return `count_${sanitizeIdentifier(childShortName)}_${sanitizeIdentifier(groupKey)}`; -} - -/** Table name for a collection's records. */ -export function recordsTableName(shortName: string): string { - return "records_" + sanitizeIdentifier(shortName); -} - -/** Table name for a collection's records inside spaces. */ -export function spacesRecordsTableName(shortName: string): string { - return "spaces_records_" + sanitizeIdentifier(shortName); -} - -/** All collection short names. */ -export function getCollectionShortNames(config: ContrailConfig): string[] { - return Object.keys(config.collections); -} - -/** Alias: collection short names (same as getCollectionShortNames). */ -export const getCollectionNames = getCollectionShortNames; - -/** All indexed record NSIDs (what Jetstream filters on). */ -export function getCollectionNsids(config: ContrailConfig): string[] { - return Object.values(config.collections).map((c) => c.collection); -} - -export function getDependentShortNames(config: ContrailConfig): string[] { - return Object.entries(config.collections) - .filter(([, c]) => c.discover === false) - .map(([name]) => name); -} - -export function getDiscoverableShortNames(config: ContrailConfig): string[] { - return Object.entries(config.collections) - .filter(([, c]) => c.discover !== false) - .map(([name]) => name); -} - -/** Aliases for readability elsewhere. These return short names (new semantic). */ -export const getDependentCollections = getDependentShortNames; -export const getDiscoverableCollections = getDiscoverableShortNames; - -/** Short names of collections the user declared with `discover !== false`, mapped to NSIDs. */ -export function getDiscoverableNsids(config: ContrailConfig): string[] { - return Object.values(config.collections) - .filter((c) => c.discover !== false) - .map((c) => c.collection); -} - -export function getDependentNsids(config: ContrailConfig): string[] { - return Object.values(config.collections) - .filter((c) => c.discover === false) - .map((c) => c.collection); -} - -/** Short name for a record NSID, if known. */ -export function shortNameForNsid( - config: ContrailConfig, - nsid: string -): string | undefined { - const resolved = (config as ResolvedContrailConfig)._resolved; - if (resolved?.nsidToShort) return resolved.nsidToShort[nsid]; - for (const [short, c] of Object.entries(config.collections)) { - if (c.collection === nsid) return short; - } - return undefined; -} - -/** Full NSID for a collection short name. */ -export function nsidForShortName( - config: ContrailConfig, - short: string -): string | undefined { - return config.collections[short]?.collection; -} - -/** The methods a collection should expose via XRPC. */ -export function getCollectionMethods(cfg: CollectionConfig): CollectionMethod[] { - return cfg.methods ?? DEFAULT_COLLECTION_METHODS; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 04f1a15..0738fd3 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -1,4 +1,7 @@ import { defineConfig } from "vitest/config"; +import path from "node:path"; + +const baseSrc = path.resolve(__dirname, "../contrail-base/src"); export default defineConfig({ test: { @@ -6,4 +9,13 @@ export default defineConfig({ // PostgreSQL tests share a single database and cannot run in parallel fileParallelism: false, }, + resolve: { + alias: { + // Resolve workspace-internal contrail-base subpaths to source so tests + // don't run through the dist (where tsup drops `node:` prefixes). + "@atmo-dev/contrail-base/sqlite": path.join(baseSrc, "adapters/sqlite.ts"), + "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), + "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), + }, + }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48ffc2e..01c655a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -376,6 +376,9 @@ importers: '@atcute/xrpc-server': specifier: ^0.1.12 version: 0.1.12 + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base cac: specifier: ^7.0.0 version: 7.0.0 @@ -411,6 +414,49 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-base: + dependencies: + '@atcute/atproto': + specifier: ^3.1.10 + version: 3.1.11 + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atcute/client': + specifier: ^4.2.1 + version: 4.2.1 + '@atcute/identity': + specifier: ^1.1.4 + version: 1.1.4 + '@atcute/identity-resolver': + specifier: ^1.2.2 + version: 1.2.2(@atcute/identity@1.1.4) + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atcute/xrpc-server': + specifier: ^0.1.12 + version: 0.1.12 + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + '@types/node': + specifier: ^25.5.0 + version: 25.6.0 + '@types/pg': + specifier: ^8.20.0 + version: 8.20.0 + pg: + specifier: ^8.20.0 + version: 8.20.0 + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/contrail-community: dependencies: '@atcute/atproto': @@ -434,6 +480,9 @@ importers: '@atmo-dev/contrail': specifier: workspace:* version: link:../contrail + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base hono: specifier: ^4.12.8 version: 4.12.15 -- 2.51.2 From edf66ed0cf948b77d63648f67074a324b084956d Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 01:35:24 +0200 Subject: [PATCH 09/25] phase 7a ^^ --- packages/contrail-authority/package.json | 44 +++ packages/contrail-authority/src/adapter.ts | 317 +++++++++++++++++ packages/contrail-authority/src/index.ts | 19 ++ packages/contrail-authority/src/schema.ts | 52 +++ .../contrail-authority/tsconfig.build.json | 7 + packages/contrail-authority/tsconfig.json | 7 + packages/contrail-authority/tsup.config.ts | 11 + packages/contrail/package.json | 1 + packages/contrail/src/core/spaces/adapter.ts | 318 +----------------- packages/contrail/vitest.config.ts | 6 +- pnpm-lock.yaml | 25 ++ 11 files changed, 505 insertions(+), 302 deletions(-) create mode 100644 packages/contrail-authority/package.json create mode 100644 packages/contrail-authority/src/adapter.ts create mode 100644 packages/contrail-authority/src/index.ts create mode 100644 packages/contrail-authority/src/schema.ts create mode 100644 packages/contrail-authority/tsconfig.build.json create mode 100644 packages/contrail-authority/tsconfig.json create mode 100644 packages/contrail-authority/tsup.config.ts diff --git a/packages/contrail-authority/package.json b/packages/contrail-authority/package.json new file mode 100644 index 0000000..07c0477 --- /dev/null +++ b/packages/contrail-authority/package.json @@ -0,0 +1,44 @@ +{ + "name": "@atmo-dev/contrail-authority", + "version": "0.6.0", + "description": "Default space-authority implementation for contrail — member list, invites, app policy, credential issuance. Contrail's binary-membership ACL flavor; for ladder-style access levels see @atmo-dev/contrail-community.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-authority" + }, + "keywords": [ + "atproto", + "contrail" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/cid": "^2.4.1", + "@atcute/lexicons": "^1.2.9", + "@atmo-dev/contrail-base": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "license": "MIT" +} diff --git a/packages/contrail-authority/src/adapter.ts b/packages/contrail-authority/src/adapter.ts new file mode 100644 index 0000000..66ffdfb --- /dev/null +++ b/packages/contrail-authority/src/adapter.ts @@ -0,0 +1,317 @@ +/** Default {@link SpaceAuthority} implementation backed by a Database. + * + * Owns three tables (`spaces`, `spaces_members`, `spaces_invites`) and + * exposes the authority surface: space lifecycle, member list, invite + * storage, app-policy management. + * + * Designed for inheritance — fields are `protected` so a record-host + * adapter (or, transitionally, contrail's all-in-one HostedAdapter) can + * extend this class to add its own methods without re-implementing the + * authority side. */ + +import type { + ContrailConfig, + Database, + SpaceAuthority, + AppPolicy, + CreateInviteInput, + InviteKind, + InviteRow, + ListSpacesOptions, + SpaceMemberRow, + SpaceRow, +} from "@atmo-dev/contrail-base"; + +export function parseJson<T>(value: unknown): T | null { + if (value == null) return null; + if (typeof value === "string") { + try { + return JSON.parse(value) as T; + } catch { + return null; + } + } + return value as T; +} + +export function toNum(v: unknown): number { + return typeof v === "string" ? Number(v) : (v as number); +} + +export function mapSpaceRow(row: any): SpaceRow { + return { + uri: row.uri, + ownerDid: row.owner_did, + type: row.type, + key: row.key, + serviceDid: row.service_did, + appPolicyRef: row.app_policy_ref ?? null, + appPolicy: parseJson<AppPolicy>(row.app_policy), + createdAt: toNum(row.created_at), + deletedAt: row.deleted_at == null ? null : toNum(row.deleted_at), + }; +} + +export function mapMemberRow(row: any): SpaceMemberRow { + return { + spaceUri: row.space_uri, + did: row.did, + addedAt: toNum(row.added_at), + addedBy: row.added_by ?? null, + }; +} + +export function mapInviteRow(row: any): InviteRow { + return { + tokenHash: row.token_hash, + spaceUri: row.space_uri, + kind: (row.kind ?? "join") as InviteKind, + expiresAt: row.expires_at == null ? null : toNum(row.expires_at), + maxUses: row.max_uses == null ? null : Number(row.max_uses), + usedCount: Number(row.used_count), + createdBy: row.created_by, + createdAt: toNum(row.created_at), + revokedAt: row.revoked_at == null ? null : toNum(row.revoked_at), + note: row.note ?? null, + }; +} + +export class HostedAuthorityAdapter implements SpaceAuthority { + constructor( + protected readonly db: Database, + protected readonly config?: ContrailConfig + ) {} + + async createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow> { + const now = Date.now(); + await this.db + .prepare( + `INSERT INTO spaces (uri, owner_did, type, key, service_did, app_policy_ref, app_policy, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ) + .bind( + space.uri, + space.ownerDid, + space.type, + space.key, + space.serviceDid, + space.appPolicyRef, + space.appPolicy ? JSON.stringify(space.appPolicy) : null, + now + ) + .run(); + return { ...space, createdAt: now, deletedAt: null }; + } + + async getSpace(spaceUri: string): Promise<SpaceRow | null> { + const row = await this.db + .prepare(`SELECT * FROM spaces WHERE uri = ? AND deleted_at IS NULL`) + .bind(spaceUri) + .first<any>(); + return row ? mapSpaceRow(row) : null; + } + + async listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }> { + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["s.deleted_at IS NULL"]; + const params: any[] = []; + let join = ""; + + if (options.type) { + clauses.push("s.type = ?"); + params.push(options.type); + } + if (options.ownerDid) { + clauses.push("s.owner_did = ?"); + params.push(options.ownerDid); + } + if (options.memberDid) { + join = "JOIN spaces_members m ON m.space_uri = s.uri"; + clauses.push("m.did = ?"); + params.push(options.memberDid); + } + if (options.cursor) { + clauses.push("s.created_at < ?"); + params.push(Number(options.cursor)); + } + + const sql = `SELECT s.* FROM spaces s ${join} + WHERE ${clauses.join(" AND ")} + ORDER BY s.created_at DESC + LIMIT ?`; + params.push(limit + 1); + + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const spaces = results.map(mapSpaceRow); + let cursor: string | undefined; + if (spaces.length > limit) { + const next = spaces.pop()!; + cursor = String(next.createdAt); + } + return { spaces, cursor }; + } + + async deleteSpace(spaceUri: string): Promise<void> { + await this.db + .prepare(`UPDATE spaces SET deleted_at = ? WHERE uri = ?`) + .bind(Date.now(), spaceUri) + .run(); + } + + async updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void> { + await this.db + .prepare(`UPDATE spaces SET app_policy = ? WHERE uri = ?`) + .bind(JSON.stringify(appPolicy), spaceUri) + .run(); + } + + async addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void> { + await this.db + .prepare( + `INSERT INTO spaces_members (space_uri, did, added_at, added_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri, did) DO NOTHING` + ) + .bind(spaceUri, did, Date.now(), addedBy) + .run(); + } + + async removeMember(spaceUri: string, did: string): Promise<void> { + await this.db + .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) + .bind(spaceUri, did) + .run(); + } + + async getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null> { + const row = await this.db + .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? AND did = ?`) + .bind(spaceUri, did) + .first<any>(); + return row ? mapMemberRow(row) : null; + } + + async listMembers(spaceUri: string): Promise<SpaceMemberRow[]> { + const { results } = await this.db + .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? ORDER BY added_at ASC`) + .bind(spaceUri) + .all<any>(); + return results.map(mapMemberRow); + } + + async applyMembershipDiff( + spaceUri: string, + adds: string[], + removes: string[], + addedBy: string | null + ): Promise<void> { + const now = Date.now(); + const stmts: any[] = []; + for (const did of adds) { + stmts.push( + this.db + .prepare( + `INSERT INTO spaces_members (space_uri, did, added_at, added_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri, did) DO NOTHING` + ) + .bind(spaceUri, did, now, addedBy) + ); + } + for (const did of removes) { + stmts.push( + this.db + .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) + .bind(spaceUri, did) + ); + } + if (stmts.length > 0) { + await this.db.batch(stmts); + } + } + + async createInvite(input: CreateInviteInput): Promise<InviteRow> { + const now = Date.now(); + await this.db + .prepare( + `INSERT INTO spaces_invites (token_hash, space_uri, kind, expires_at, max_uses, used_count, created_by, created_at, note) + VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?)` + ) + .bind( + input.tokenHash, + input.spaceUri, + input.kind, + input.expiresAt, + input.maxUses, + input.createdBy, + now, + input.note + ) + .run(); + return { + tokenHash: input.tokenHash, + spaceUri: input.spaceUri, + kind: input.kind, + expiresAt: input.expiresAt, + maxUses: input.maxUses, + usedCount: 0, + createdBy: input.createdBy, + createdAt: now, + revokedAt: null, + note: input.note, + }; + } + + async getInvite(tokenHash: string): Promise<InviteRow | null> { + const row = await this.db + .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) + .bind(tokenHash) + .first<any>(); + return row ? mapInviteRow(row) : null; + } + + async listInvites( + spaceUri: string, + options: { includeRevoked?: boolean } = {} + ): Promise<InviteRow[]> { + const sql = options.includeRevoked + ? `SELECT * FROM spaces_invites WHERE space_uri = ? ORDER BY created_at DESC` + : `SELECT * FROM spaces_invites WHERE space_uri = ? AND revoked_at IS NULL ORDER BY created_at DESC`; + const { results } = await this.db.prepare(sql).bind(spaceUri).all<any>(); + return results.map(mapInviteRow); + } + + async revokeInvite(tokenHash: string): Promise<boolean> { + const res = await this.db + .prepare(`UPDATE spaces_invites SET revoked_at = ? WHERE token_hash = ? AND revoked_at IS NULL`) + .bind(Date.now(), tokenHash) + .run(); + const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; + return Number(changes) > 0; + } + + async redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null> { + // Atomic: increment used_count only if the invite is usable right now AND + // its kind allows redemption (read-only tokens cannot be consumed for membership). + const res = await this.db + .prepare( + `UPDATE spaces_invites + SET used_count = used_count + 1 + WHERE token_hash = ? + AND kind IN ('join', 'read-join') + AND revoked_at IS NULL + AND (expires_at IS NULL OR expires_at > ?) + AND (max_uses IS NULL OR used_count < max_uses)` + ) + .bind(tokenHash, now) + .run(); + const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; + if (Number(changes) === 0) return null; + + const row = await this.db + .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) + .bind(tokenHash) + .first<any>(); + return row ? mapInviteRow(row) : null; + } +} diff --git a/packages/contrail-authority/src/index.ts b/packages/contrail-authority/src/index.ts new file mode 100644 index 0000000..638b762 --- /dev/null +++ b/packages/contrail-authority/src/index.ts @@ -0,0 +1,19 @@ +/** @atmo-dev/contrail-authority — default space-authority implementation. + * + * Owns the authority-side adapter (member list, invites, app policy, space + * lifecycle) and DDL. Route registration currently lives in + * @atmo-dev/contrail and will move here in a subsequent extraction pass. */ + +export { + HostedAuthorityAdapter, + parseJson, + toNum, + mapSpaceRow, + mapMemberRow, + mapInviteRow, +} from "./adapter"; + +export { + buildAuthoritySchema, + applyAuthoritySchema, +} from "./schema"; diff --git a/packages/contrail-authority/src/schema.ts b/packages/contrail-authority/src/schema.ts new file mode 100644 index 0000000..e9f19e9 --- /dev/null +++ b/packages/contrail-authority/src/schema.ts @@ -0,0 +1,52 @@ +/** Authority-side DDL: `spaces`, `spaces_members`, `spaces_invites`. */ + +import type { Database, SqlDialect } from "@atmo-dev/contrail-base"; +import { getDialect } from "@atmo-dev/contrail-base"; + +export function buildAuthoritySchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS spaces ( + uri TEXT PRIMARY KEY, + owner_did TEXT NOT NULL, + type TEXT NOT NULL, + key TEXT NOT NULL, + service_did TEXT NOT NULL, + app_policy_ref TEXT, + app_policy ${dialect.recordColumnType}, + created_at ${dialect.bigintType} NOT NULL, + deleted_at ${dialect.bigintType} + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_owner ON spaces(owner_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_type ON spaces(type)`, + + `CREATE TABLE IF NOT EXISTS spaces_members ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + added_at ${dialect.bigintType} NOT NULL, + added_by TEXT, + PRIMARY KEY (space_uri, did) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_members_did ON spaces_members(did)`, + + `CREATE TABLE IF NOT EXISTS spaces_invites ( + token_hash TEXT PRIMARY KEY, + space_uri TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'join', + expires_at ${dialect.bigintType}, + max_uses INTEGER, + used_count INTEGER NOT NULL DEFAULT 0, + created_by TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + revoked_at ${dialect.bigintType}, + note TEXT + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, + ]; +} + +/** SchemaModule-shaped function suitable for `initSchema({ extraSchemas: [...] })`. */ +export async function applyAuthoritySchema(db: Database): Promise<void> { + const dialect = getDialect(db); + const stmts = buildAuthoritySchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); +} diff --git a/packages/contrail-authority/tsconfig.build.json b/packages/contrail-authority/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-authority/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-authority/tsconfig.json b/packages/contrail-authority/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-authority/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-authority/tsup.config.ts b/packages/contrail-authority/tsup.config.ts new file mode 100644 index 0000000..7d6b7eb --- /dev/null +++ b/packages/contrail-authority/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["@atmo-dev/contrail-base"], +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 8409809..02a833f 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,6 +73,7 @@ "@atcute/jetstream": "^1.0.2", "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-authority": "workspace:*", "@atmo-dev/contrail-base": "workspace:*", "cac": "^7.0.0", "hono": "^4.12.8", diff --git a/packages/contrail/src/core/spaces/adapter.ts b/packages/contrail/src/core/spaces/adapter.ts index f80ba5e..50fb33d 100644 --- a/packages/contrail/src/core/spaces/adapter.ts +++ b/packages/contrail/src/core/spaces/adapter.ts @@ -1,4 +1,13 @@ -import type { ContrailConfig, Database, RelationConfig, ResolvedContrailConfig } from "../types"; +/** Contrail's all-in-one default adapter — extends the authority package's + * {@link HostedAuthorityAdapter} (which owns space lifecycle, member list, + * invites) and adds the record-host methods (records, blobs, enrollment). + * + * Phase 7a step 3 will lift the record-host methods into a separate + * HostedRecordHostAdapter, at which point this class becomes a thin + * composition / re-export. For now we keep both roles in one class so + * consumers can wire a single object that satisfies the full StorageAdapter. */ + +import type { ContrailConfig, RelationConfig, ResolvedContrailConfig } from "../types"; import { shortNameForNsid, spacesRecordsTableName, @@ -9,63 +18,19 @@ import { } from "../types"; import { getDialect } from "../dialect"; import type { - AppPolicy, BlobMetaRow, CollectionCount, - CreateInviteInput, EnrollmentRow, - InviteKind, - InviteRow, ListBlobsOptions, ListBlobsResult, ListOptions, ListResult, - ListSpacesOptions, - SpaceMemberRow, - SpaceRow, StorageAdapter, StoredRecord, } from "./types"; +import type { Database } from "../types"; import { buildRecordUri } from "./uri"; - -function parseJson<T>(value: unknown): T | null { - if (value == null) return null; - if (typeof value === "string") { - try { - return JSON.parse(value) as T; - } catch { - return null; - } - } - return value as T; -} - -function toNum(v: unknown): number { - return typeof v === "string" ? Number(v) : (v as number); -} - -function mapSpaceRow(row: any): SpaceRow { - return { - uri: row.uri, - ownerDid: row.owner_did, - type: row.type, - key: row.key, - serviceDid: row.service_did, - appPolicyRef: row.app_policy_ref ?? null, - appPolicy: parseJson<AppPolicy>(row.app_policy), - createdAt: toNum(row.created_at), - deletedAt: row.deleted_at == null ? null : toNum(row.deleted_at), - }; -} - -function mapMemberRow(row: any): SpaceMemberRow { - return { - spaceUri: row.space_uri, - did: row.did, - addedAt: toNum(row.added_at), - addedBy: row.added_by ?? null, - }; -} +import { HostedAuthorityAdapter, parseJson, toNum } from "@atmo-dev/contrail-authority"; function mapBlobMetaRow(row: any): BlobMetaRow { return { @@ -87,21 +52,6 @@ function mapEnrollmentRow(row: any): EnrollmentRow { }; } -function mapInviteRow(row: any): InviteRow { - return { - tokenHash: row.token_hash, - spaceUri: row.space_uri, - kind: (row.kind ?? "join") as InviteKind, - expiresAt: row.expires_at == null ? null : toNum(row.expires_at), - maxUses: row.max_uses == null ? null : Number(row.max_uses), - usedCount: Number(row.used_count), - createdBy: row.created_by, - createdAt: toNum(row.created_at), - revokedAt: row.revoked_at == null ? null : toNum(row.revoked_at), - note: row.note ?? null, - }; -} - /** Row mapper for per-collection spaces_records_<short> tables. * `collection` is injected by the caller (known from the table name). */ function mapRecordRow(row: any, collection: string): StoredRecord { @@ -116,12 +66,7 @@ function mapRecordRow(row: any, collection: string): StoredRecord { }; } -export class HostedAdapter implements StorageAdapter { - constructor( - private readonly db: Database, - private readonly config?: ContrailConfig - ) {} - +export class HostedAdapter extends HostedAuthorityAdapter implements StorageAdapter { /** Resolve the per-collection spaces table name, or throw if the collection * isn't configured (and therefore has no table). */ private tableFor(collection: string): string { @@ -139,238 +84,7 @@ export class HostedAdapter implements StorageAdapter { return spacesRecordsTableName(short); } - async createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow> { - const now = Date.now(); - await this.db - .prepare( - `INSERT INTO spaces (uri, owner_did, type, key, service_did, app_policy_ref, app_policy, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - ) - .bind( - space.uri, - space.ownerDid, - space.type, - space.key, - space.serviceDid, - space.appPolicyRef, - space.appPolicy ? JSON.stringify(space.appPolicy) : null, - now - ) - .run(); - return { ...space, createdAt: now, deletedAt: null }; - } - - async getSpace(spaceUri: string): Promise<SpaceRow | null> { - const row = await this.db - .prepare(`SELECT * FROM spaces WHERE uri = ? AND deleted_at IS NULL`) - .bind(spaceUri) - .first<any>(); - return row ? mapSpaceRow(row) : null; - } - - async listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }> { - const limit = Math.min(options.limit ?? 50, 200); - const clauses: string[] = ["s.deleted_at IS NULL"]; - const params: any[] = []; - let join = ""; - - if (options.type) { - clauses.push("s.type = ?"); - params.push(options.type); - } - if (options.ownerDid) { - clauses.push("s.owner_did = ?"); - params.push(options.ownerDid); - } - if (options.memberDid) { - join = "JOIN spaces_members m ON m.space_uri = s.uri"; - clauses.push("m.did = ?"); - params.push(options.memberDid); - } - if (options.cursor) { - clauses.push("s.created_at < ?"); - params.push(Number(options.cursor)); - } - - const sql = `SELECT s.* FROM spaces s ${join} - WHERE ${clauses.join(" AND ")} - ORDER BY s.created_at DESC - LIMIT ?`; - params.push(limit + 1); - - const { results } = await this.db.prepare(sql).bind(...params).all<any>(); - const spaces = results.map(mapSpaceRow); - let cursor: string | undefined; - if (spaces.length > limit) { - const next = spaces.pop()!; - cursor = String(next.createdAt); - } - return { spaces, cursor }; - } - - async deleteSpace(spaceUri: string): Promise<void> { - await this.db - .prepare(`UPDATE spaces SET deleted_at = ? WHERE uri = ?`) - .bind(Date.now(), spaceUri) - .run(); - } - - async updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void> { - await this.db - .prepare(`UPDATE spaces SET app_policy = ? WHERE uri = ?`) - .bind(JSON.stringify(appPolicy), spaceUri) - .run(); - } - - async addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void> { - await this.db - .prepare( - `INSERT INTO spaces_members (space_uri, did, added_at, added_by) - VALUES (?, ?, ?, ?) - ON CONFLICT (space_uri, did) DO NOTHING` - ) - .bind(spaceUri, did, Date.now(), addedBy) - .run(); - } - - async removeMember(spaceUri: string, did: string): Promise<void> { - await this.db - .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) - .bind(spaceUri, did) - .run(); - } - - async getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null> { - const row = await this.db - .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? AND did = ?`) - .bind(spaceUri, did) - .first<any>(); - return row ? mapMemberRow(row) : null; - } - - async listMembers(spaceUri: string): Promise<SpaceMemberRow[]> { - const { results } = await this.db - .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? ORDER BY added_at ASC`) - .bind(spaceUri) - .all<any>(); - return results.map(mapMemberRow); - } - - async applyMembershipDiff( - spaceUri: string, - adds: string[], - removes: string[], - addedBy: string | null - ): Promise<void> { - const now = Date.now(); - const stmts: any[] = []; - for (const did of adds) { - stmts.push( - this.db - .prepare( - `INSERT INTO spaces_members (space_uri, did, added_at, added_by) - VALUES (?, ?, ?, ?) - ON CONFLICT (space_uri, did) DO NOTHING` - ) - .bind(spaceUri, did, now, addedBy) - ); - } - for (const did of removes) { - stmts.push( - this.db - .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) - .bind(spaceUri, did) - ); - } - if (stmts.length > 0) { - await this.db.batch(stmts); - } - } - - async createInvite(input: CreateInviteInput): Promise<InviteRow> { - const now = Date.now(); - await this.db - .prepare( - `INSERT INTO spaces_invites (token_hash, space_uri, kind, expires_at, max_uses, used_count, created_by, created_at, note) - VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?)` - ) - .bind( - input.tokenHash, - input.spaceUri, - input.kind, - input.expiresAt, - input.maxUses, - input.createdBy, - now, - input.note - ) - .run(); - return { - tokenHash: input.tokenHash, - spaceUri: input.spaceUri, - kind: input.kind, - expiresAt: input.expiresAt, - maxUses: input.maxUses, - usedCount: 0, - createdBy: input.createdBy, - createdAt: now, - revokedAt: null, - note: input.note, - }; - } - - async getInvite(tokenHash: string): Promise<InviteRow | null> { - const row = await this.db - .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) - .bind(tokenHash) - .first<any>(); - return row ? mapInviteRow(row) : null; - } - - async listInvites( - spaceUri: string, - options: { includeRevoked?: boolean } = {} - ): Promise<InviteRow[]> { - const sql = options.includeRevoked - ? `SELECT * FROM spaces_invites WHERE space_uri = ? ORDER BY created_at DESC` - : `SELECT * FROM spaces_invites WHERE space_uri = ? AND revoked_at IS NULL ORDER BY created_at DESC`; - const { results } = await this.db.prepare(sql).bind(spaceUri).all<any>(); - return results.map(mapInviteRow); - } - - async revokeInvite(tokenHash: string): Promise<boolean> { - const res = await this.db - .prepare(`UPDATE spaces_invites SET revoked_at = ? WHERE token_hash = ? AND revoked_at IS NULL`) - .bind(Date.now(), tokenHash) - .run(); - const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; - return Number(changes) > 0; - } - - async redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null> { - // Atomic: increment used_count only if the invite is usable right now AND - // its kind allows redemption (read-only tokens cannot be consumed for membership). - const res = await this.db - .prepare( - `UPDATE spaces_invites - SET used_count = used_count + 1 - WHERE token_hash = ? - AND kind IN ('join', 'read-join') - AND revoked_at IS NULL - AND (expires_at IS NULL OR expires_at > ?) - AND (max_uses IS NULL OR used_count < max_uses)` - ) - .bind(tokenHash, now) - .run(); - const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; - if (Number(changes) === 0) return null; - - const row = await this.db - .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) - .bind(tokenHash) - .first<any>(); - return row ? mapInviteRow(row) : null; - } + // ---- Enrollment ---- async enroll(input: EnrollmentRow): Promise<void> { await this.db @@ -421,6 +135,8 @@ export class HostedAdapter implements StorageAdapter { .run(); } + // ---- Records ---- + async putRecord(record: StoredRecord): Promise<void> { const table = this.tableFor(record.collection); const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); @@ -675,6 +391,8 @@ export class HostedAdapter implements StorageAdapter { return results; } + // ---- Blobs ---- + async putBlobMeta(row: BlobMetaRow): Promise<void> { const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) VALUES (?, ?, ?, ?, ?, ?) diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 0738fd3..9c32af0 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from "vitest/config"; import path from "node:path"; const baseSrc = path.resolve(__dirname, "../contrail-base/src"); +const authoritySrc = path.resolve(__dirname, "../contrail-authority/src"); export default defineConfig({ test: { @@ -11,11 +12,12 @@ export default defineConfig({ }, resolve: { alias: { - // Resolve workspace-internal contrail-base subpaths to source so tests - // don't run through the dist (where tsup drops `node:` prefixes). + // Resolve workspace-internal contrail-* subpaths to source so tests + // don't run through the dists (where tsup drops `node:` prefixes). "@atmo-dev/contrail-base/sqlite": path.join(baseSrc, "adapters/sqlite.ts"), "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), + "@atmo-dev/contrail-authority": path.join(authoritySrc, "index.ts"), }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01c655a..6e26e5a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -376,6 +376,9 @@ importers: '@atcute/xrpc-server': specifier: ^0.1.12 version: 0.1.12 + '@atmo-dev/contrail-authority': + specifier: workspace:* + version: link:../contrail-authority '@atmo-dev/contrail-base': specifier: workspace:* version: link:../contrail-base @@ -414,6 +417,28 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-authority: + dependencies: + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/contrail-base: dependencies: '@atcute/atproto': -- 2.51.2 From b003ad6b82da7c43a02344cd0142d3eb02cf9637 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 02:16:03 +0200 Subject: [PATCH 10/25] phase 7a pt2 --- packages/contrail-appview/package.json | 56 + .../contrail-appview/src/core/backfill.ts | 523 +++++++ packages/contrail-appview/src/core/client.ts | 1 + .../src/core/community-integration.ts | 1 + .../contrail-appview/src/core/db/index.ts | 4 + .../contrail-appview/src/core/db/records.ts | 896 ++++++++++++ .../contrail-appview/src/core/db/schema.ts | 355 +++++ packages/contrail-appview/src/core/dialect.ts | 1 + .../contrail-appview/src/core/identity.ts | 1 + .../src/core/invite/community-handler.ts | 1 + .../contrail-appview/src/core/invite/index.ts | 3 + .../src/core/invite/router.ts | 1 + .../contrail-appview/src/core/invite/token.ts | 1 + .../contrail-appview/src/core/jetstream.ts | 280 ++++ .../contrail-appview/src/core/labels/apply.ts | 64 + .../src/core/labels/hydrate.ts | Bin 0 -> 3547 bytes .../src/core/labels/resolve.ts | 134 ++ .../src/core/labels/schema.ts | 30 + .../src/core/labels/select.ts | 64 + .../src/core/labels/subscribe.ts | 315 +++++ .../contrail-appview/src/core/labels/types.ts | 1 + .../contrail-appview/src/core/persistent.ts | 243 ++++ .../src/core/realtime/durable-object.ts | 1 + .../src/core/realtime/in-memory.ts | 1 + .../src/core/realtime/index.ts | 37 + .../src/core/realtime/merge.ts | 1 + .../src/core/realtime/publishing-adapter.ts | 157 +++ .../src/core/realtime/query-filter.ts | 1 + .../src/core/realtime/resolve.ts | 92 ++ .../src/core/realtime/router.ts | 255 ++++ .../contrail-appview/src/core/realtime/sse.ts | 1 + .../src/core/realtime/ticket.ts | 1 + .../src/core/realtime/types.ts | 1 + .../src/core/realtime/websocket.ts | 1 + packages/contrail-appview/src/core/refresh.ts | 267 ++++ .../contrail-appview/src/core/router/admin.ts | 44 + .../src/core/router/collection.ts | 1225 ++++++++++++++++ .../contrail-appview/src/core/router/feed.ts | 134 ++ .../src/core/router/helpers.ts | 67 + .../src/core/router/hydrate.ts | 223 +++ .../contrail-appview/src/core/router/index.ts | 247 ++++ .../src/core/router/notify.ts | 179 +++ .../src/core/router/profiles.ts | 181 +++ packages/contrail-appview/src/core/search.ts | 31 + .../contrail-appview/src/core/spaces/acl.ts | 1 + .../src/core/spaces/adapter.ts | 505 +++++++ .../contrail-appview/src/core/spaces/auth.ts | 1 + .../src/core/spaces/binding.ts | 1 + .../src/core/spaces/blob-adapter.ts | 1 + .../src/core/spaces/blob-gc.ts | 38 + .../src/core/spaces/blob-refs.ts | 27 + .../src/core/spaces/credentials.ts | 1 + .../src/core/spaces/in-process.ts | 1 + .../src/core/spaces/router.ts | 94 ++ .../src/core/spaces/schema.ts | 103 ++ .../contrail-appview/src/core/spaces/tid.ts | 1 + .../contrail-appview/src/core/spaces/types.ts | 1 + .../contrail-appview/src/core/spaces/uri.ts | 1 + packages/contrail-appview/src/core/types.ts | 1 + packages/contrail-appview/src/index.ts | 65 + packages/contrail-appview/tsconfig.build.json | 7 + packages/contrail-appview/tsconfig.json | 7 + packages/contrail-appview/tsup.config.ts | 15 + packages/contrail-authority/src/index.ts | 8 +- .../contrail-authority/src/invite-routes.ts | 240 ++++ packages/contrail-authority/src/routes.ts | 412 ++++++ packages/contrail-record-host/package.json | 43 + packages/contrail-record-host/src/adapter.ts | 530 +++++++ packages/contrail-record-host/src/blob-gc.ts | 37 + .../contrail-record-host/src/blob-refs.ts | 27 + packages/contrail-record-host/src/index.ts | 27 + packages/contrail-record-host/src/routes.ts | 510 +++++++ packages/contrail-record-host/src/schema.ts | 51 + .../contrail-record-host/tsconfig.build.json | 7 + packages/contrail-record-host/tsconfig.json | 7 + packages/contrail-record-host/tsup.config.ts | 11 + packages/contrail/package.json | 2 + packages/contrail/src/core/backfill.ts | 524 +------ packages/contrail/src/core/client.ts | 2 +- .../src/core/community-integration.ts | 2 +- packages/contrail/src/core/db/index.ts | 5 +- packages/contrail/src/core/db/records.ts | 897 +----------- packages/contrail/src/core/db/schema.ts | 356 +---- packages/contrail/src/core/dialect.ts | 2 +- packages/contrail/src/core/identity.ts | 2 +- .../src/core/invite/community-handler.ts | 2 +- packages/contrail/src/core/invite/index.ts | 4 +- packages/contrail/src/core/invite/router.ts | 251 +--- packages/contrail/src/core/invite/token.ts | 2 +- packages/contrail/src/core/jetstream.ts | 281 +--- packages/contrail/src/core/labels/apply.ts | 65 +- packages/contrail/src/core/labels/hydrate.ts | Bin 3547 -> 44 bytes packages/contrail/src/core/labels/resolve.ts | 135 +- packages/contrail/src/core/labels/schema.ts | 31 +- packages/contrail/src/core/labels/select.ts | 65 +- .../contrail/src/core/labels/subscribe.ts | 316 +---- packages/contrail/src/core/labels/types.ts | 2 +- packages/contrail/src/core/persistent.ts | 244 +--- .../src/core/realtime/durable-object.ts | 2 +- .../contrail/src/core/realtime/in-memory.ts | 2 +- packages/contrail/src/core/realtime/index.ts | 38 +- packages/contrail/src/core/realtime/merge.ts | 2 +- .../src/core/realtime/publishing-adapter.ts | 158 +-- .../src/core/realtime/query-filter.ts | 2 +- .../contrail/src/core/realtime/resolve.ts | 93 +- packages/contrail/src/core/realtime/router.ts | 256 +--- packages/contrail/src/core/realtime/sse.ts | 2 +- packages/contrail/src/core/realtime/ticket.ts | 2 +- packages/contrail/src/core/realtime/types.ts | 2 +- .../contrail/src/core/realtime/websocket.ts | 2 +- packages/contrail/src/core/refresh.ts | 268 +--- packages/contrail/src/core/router/admin.ts | 45 +- .../contrail/src/core/router/collection.ts | 1226 +---------------- packages/contrail/src/core/router/feed.ts | 135 +- packages/contrail/src/core/router/helpers.ts | 68 +- packages/contrail/src/core/router/hydrate.ts | 224 +-- packages/contrail/src/core/router/index.ts | 248 +--- packages/contrail/src/core/router/notify.ts | 180 +-- packages/contrail/src/core/router/profiles.ts | 182 +-- packages/contrail/src/core/search.ts | 32 +- packages/contrail/src/core/spaces/acl.ts | 2 +- packages/contrail/src/core/spaces/adapter.ts | 506 +------ packages/contrail/src/core/spaces/auth.ts | 2 +- packages/contrail/src/core/spaces/binding.ts | 2 +- .../contrail/src/core/spaces/blob-adapter.ts | 2 +- packages/contrail/src/core/spaces/blob-gc.ts | 39 +- .../contrail/src/core/spaces/blob-refs.ts | 28 +- .../contrail/src/core/spaces/credentials.ts | 2 +- .../contrail/src/core/spaces/in-process.ts | 2 +- packages/contrail/src/core/spaces/router.ts | 1028 +------------- packages/contrail/src/core/spaces/schema.ts | 104 +- packages/contrail/src/core/spaces/tid.ts | 2 +- packages/contrail/src/core/spaces/types.ts | 2 +- packages/contrail/src/core/spaces/uri.ts | 2 +- packages/contrail/src/core/types.ts | 2 +- packages/contrail/tests/persistent.test.ts | 12 +- packages/contrail/tests/refresh.test.ts | 36 +- packages/contrail/vitest.config.ts | 2 + pnpm-lock.yaml | 77 ++ 139 files changed, 9068 insertions(+), 8046 deletions(-) create mode 100644 packages/contrail-appview/package.json create mode 100644 packages/contrail-appview/src/core/backfill.ts create mode 100644 packages/contrail-appview/src/core/client.ts create mode 100644 packages/contrail-appview/src/core/community-integration.ts create mode 100644 packages/contrail-appview/src/core/db/index.ts create mode 100644 packages/contrail-appview/src/core/db/records.ts create mode 100644 packages/contrail-appview/src/core/db/schema.ts create mode 100644 packages/contrail-appview/src/core/dialect.ts create mode 100644 packages/contrail-appview/src/core/identity.ts create mode 100644 packages/contrail-appview/src/core/invite/community-handler.ts create mode 100644 packages/contrail-appview/src/core/invite/index.ts create mode 100644 packages/contrail-appview/src/core/invite/router.ts create mode 100644 packages/contrail-appview/src/core/invite/token.ts create mode 100644 packages/contrail-appview/src/core/jetstream.ts create mode 100644 packages/contrail-appview/src/core/labels/apply.ts create mode 100644 packages/contrail-appview/src/core/labels/hydrate.ts create mode 100644 packages/contrail-appview/src/core/labels/resolve.ts create mode 100644 packages/contrail-appview/src/core/labels/schema.ts create mode 100644 packages/contrail-appview/src/core/labels/select.ts create mode 100644 packages/contrail-appview/src/core/labels/subscribe.ts create mode 100644 packages/contrail-appview/src/core/labels/types.ts create mode 100644 packages/contrail-appview/src/core/persistent.ts create mode 100644 packages/contrail-appview/src/core/realtime/durable-object.ts create mode 100644 packages/contrail-appview/src/core/realtime/in-memory.ts create mode 100644 packages/contrail-appview/src/core/realtime/index.ts create mode 100644 packages/contrail-appview/src/core/realtime/merge.ts create mode 100644 packages/contrail-appview/src/core/realtime/publishing-adapter.ts create mode 100644 packages/contrail-appview/src/core/realtime/query-filter.ts create mode 100644 packages/contrail-appview/src/core/realtime/resolve.ts create mode 100644 packages/contrail-appview/src/core/realtime/router.ts create mode 100644 packages/contrail-appview/src/core/realtime/sse.ts create mode 100644 packages/contrail-appview/src/core/realtime/ticket.ts create mode 100644 packages/contrail-appview/src/core/realtime/types.ts create mode 100644 packages/contrail-appview/src/core/realtime/websocket.ts create mode 100644 packages/contrail-appview/src/core/refresh.ts create mode 100644 packages/contrail-appview/src/core/router/admin.ts create mode 100644 packages/contrail-appview/src/core/router/collection.ts create mode 100644 packages/contrail-appview/src/core/router/feed.ts create mode 100644 packages/contrail-appview/src/core/router/helpers.ts create mode 100644 packages/contrail-appview/src/core/router/hydrate.ts create mode 100644 packages/contrail-appview/src/core/router/index.ts create mode 100644 packages/contrail-appview/src/core/router/notify.ts create mode 100644 packages/contrail-appview/src/core/router/profiles.ts create mode 100644 packages/contrail-appview/src/core/search.ts create mode 100644 packages/contrail-appview/src/core/spaces/acl.ts create mode 100644 packages/contrail-appview/src/core/spaces/adapter.ts create mode 100644 packages/contrail-appview/src/core/spaces/auth.ts create mode 100644 packages/contrail-appview/src/core/spaces/binding.ts create mode 100644 packages/contrail-appview/src/core/spaces/blob-adapter.ts create mode 100644 packages/contrail-appview/src/core/spaces/blob-gc.ts create mode 100644 packages/contrail-appview/src/core/spaces/blob-refs.ts create mode 100644 packages/contrail-appview/src/core/spaces/credentials.ts create mode 100644 packages/contrail-appview/src/core/spaces/in-process.ts create mode 100644 packages/contrail-appview/src/core/spaces/router.ts create mode 100644 packages/contrail-appview/src/core/spaces/schema.ts create mode 100644 packages/contrail-appview/src/core/spaces/tid.ts create mode 100644 packages/contrail-appview/src/core/spaces/types.ts create mode 100644 packages/contrail-appview/src/core/spaces/uri.ts create mode 100644 packages/contrail-appview/src/core/types.ts create mode 100644 packages/contrail-appview/src/index.ts create mode 100644 packages/contrail-appview/tsconfig.build.json create mode 100644 packages/contrail-appview/tsconfig.json create mode 100644 packages/contrail-appview/tsup.config.ts create mode 100644 packages/contrail-authority/src/invite-routes.ts create mode 100644 packages/contrail-authority/src/routes.ts create mode 100644 packages/contrail-record-host/package.json create mode 100644 packages/contrail-record-host/src/adapter.ts create mode 100644 packages/contrail-record-host/src/blob-gc.ts create mode 100644 packages/contrail-record-host/src/blob-refs.ts create mode 100644 packages/contrail-record-host/src/index.ts create mode 100644 packages/contrail-record-host/src/routes.ts create mode 100644 packages/contrail-record-host/src/schema.ts create mode 100644 packages/contrail-record-host/tsconfig.build.json create mode 100644 packages/contrail-record-host/tsconfig.json create mode 100644 packages/contrail-record-host/tsup.config.ts diff --git a/packages/contrail-appview/package.json b/packages/contrail-appview/package.json new file mode 100644 index 0000000..c76172b --- /dev/null +++ b/packages/contrail-appview/package.json @@ -0,0 +1,56 @@ +{ + "name": "@atmo-dev/contrail-appview", + "version": "0.6.0", + "description": "Public-records appview for contrail — jetstream ingestion, backfill, query layer, feeds, labels, profiles, per-collection XRPC routes.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-appview" + }, + "keywords": [ + "atproto", + "contrail", + "appview", + "jetstream" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/atproto": "^3.1.10", + "@atcute/cbor": "^2.3.2", + "@atcute/cid": "^2.4.1", + "@atcute/client": "^4.2.1", + "@atcute/identity": "^1.1.4", + "@atcute/identity-resolver": "^1.2.2", + "@atcute/jetstream": "^1.0.2", + "@atcute/lexicons": "^1.2.9", + "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-authority": "workspace:*", + "@atmo-dev/contrail-base": "workspace:*", + "@atmo-dev/contrail-record-host": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "@types/node": "^25.5.0", + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "license": "MIT" +} diff --git a/packages/contrail-appview/src/core/backfill.ts b/packages/contrail-appview/src/core/backfill.ts new file mode 100644 index 0000000..713e23e --- /dev/null +++ b/packages/contrail-appview/src/core/backfill.ts @@ -0,0 +1,523 @@ +import type {} from "@atcute/atproto"; +import { type Did } from "@atcute/lexicons"; +import { isDid, isNsid } from "@atcute/lexicons/syntax"; + +import type { Client } from "@atcute/client"; +import type { ContrailConfig, Database, IngestEvent } from "./types"; +import { getDiscoverableNsids, getDependentNsids, DEFAULT_RELAYS } from "./types"; +import { applyEvents, getLastCursor, saveCursor } from "./db"; +import { getClient, getPDS } from "./client"; + +const PAGE_SIZE = 100; +const BATCH_SIZE = 100; +const MAX_RETRIES = 5; + +const REQUEST_TIMEOUT_MS = 10_000; + +async function withRetry<T>( + fn: () => Promise<T>, + label: string, + maxRetries = 3, + timeoutMs = REQUEST_TIMEOUT_MS +): Promise<T> { + let lastError: unknown; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await Promise.race([ + fn(), + new Promise<never>((_, reject) => + setTimeout(() => reject(new Error(`Timeout: ${label}`)), timeoutMs) + ), + ]); + } catch (err) { + lastError = err; + if (attempt < maxRetries) { + const delay = Math.min(1000 * 2 ** attempt, 10000); + await new Promise((r) => setTimeout(r, delay)); + } + } + } + throw lastError; +} + +async function markFailed( + db: Database, + did: string, + collection: string, + error: string +): Promise<void> { + await db + .prepare( + "UPDATE backfills SET retries = retries + 1, last_error = ? WHERE did = ? AND collection = ?" + ) + .bind(error, did, collection) + .run(); +} + +export interface BackfillOptions { + /** Pre-resolved client — avoids redundant PDS lookups when batching by DID */ + client?: Client; + /** Skip replay detection in applyEvents (safe during initial backfill) */ + skipReplayDetection?: boolean; + /** Max retries per request (default: 3). Set to 0 for single-attempt mode. */ + maxRetries?: number; + /** Per-request timeout in ms (default: 10000). */ + requestTimeout?: number; +} + +export async function backfillUser( + db: Database, + did: string, + collection: string, + deadline: number, + config?: ContrailConfig, + options?: BackfillOptions +): Promise<number> { + if (Date.now() >= deadline) return 0; + + const status = await db + .prepare( + "SELECT completed, pds_cursor, retries FROM backfills WHERE did = ? AND collection = ?" + ) + .bind(did, collection) + .first<{ completed: number; pds_cursor: string | null; retries: number }>(); + + if (status?.completed) return 0; + + if (!status) { + await db + .prepare( + "INSERT INTO backfills (did, collection, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" + ) + .bind(did, collection) + .run(); + } + + let currentCursor: string | undefined = status?.pds_cursor ?? undefined; + const retries = options?.maxRetries ?? 3; + const timeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; + + if (!isDid(did)) { + await markFailed(db, did, collection, `Invalid DID: ${did}`); + return 0; + } + + if (!isNsid(collection)) { + await markFailed(db, did, collection, `Invalid NSID: ${collection}`); + return 0; + } + + let client = options?.client; + if (!client) { + try { + client = await withRetry( + () => getClient(did as Did, db), + `getClient(${did})`, + Math.min(retries, 1), + timeout + ); + } catch (err) { + await markFailed(db, did, collection, String(err)); + return 0; + } + } + + let totalInserted = 0; + let done = false; + + try { + while (Date.now() < deadline) { + const response = await withRetry( + () => + client!.get("com.atproto.repo.listRecords", { + params: { + repo: did as Did, + collection, + limit: PAGE_SIZE, + cursor: currentCursor, + }, + }), + `listRecords(${did}/${collection})`, + retries, + timeout + ); + if (!response.ok) { + await markFailed( + db, + did, + collection, + `listRecords status ${response.status}` + ); + return totalInserted; + } + + if (response.data.records.length === 0) { + done = true; + break; + } + + const now = Date.now(); + const events: IngestEvent[] = response.data.records.map((r) => ({ + uri: r.uri, + did, + collection, + rkey: r.uri.split("/").pop()!, + operation: "create" as const, + cid: r.cid, + record: JSON.stringify(r.value), + time_us: now * 1000, + indexed_at: now * 1000, + })); + + await applyEvents(db, events, config, { + skipReplayDetection: options?.skipReplayDetection, + skipFeedFanout: true, + }); + totalInserted += events.length; + + currentCursor = response.data.cursor ?? undefined; + + await db + .prepare( + "UPDATE backfills SET pds_cursor = ? WHERE did = ? AND collection = ?" + ) + .bind(currentCursor ?? null, did, collection) + .run(); + + if (!currentCursor) { + done = true; + break; + } + } + } catch (err) { + await markFailed(db, did, collection, String(err)); + return totalInserted; + } + + if (done) { + await db + .prepare( + "UPDATE backfills SET completed = 1 WHERE did = ? AND collection = ?" + ) + .bind(did, collection) + .run(); + } + + return totalInserted; +} + +// --- Bulk backfill (groups by DID, resolves client once) --- + +export interface BackfillProgress { + records: number; + usersComplete: number; + usersTotal: number; + usersFailed: number; +} + +export interface BackfillAllOptions { + concurrency?: number; + onProgress?: (progress: BackfillProgress) => void; +} + +export async function backfillPending( + db: Database, + config: ContrailConfig, + options?: BackfillAllOptions +): Promise<number> { + const concurrency = options?.concurrency ?? 100; + let totalBackfilled = 0; + + // Anchor the jetstream cursor to now if it hasn't been set yet, so records + // emitted during backfill are replayed once jetstream starts. + if ((await getLastCursor(db)) === null) { + await saveCursor(db, Date.now() * 1000); + } + + // Reset retries so users that hit the cap in a prior run get another chance. + await db + .prepare("UPDATE backfills SET retries = 0 WHERE completed = 0") + .run(); + + while (true) { + const pending = await db + .prepare( + "SELECT did, collection FROM backfills WHERE completed = 0 AND retries < ? ORDER BY did" + ) + .bind(MAX_RETRIES) + .all<{ did: string; collection: string }>(); + + const rows = pending.results ?? []; + if (rows.length === 0) break; + + // Group by DID so we resolve PDS once per user + const byDid = new Map<string, string[]>(); + for (const row of rows) { + const cols = byDid.get(row.did) ?? []; + cols.push(row.collection); + byDid.set(row.did, cols); + } + + const dids = [...byDid.keys()]; + + // Resolve PDS endpoints in background (populates in-memory cache) + const resolvePromise = (async () => { + for (let i = 0; i < dids.length; i += 200) { + await Promise.allSettled( + dids.slice(i, i + 200).map((did) => + getPDS(did as Did, db).catch(() => {}) + ) + ); + } + })(); + + let roundBackfilled = 0; + let usersComplete = 0; + let usersFailed = 0; + const failedDids: string[] = []; + + const FAST_TIMEOUT = 3_000; + + const emitProgress = () => + options?.onProgress?.({ + records: totalBackfilled + roundBackfilled, + usersComplete, + usersTotal: dids.length, + usersFailed, + }); + + // Fast pass: single attempt per user with short timeout + for (let i = 0; i < dids.length; i += concurrency) { + const batch = dids.slice(i, i + concurrency); + + const results = await Promise.allSettled( + batch.map(async (did) => { + let client: Client | undefined; + try { + client = await withRetry( + () => getClient(did as Did, db), + `getClient(${did})`, + 0, + FAST_TIMEOUT + ); + } catch { + failedDids.push(did); + return 0; + } + + const cols = byDid.get(did)!; + const counts = await Promise.all( + cols.map((col) => + backfillUser(db, did, col, Infinity, config, { + client, + skipReplayDetection: true, + maxRetries: 0, + requestTimeout: FAST_TIMEOUT, + }).catch(() => { + failedDids.push(did); + return 0; + }) + ) + ); + + usersComplete++; + return counts.reduce((a, b) => a + b, 0); + }) + ); + + for (const r of results) { + if (r.status === "fulfilled") roundBackfilled += r.value; + } + + emitProgress(); + } + + // Retry pass: failed DIDs get retries with backoff, still in concurrent batches + if (failedDids.length > 0) { + const uniqueFailed = [...new Set(failedDids)]; + usersComplete -= uniqueFailed.length; // don't count them yet + + for (let i = 0; i < uniqueFailed.length; i += concurrency) { + const batch = uniqueFailed.slice(i, i + concurrency); + + const results = await Promise.allSettled( + batch.map(async (did) => { + let client: Client | undefined; + try { + client = await withRetry( + () => getClient(did as Did, db), + `getClient(${did})`, + 2 + ); + } catch (err) { + for (const col of byDid.get(did)!) { + await markFailed(db, did, col, String(err)); + } + usersFailed++; + usersComplete++; + return 0; + } + + const cols = byDid.get(did)!; + const counts = await Promise.all( + cols.map((col) => + backfillUser(db, did, col, Infinity, config, { + client, + skipReplayDetection: true, + maxRetries: 2, + }) + ) + ); + usersComplete++; + return counts.reduce((a, b) => a + b, 0); + }) + ); + + for (const r of results) { + if (r.status === "fulfilled") roundBackfilled += r.value; + } + + emitProgress(); + } + } + + await resolvePromise; + totalBackfilled += roundBackfilled; + + // If nothing was backfilled this round, we're stuck + if (roundBackfilled === 0) break; + } + + return totalBackfilled; +} + +// --- Discovery --- + +interface DiscoveryPage { + repos: { did: string }[]; + cursor?: string; +} + +async function fetchPage( + relay: string, + collection: string, + cursor?: string +): Promise<DiscoveryPage | null> { + const url = new URL( + `/xrpc/com.atproto.sync.listReposByCollection`, + relay + ); + url.searchParams.set("collection", collection); + url.searchParams.set("limit", "1000"); + if (cursor) { + url.searchParams.set("cursor", cursor); + } + + try { + return await withRetry( + async () => { + const response = await fetch(url.toString()); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + return (await response.json()) as DiscoveryPage; + }, + `fetchPage(${relay}, ${collection})` + ); + } catch (err) { + // Discovery page fetch failed after retries — skip this relay + return null; + } +} + +async function insertDiscoveredDIDs( + db: Database, + dids: string[], + collection: string +): Promise<void> { + if (dids.length === 0) return; + + // Use multi-row INSERT to reduce the number of statements + const CHUNK_SIZE = 50; + for (let i = 0; i < dids.length; i += CHUNK_SIZE) { + const chunk = dids.slice(i, i + CHUNK_SIZE); + const placeholders = chunk.map(() => "(?, ?, 0)").join(", "); + const bindings: string[] = []; + for (const did of chunk) { + bindings.push(did, collection); + } + await db + .prepare( + `INSERT INTO backfills (did, collection, completed) VALUES ${placeholders} ON CONFLICT DO NOTHING` + ) + .bind(...bindings) + .run(); + } +} + +async function saveDiscoveryState( + db: Database, + collection: string, + relay: string, + cursor: string | null, + completed: boolean +): Promise<void> { + await db + .prepare( + "INSERT INTO discovery (collection, relay, cursor, completed) VALUES (?, ?, ?, ?) ON CONFLICT(collection, relay) DO UPDATE SET cursor = excluded.cursor, completed = excluded.completed" + ) + .bind(collection, relay, cursor, completed ? 1 : 0) + .run(); +} + +export async function discoverDIDs( + db: Database, + config: ContrailConfig, + deadline: number +): Promise<string[]> { + const collections = getDiscoverableNsids(config); + const relays = config.relays ?? DEFAULT_RELAYS; + if (relays.length === 0 || collections.length === 0) return []; + + const discovered: string[] = []; + + for (const collection of collections) { + if (Date.now() >= deadline) break; + + let data: DiscoveryPage | null = null; + let relay: string | null = null; + + for (const r of relays) { + const row = await db + .prepare( + "SELECT cursor, completed FROM discovery WHERE collection = ? AND relay = ?" + ) + .bind(collection, r) + .first<{ cursor: string | null; completed: number }>(); + + if (row?.completed) continue; + + data = await fetchPage(r, collection, row?.cursor ?? undefined); + if (data) { + relay = r; + break; + } else { + await saveDiscoveryState(db, collection, r, null, true); + } + } + if (!data || !relay) continue; + + const dids = data.repos?.map((r) => r.did) ?? []; + await insertDiscoveredDIDs(db, dids, collection); + discovered.push(...dids); + + for (const depCollection of getDependentNsids(config)) { + await insertDiscoveredDIDs(db, dids, depCollection); + } + + const completed = !data.cursor; + await saveDiscoveryState(db, collection, relay, data.cursor ?? null, completed); + } + + return discovered; +} diff --git a/packages/contrail-appview/src/core/client.ts b/packages/contrail-appview/src/core/client.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/client.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/community-integration.ts b/packages/contrail-appview/src/core/community-integration.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/community-integration.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/db/index.ts b/packages/contrail-appview/src/core/db/index.ts new file mode 100644 index 0000000..2469f31 --- /dev/null +++ b/packages/contrail-appview/src/core/db/index.ts @@ -0,0 +1,4 @@ +export { initSchema } from "./schema"; +export { getLastCursor, saveCursor, applyEvents, lookupExistingRecords, queryRecords, queryAcrossSources, pruneFeedItems } from "./records"; +export type { QueryOptions, SortOption, ExistingRecordInfo } from "./records"; +export type { RecordSource } from "../types"; diff --git a/packages/contrail-appview/src/core/db/records.ts b/packages/contrail-appview/src/core/db/records.ts new file mode 100644 index 0000000..a45ab9b --- /dev/null +++ b/packages/contrail-appview/src/core/db/records.ts @@ -0,0 +1,896 @@ +import type { + ContrailConfig, + ResolvedContrailConfig, + RelationConfig, + Database, + Statement, + IngestEvent, + RecordRow, + RecordSource, +} from "../types"; +import { + getNestedValue, + getRelationField, + countColumnName, + groupedCountColumnName, + getFeedFollowShortNames, + recordsTableName, + spacesRecordsTableName, + shortNameForNsid, + nsidForShortName, +} from "../types"; +import { getSearchableFields, ftsTableName, buildFtsContent } from "../search"; +import { ftsQueryClause, getDialect } from "../dialect"; + +// --- Counts --- + +interface InboundRelation { + /** Short name of the parent collection. */ + parentCollection: string; + relationName: string; + rel: RelationConfig; +} + +/** Find relations that target the given short-named child collection. */ +function getInboundRelations( + config: ContrailConfig, + childShortName: string +): InboundRelation[] { + const results: InboundRelation[] = []; + for (const [colName, colConfig] of Object.entries(config.collections)) { + for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { + if (rel.collection === childShortName) { + results.push({ parentCollection: colName, relationName: relName, rel }); + } + } + } + return results; +} + +/** + * Collect recount targets from a single event into a shared map. + * The map is keyed by `parentCollection:relationName:targetValue` to deduplicate + * across the entire batch — so 50 RSVPs to the same event produce one recount, not 50. + */ +function collectCountTargets( + event: IngestEvent, + config: ContrailConfig, + existingRecordJson: string | null, + targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> +): void { + const childShort = shortNameForNsid(config, event.collection); + if (!childShort) return; + const inbound = getInboundRelations(config, childShort); + if (inbound.length === 0) return; + + const record = event.record ? JSON.parse(event.record) : null; + const existingRecord = existingRecordJson ? JSON.parse(existingRecordJson) : null; + + for (const { parentCollection, relationName, rel } of inbound) { + if (rel.count === false) continue; + + const field = getRelationField(rel); + + const values: string[] = []; + if (record) { + const t = getNestedValue(record, field); + if (t) values.push(t); + } + if (existingRecord) { + const t = getNestedValue(existingRecord, field); + if (t && !values.includes(t)) values.push(t); + } + + for (const targetValue of values) { + const key = `${parentCollection}:${relationName}:${targetValue}`; + if (!targets.has(key)) { + targets.set(key, { parentCollection, relationName, rel, targetValue }); + } + } + } +} + +/** + * Build deduplicated count UPDATE statements from collected targets. + * One UPDATE per unique parent+relation+target, regardless of how many + * events in the batch affected that target. + */ +function buildBatchCountStatements( + db: Database, + config: ContrailConfig, + targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> +): Statement[] { + const statements: Statement[] = []; + + for (const { parentCollection, relationName, rel, targetValue } of targets.values()) { + const field = getRelationField(rel); + const matchColumn = rel.match === "did" ? "did" : "uri"; + const childTable = recordsTableName(rel.collection); + const parentTable = recordsTableName(parentCollection); + + const setClauses: string[] = []; + const setBindings: (string | number)[] = []; + + const countExpr = rel.countDistinct + ? `COUNT(DISTINCT ${rel.countDistinct})` + : "COUNT(*)"; + + // Total count + const totalCol = countColumnName(rel.collection); + setClauses.push( + `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ?)` + ); + setBindings.push(targetValue); + + // Grouped counts — column names are `count_<child-short>_<group-key>`; match + // against the group's full token value in the record. + if (rel.groupBy) { + const mapping = (config as ResolvedContrailConfig)._resolved?.relations[parentCollection]?.[relationName]; + if (mapping?.groups) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + setClauses.push( + `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ? AND ${getDialect(db).jsonExtract('record', rel.groupBy)} = ?)` + ); + setBindings.push(targetValue, fullToken); + } + } + } + + if (setClauses.length > 0) { + statements.push( + db + .prepare( + `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE ${matchColumn} = ?` + ) + .bind(...setBindings, targetValue) + ); + } + } + + return statements; +} + +// --- FTS --- + +function buildFtsStatements( + db: Database, + event: IngestEvent, + config: ContrailConfig, + existingMap: Map<string, ExistingRecordInfo> +): Statement[] { + // PostgreSQL: tsvector generated column is auto-maintained, no manual FTS sync + if (getDialect(db).ftsStrategy === "generated-column") return []; + + const short = shortNameForNsid(config, event.collection); + if (!short) return []; + const colConfig = config.collections[short]; + if (!colConfig) return []; + + const fields = getSearchableFields(short, colConfig); + if (!fields || fields.length === 0) return []; + + const table = ftsTableName(short); + const stmts: Statement[] = []; + + if (event.operation === "delete") { + stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); + } else { + const record = event.record ? JSON.parse(event.record) : null; + if (!record) return []; + + const content = buildFtsContent(record, fields); + if (!content) return []; + + // Only delete existing FTS row if this is an update (record already existed) + if (existingMap.has(event.uri)) { + stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); + } + stmts.push( + db.prepare(`INSERT INTO ${table} (uri, content) VALUES (?, ?)`).bind(event.uri, content) + ); + } + + return stmts; +} + +// --- Feeds --- + +function buildFeedStatements( + db: Database, + event: IngestEvent, + config: ContrailConfig, + existingRecords: Map<string, string | null> +): Statement[] { + if (!config.feeds) return []; + + const stmts: Statement[] = []; + + const eventShort = shortNameForNsid(config, event.collection); + if (!eventShort) return []; + + for (const [, feedConfig] of Object.entries(config.feeds)) { + const followTable = recordsTableName(feedConfig.follow); + + // Target collection: fan out to followers + if (feedConfig.targets.includes(eventShort)) { + if (event.operation === "create" || event.operation === "update") { + stmts.push( + db + .prepare( + getDialect(db).insertOrIgnore( + `INSERT INTO feed_items (actor, uri, collection, time_us) + SELECT r.did, ?, ?, ? + FROM ${followTable} r + WHERE ${getDialect(db).jsonExtract('r.record', 'subject')} = ?` + ) + ) + .bind(event.uri, event.collection, event.time_us, event.did) + ); + } else if (event.operation === "delete") { + stmts.push( + db.prepare("DELETE FROM feed_items WHERE uri = ?").bind(event.uri) + ); + } + } + + // Follow collection: handle follow/unfollow + if (eventShort === feedConfig.follow) { + if (event.operation === "create") { + const record = event.record ? JSON.parse(event.record) : null; + const subject = record?.subject; + if (subject) { + for (const targetShort of feedConfig.targets) { + const targetTable = recordsTableName(targetShort); + const targetNsid = nsidForShortName(config, targetShort) ?? targetShort; + stmts.push( + db + .prepare( + getDialect(db).insertOrIgnore( + `INSERT INTO feed_items (actor, uri, collection, time_us) + SELECT ?, r.uri, ?, r.time_us + FROM ${targetTable} r + WHERE r.did = ? + ORDER BY r.time_us DESC + LIMIT 100` + ) + ) + .bind(event.did, targetNsid, subject) + ); + } + } + } else if (event.operation === "delete") { + const existingRecord = existingRecords.get(event.uri); + if (existingRecord) { + const parsed = JSON.parse(existingRecord); + const subject = parsed?.subject; + if (subject) { + for (const targetShort of feedConfig.targets) { + const targetTable = recordsTableName(targetShort); + stmts.push( + db + .prepare( + `DELETE FROM feed_items WHERE actor = ? AND uri IN ( + SELECT uri FROM ${targetTable} WHERE did = ? + )` + ) + .bind(event.did, subject) + ); + } + } + } + } + } + } + + return stmts; +} + +// --- Feed pruning --- + +export async function pruneFeedItems( + db: Database, + maxItems: number +): Promise<number> { + const result = await db + .prepare( + `DELETE FROM feed_items WHERE (actor, uri) NOT IN ( + SELECT actor, uri FROM ( + SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn + FROM feed_items + ) sub WHERE rn <= ? + )` + ) + .bind(maxItems) + .run(); + return (result as any)?.changes ?? 0; +} + +// --- Cursor --- + +export async function getLastCursor(db: Database): Promise<number | null> { + const row = await db + .prepare("SELECT time_us FROM cursor WHERE id = 1") + .first<{ time_us: number }>(); + return row ? row.time_us : null; +} + +export async function saveCursor( + db: Database, + timeUs: number +): Promise<void> { + await db + .prepare( + "INSERT INTO cursor (id, time_us) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET time_us = excluded.time_us" + ) + .bind(timeUs) + .run(); +} + +// --- Existing record lookup --- + +export interface ExistingRecordInfo { + cid: string | null; + record: string | null; + /** When the row was last written to our DB (microseconds). Populated + * whenever `lookupExistingRecords` runs, regardless of `includeRecord`. */ + indexed_at: number | null; +} + +/** + * Look up existing records for a set of events, grouped by collection. + * Returns a map of uri → { cid, record }. + * When includeRecord is false, record will always be null (saves reading large blobs). + */ +export async function lookupExistingRecords( + db: Database, + events: { uri: string; collection: string }[], + includeRecord: boolean = true, + config?: ContrailConfig +): Promise<Map<string, ExistingRecordInfo>> { + const result = new Map<string, ExistingRecordInfo>(); + if (events.length === 0) return result; + + // Group by short name (config lookup); skip events for collections not in our config. + const byShort = new Map<string, string[]>(); + for (const e of events) { + const short = config ? shortNameForNsid(config, e.collection) : e.collection; + if (!short) continue; + const uris = byShort.get(short) ?? []; + uris.push(e.uri); + byShort.set(short, uris); + } + + const selectCols = includeRecord ? "uri, cid, record, indexed_at" : "uri, cid, indexed_at"; + for (const [short, uris] of byShort) { + const table = recordsTableName(short); + for (let i = 0; i < uris.length; i += 50) { + const chunk = uris.slice(i, i + 50); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri IN (${placeholders})`) + .bind(...chunk) + .all<{ + uri: string; + cid: string | null; + record?: string | null; + indexed_at: number | null; + }>(); + for (const row of rows.results ?? []) { + result.set(row.uri, { + cid: row.cid, + record: includeRecord ? (row.record ?? null) : null, + indexed_at: row.indexed_at ?? null, + }); + } + } + } + + return result; +} + +// --- Events --- + +export async function applyEvents( + db: Database, + events: IngestEvent[], + config?: ContrailConfig, + options?: { + skipReplayDetection?: boolean; + skipFeedFanout?: boolean; + /** Pre-fetched existing records — skips the internal lookup when provided */ + existing?: Map<string, ExistingRecordInfo>; + /** When provided, publish `collection:<nsid>` and `actor:<did>` realtime + * events for each applied event. Space-scoped publishing happens elsewhere + * (see `realtime/publishing-adapter.ts`); public topics carry public + * records only, which is exactly the scope of this function. */ + pubsub?: import("../realtime/types").PubSub; + } +): Promise<void> { + if (events.length === 0) return; + + const followCollections = config ? getFeedFollowShortNames(config) : []; + const hasCountingRelations = config ? Object.values(config.collections).some(c => + Object.values(c.relations ?? {}).some(r => r.count !== false) + ) : false; + const needRecordContent = followCollections.length > 0 || hasCountingRelations; + + // Use pre-fetched data or look up existing records + let existingMap: Map<string, ExistingRecordInfo>; + if (options?.existing) { + existingMap = options.existing; + } else if (config && !options?.skipReplayDetection) { + existingMap = await lookupExistingRecords(db, events, needRecordContent, config); + } else { + existingMap = new Map(); + } + + const batch: Statement[] = []; + + // Build a record-content map for feed statements (needs string values) + const existingRecordStrings = new Map<string, string | null>(); + for (const [uri, info] of existingMap) { + existingRecordStrings.set(uri, info.record); + } + + // Collect all count recount targets across the batch, deduplicated + const countTargets = new Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }>(); + + for (const e of events) { + // Event's collection is an NSID. Look up the short name from config. + // If no config or not found, treat collection string as-is (for tests that pre-populate tables). + const short = config + ? shortNameForNsid(config, e.collection) ?? (config.collections[e.collection] ? e.collection : null) + : e.collection; + if (!short) { + (config?.logger ?? console).warn( + `[ingest] drop (unknown collection in applyEvents): ${e.operation} ${e.uri} collection=${e.collection}` + ); + continue; + } + const table = recordsTableName(short); + + if (e.operation === "delete") { + batch.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(e.uri)); + } else { + batch.push( + db.prepare( + `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, time_us = excluded.time_us, indexed_at = excluded.indexed_at` + ).bind( + e.uri, + e.did, + e.rkey, + e.cid, + e.record, + e.time_us, + e.indexed_at + ) + ); + } + + if (config) { + // Collect count targets (deduplicated across the whole batch) + const existingRecordJson = existingMap.get(e.uri)?.record ?? null; + collectCountTargets(e, config, existingRecordJson, countTargets); + + // Feed fanout still needs replay detection + const existingInfo = existingMap.get(e.uri); + const isReplay = + e.operation === "delete" + ? existingInfo === undefined + : existingInfo?.cid === e.cid; + + if (!isReplay && !options?.skipFeedFanout) { + batch.push(...buildFeedStatements(db, e, config, existingRecordStrings)); + } + batch.push(...buildFtsStatements(db, e, config, existingMap)); + } + } + + // Build deduplicated count statements — one UPDATE per unique target + if (config) { + batch.push(...buildBatchCountStatements(db, config, countTargets)); + } + + await db.batch(batch); + + // Publish realtime events for public records (collection: and actor:). + // Space records publish via the wrapping adapter; this path is public-only. + if (options?.pubsub) { + const pubsub = options.pubsub; + const ts = Date.now(); + for (const e of events) { + if (e.operation === "delete") { + const payload = { + uri: e.uri, + did: e.did, + collection: e.collection, + rkey: e.rkey, + }; + await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.deleted", payload, ts }); + await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.deleted", payload, ts }); + } else { + const record = e.record ? safeParseJson(e.record) : {}; + const payload = { + uri: e.uri, + did: e.did, + collection: e.collection, + rkey: e.rkey, + cid: e.cid, + record, + time_us: e.time_us, + }; + await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.created", payload, ts }); + await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.created", payload, ts }); + } + } + } +} + +function safeParseJson(s: string): Record<string, unknown> { + try { + const v = JSON.parse(s); + return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {}; + } catch { + return {}; + } +} + +// --- Count columns --- + +/** Count column descriptor. `type` is the identifier returned in API responses and + * accepted in countFilters — we keep the full record token for grouped counts so + * callers pass e.g. "community.lexicon.calendar.rsvp#going" and filter/hydrate by it. */ +function getCountColumns( + config: ContrailConfig, + shortName: string +): { type: string; column: string }[] { + const colConfig = config.collections[shortName]; + if (!colConfig?.relations) return []; + const columns: { type: string; column: string }[] = []; + const relMap = (config as ResolvedContrailConfig)._resolved?.relations[shortName] ?? {}; + + for (const [relName, rel] of Object.entries(colConfig.relations)) { + if (rel.count === false) continue; + // Total: identifier is the child's short name; column is `count_<child-short>`. + columns.push({ type: rel.collection, column: countColumnName(rel.collection) }); + const mapping = relMap[relName]; + if (mapping) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + // Grouped: identifier is the full record token (stable across deployments); + // column is `count_<child-short>_<group-key>`. + columns.push({ + type: fullToken, + column: groupedCountColumnName(rel.collection, groupKey), + }); + } + } + } + return columns; +} + +/** For a given "count type" (short name or full group token), return the DB column. */ +function countColumnForType( + config: ContrailConfig, + shortName: string, + type: string +): string | null { + for (const col of getCountColumns(config, shortName)) { + if (col.type === type) return col.column; + } + return null; +} + +// --- Query --- + +export interface SortOption { + recordField?: string; + countType?: string; + direction: "asc" | "desc"; +} + +/** Opaque keyset cursor. `t` is the tiebreaker (time_us of the last row), + * `v` is the sort-key value (string for record fields, number for counts), + * `k` identifies the sort so we can reject mismatched cursors. */ +interface CursorPayload { + t: number; + v?: string | number; + k: "time" | string; // "time" | `field:<name>` | `count:<type>` +} + +function sortKind(sort?: SortOption): "time" | string { + if (sort?.recordField) return `field:${sort.recordField}`; + if (sort?.countType) return `count:${sort.countType}`; + return "time"; +} + +function encodeCursor(payload: CursorPayload): string { + return Buffer.from(JSON.stringify(payload), "utf8").toString("base64url"); +} + +function decodeCursor(cursor: string): CursorPayload | null { + try { + const json = Buffer.from(cursor, "base64url").toString("utf8"); + const p = JSON.parse(json); + if (typeof p?.t !== "number" || typeof p?.k !== "string") return null; + return p as CursorPayload; + } catch { + return null; + } +} + +export interface QueryOptions { + collection: string; + did?: string; + limit?: number; + cursor?: string; + filters?: Record<string, string>; + rangeFilters?: Record<string, { min?: string; max?: string }>; + countFilters?: Record<string, number>; + sort?: SortOption; + search?: string; + source?: RecordSource; + /** When set, query the per-space table (`spaces_records_<short>`) instead of the + * public `records_<short>` table, scoped to rows where `space_uri = ?`. */ + spaceUri?: string; +} + +export async function queryRecords( + db: Database, + config: ContrailConfig, + options: QueryOptions +): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { + const { + collection: collectionInput, + did, + limit: rawLimit, + cursor, + filters = {}, + rangeFilters = {}, + countFilters = {}, + sort, + search, + source, + spaceUri, + } = options; + + // Accept either the short name (canonical) or the full NSID for convenience. + const collection = + config.collections[collectionInput] + ? collectionInput + : shortNameForNsid(config, collectionInput) ?? collectionInput; + + const table = spaceUri ? spacesRecordsTableName(collection) : recordsTableName(collection); + const limit = Math.min(Math.max(1, rawLimit ?? 50), 200); + const conditions: string[] = []; + const bindings: (string | number)[] = []; + + if (spaceUri) { + conditions.push("r.space_uri = ?"); + bindings.push(spaceUri); + } + + if (source?.conditions) conditions.push(...source.conditions); + if (source?.params) bindings.push(...source.params); + + const countCols = getCountColumns(config, collection); + + if (did) { + conditions.push("r.did = ?"); + bindings.push(did); + } + + // Opaque keyset cursor encoding { t, v?, k }. Silently ignored if it doesn't + // match the current sort — callers shouldn't mix sort params with stale cursors. + const expectedKind = sortKind(sort); + if (cursor) { + const payload = decodeCursor(cursor); + if (payload && payload.k === expectedKind) { + if (sort?.recordField) { + const sortExpr = getDialect(db).jsonExtract('r.record', sort.recordField); + const cmp = sort.direction === "desc" ? "<" : ">"; + conditions.push(`(${sortExpr} ${cmp} ? OR (${sortExpr} = ? AND r.time_us < ?))`); + const v = payload.v ?? ""; + bindings.push(v as string | number, v as string | number, payload.t); + } else if (sort?.countType) { + const sortCol = countColumnForType(config, collection, sort.countType); + if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); + const cmp = sort.direction === "desc" ? "<" : ">"; + conditions.push(`(r.${sortCol} ${cmp} ? OR (r.${sortCol} = ? AND r.time_us < ?))`); + const v = Number(payload.v ?? 0); + bindings.push(v, v, payload.t); + } else { + conditions.push("r.time_us < ?"); + bindings.push(payload.t); + } + } + } + + for (const [field, value] of Object.entries(filters)) { + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} = ?`); + bindings.push(value); + } + + for (const [field, range] of Object.entries(rangeFilters)) { + if (range.min != null) { + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} >= ?`); + bindings.push(range.min); + } + if (range.max != null) { + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} <= ?`); + bindings.push(range.max); + } + } + + for (const [type, minCount] of Object.entries(countFilters)) { + const col = countColumnForType(config, collection, type); + if (!col) continue; // unknown count type — skip filter + conditions.push(`r.${col} >= ?`); + bindings.push(minCount); + } + + // FTS search. Not supported in space mode yet (would need composite keying + // because the same at-URI can appear in multiple spaces). + let ftsJoin = ""; + let ftsClause: ReturnType<typeof ftsQueryClause> | null = null; + if (search && !spaceUri) { + const colConfig2 = config.collections[collection]; + const fields = colConfig2 ? getSearchableFields(collection, colConfig2) : null; + if (fields && fields.length > 0) { + ftsClause = ftsQueryClause(getDialect(db), recordsTableName(collection)); + ftsJoin = ftsClause.join; + conditions.push(ftsClause.condition); + // SECURITY: `search` is user input bound as a parameter, not interpolated. + bindings.push(search); + } + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + + const countSelect = countCols.length > 0 + ? ", " + countCols.map(({ column }) => `r.${column}`).join(", ") + : ""; + const select = `r.uri, r.did, r.rkey, r.cid, r.record, r.time_us, r.indexed_at${countSelect}`; + + const join = [source?.joins, ftsJoin].filter(Boolean).join(" "); + + let orderBy: string; + if (sort?.recordField) { + const dir = sort.direction === "desc" ? "DESC" : "ASC"; + orderBy = `${getDialect(db).jsonExtract('r.record', sort.recordField)} ${dir}, r.time_us DESC`; + } else if (sort?.countType) { + const dir = sort.direction === "desc" ? "DESC" : "ASC"; + const sortCol = countColumnForType(config, collection, sort.countType); + if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); + orderBy = `r.${sortCol} ${dir}, r.time_us DESC`; + } else if (ftsClause) { + orderBy = `${ftsClause.orderExpr}, r.time_us DESC`; + // PG ts_rank needs the search term bound again for ORDER BY + if (getDialect(db).ftsStrategy === "generated-column" && search) { + bindings.push(search); + } + } else { + orderBy = "r.time_us DESC"; + } + + bindings.push(limit); + + const query = `SELECT ${select} FROM ${table} r ${join} ${where} ORDER BY ${orderBy} LIMIT ?`; + + const result = await db + .prepare(query) + .bind(...bindings) + .all<any>(); + + const nsid = nsidForShortName(config, collection) ?? collection; + const records = (result.results ?? []).map((row: any) => { + const rec: RecordRow & { counts?: Record<string, number> } = { + uri: row.uri, + did: row.did, + collection: nsid, + rkey: row.rkey, + cid: row.cid, + record: row.record, + time_us: row.time_us, + indexed_at: row.indexed_at, + ...(spaceUri ? { space: spaceUri } : {}), + }; + if (countCols.length > 0) { + const counts: Record<string, number> = {}; + for (const { type, column } of countCols) { + const val = row[column]; + if (val != null && val !== 0) counts[type] = val; + } + if (Object.keys(counts).length > 0) rec.counts = counts; + } + return rec; + }); + + const nextCursor = + records.length === limit + ? buildCursor(records[records.length - 1], sort, expectedKind) + : undefined; + + return { records, cursor: nextCursor }; +} + +/** Build an opaque keyset cursor from the last row of a page. */ +function buildCursor( + row: RecordRow & { counts?: Record<string, number> }, + sort: SortOption | undefined, + kind: string +): string { + const t = Number(row.time_us); + if (sort?.recordField) { + const parsed = row.record ? JSON.parse(row.record) : null; + const v = parsed ? getNestedValue(parsed, sort.recordField) : undefined; + return encodeCursor({ t, v: v == null ? "" : String(v), k: kind }); + } + if (sort?.countType) { + const v = row.counts?.[sort.countType] ?? 0; + return encodeCursor({ t, v, k: kind }); + } + return encodeCursor({ t, k: kind }); +} + +/** Compare two rows according to the active sort order. Returns negative if + * `a` should come before `b`, positive otherwise. Matches the SQL ORDER BY. */ +function compareRows( + a: RecordRow & { counts?: Record<string, number> }, + b: RecordRow & { counts?: Record<string, number> }, + sort: SortOption | undefined +): number { + const timeCmp = Number(b.time_us) - Number(a.time_us); // time_us DESC + if (sort?.recordField) { + const ar = a.record ? JSON.parse(a.record) : null; + const br = b.record ? JSON.parse(b.record) : null; + const av = ar ? getNestedValue(ar, sort.recordField) : undefined; + const bv = br ? getNestedValue(br, sort.recordField) : undefined; + const dir = sort.direction === "desc" ? -1 : 1; + const cmp = (av === bv ? 0 : (av! < bv! ? -1 : 1)) * dir; + return cmp !== 0 ? cmp : timeCmp; + } + if (sort?.countType) { + const av = a.counts?.[sort.countType] ?? 0; + const bv = b.counts?.[sort.countType] ?? 0; + const dir = sort.direction === "desc" ? -1 : 1; + const cmp = (av === bv ? 0 : (av < bv ? -1 : 1)) * dir; + return cmp !== 0 ? cmp : timeCmp; + } + return timeCmp; +} + +/** Run a listRecords query across the public table and a set of per-space tables + * in parallel, then merge according to the active sort order. The cursor is a + * shared keyset cursor — every sub-query applies the same `WHERE` keyset, so + * pagination is consistent across sources. */ +export async function queryAcrossSources( + db: Database, + config: ContrailConfig, + options: QueryOptions, + spaceUris: string[] +): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { + if (spaceUris.length === 0) { + return queryRecords(db, config, options); + } + const limit = Math.min(Math.max(1, options.limit ?? 50), 200); + const perSourceLimit = limit; // each source fetches up to `limit`; we trim after merge + + const tasks: Promise<{ records: (RecordRow & { counts?: Record<string, number> })[] }>[] = [ + queryRecords(db, config, { ...options, limit: perSourceLimit }), + ]; + for (const spaceUri of spaceUris) { + tasks.push(queryRecords(db, config, { ...options, spaceUri, limit: perSourceLimit })); + } + const results = await Promise.all(tasks); + const merged = results.flatMap((r) => r.records); + merged.sort((a, b) => compareRows(a, b, options.sort)); + const trimmed = merged.slice(0, limit); + const kind = sortKind(options.sort); + const cursor = + trimmed.length === limit ? buildCursor(trimmed[trimmed.length - 1], options.sort, kind) : undefined; + return { records: trimmed, cursor }; +} + +// --- Users --- + diff --git a/packages/contrail-appview/src/core/db/schema.ts b/packages/contrail-appview/src/core/db/schema.ts new file mode 100644 index 0000000..562a6ee --- /dev/null +++ b/packages/contrail-appview/src/core/db/schema.ts @@ -0,0 +1,355 @@ +import type { ContrailConfig, Database, ResolvedContrailConfig, ResolvedMaps } from "../types"; +import type { SqlDialect } from "../dialect"; +import { buildFtsSchema, getDialect } from "../dialect"; +import { + getRelationField, + countColumnName, + groupedCountColumnName, + recordsTableName, + spacesRecordsTableName, + resolveConfig, +} from "../types"; +import { getSearchableFields } from "../search"; +import { buildSpacesBaseSchema } from "../spaces/schema"; +import { buildLabelsSchema } from "../labels/schema"; + +function getResolved(config: ContrailConfig): ResolvedMaps { + return (config as ResolvedContrailConfig)._resolved ?? resolveConfig(config)._resolved; +} + +function buildBaseSchema(dialect: SqlDialect): string { + return ` +CREATE TABLE IF NOT EXISTS backfills ( + did TEXT NOT NULL, + collection TEXT NOT NULL, + completed INTEGER NOT NULL DEFAULT 0, + pds_cursor TEXT, + retries INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + PRIMARY KEY (did, collection) +); +CREATE TABLE IF NOT EXISTS discovery ( + collection TEXT NOT NULL, + relay TEXT NOT NULL, + cursor TEXT, + completed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (collection, relay) +); +CREATE TABLE IF NOT EXISTS cursor ( + id INTEGER PRIMARY KEY CHECK (id = 1), + time_us ${dialect.bigintType} NOT NULL +); +CREATE TABLE IF NOT EXISTS identities ( + did TEXT PRIMARY KEY, + handle TEXT, + pds TEXT, + resolved_at ${dialect.bigintType} NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_identities_handle ON identities(handle); +`; +} + +function sanitizeName(name: string): string { + return name.replace(/[^a-zA-Z0-9]/g, "_"); +} + +interface BuilderOpts { + /** Emit tables for the spaces variant (spaces_records_<short> with space_uri column). */ + forSpaces?: boolean; +} + +function tableFor(shortName: string, opts: BuilderOpts): string { + return opts.forSpaces ? spacesRecordsTableName(shortName) : recordsTableName(shortName); +} + +function namePrefix(opts: BuilderOpts): string { + return opts.forSpaces ? "sp_" : ""; +} + +export function buildCollectionTables( + config: ContrailConfig, + dialect: SqlDialect, + opts: BuilderOpts = {} +): string[] { + const stmts: string[] = []; + for (const [shortName, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const table = tableFor(shortName, opts); + const np = namePrefix(opts); + if (opts.forSpaces) { + stmts.push( + `CREATE TABLE IF NOT EXISTS ${table} ( + space_uri TEXT NOT NULL, + uri TEXT NOT NULL, + did TEXT NOT NULL, + rkey TEXT NOT NULL, + cid TEXT, + record ${dialect.recordColumnType}, + time_us ${dialect.bigintType} NOT NULL, + indexed_at ${dialect.bigintType} NOT NULL, + PRIMARY KEY (space_uri, did, rkey) + )` + ); + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_time ON ${table}(space_uri, time_us DESC)` + ); + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_did ON ${table}(space_uri, did)` + ); + } else { + stmts.push( + `CREATE TABLE IF NOT EXISTS ${table} ( + uri TEXT PRIMARY KEY, + did TEXT NOT NULL, + rkey TEXT NOT NULL, + cid TEXT, + record ${dialect.recordColumnType}, + time_us ${dialect.bigintType} NOT NULL, + indexed_at ${dialect.bigintType} NOT NULL + )` + ); + stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_did ON ${table}(did)`); + stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_time ON ${table}(time_us DESC)`); + } + } + return stmts; +} + +export function buildDynamicIndexes( + config: ContrailConfig, + dialect: SqlDialect, + opts: BuilderOpts = {} +): string[] { + const resolved = getResolved(config); + const indexes: string[] = []; + const np = namePrefix(opts); + for (const [collection, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const table = tableFor(collection, opts); + const queryable = resolved.queryable[collection] ?? colConfig.queryable ?? {}; + for (const field of Object.keys(queryable)) { + const idxName = `idx_${np}${sanitizeName(collection)}_${sanitizeName(field)}`; + indexes.push( + `CREATE INDEX IF NOT EXISTS ${idxName} ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', field))})` + ); + } + + for (const [, rel] of Object.entries(colConfig.relations ?? {})) { + const childShort = rel.collection; + const childConfig = config.collections[childShort]; + if (opts.forSpaces && childConfig?.allowInSpaces === false) continue; + const on = getRelationField(rel); + const childTable = tableFor(childShort, opts); + const idxName = `idx_${np}${sanitizeName(childShort)}_${sanitizeName(on)}`; + indexes.push( + `CREATE INDEX IF NOT EXISTS ${idxName} ON ${childTable}(${dialect.indexExpression(dialect.jsonExtract('record', on))})` + ); + } + } + return indexes; +} + +export function buildCountColumns(config: ContrailConfig, opts: BuilderOpts = {}): string[] { + const resolved = getResolved(config); + const stmts: string[] = []; + const addedColumns = new Map<string, Set<string>>(); + const np = namePrefix(opts); + + for (const [collection, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const table = tableFor(collection, opts); + const relMap = resolved.relations[collection] ?? {}; + + if (!addedColumns.has(table)) addedColumns.set(table, new Set()); + const tableColumns = addedColumns.get(table)!; + + for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { + if (rel.count === false) continue; + if (opts.forSpaces && config.collections[rel.collection]?.allowInSpaces === false) continue; + const totalCol = countColumnName(rel.collection); + if (!tableColumns.has(totalCol)) { + tableColumns.add(totalCol); + stmts.push( + `ALTER TABLE ${table} ADD COLUMN ${totalCol} INTEGER NOT NULL DEFAULT 0` + ); + } + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${totalCol} ON ${table}(${totalCol} DESC, time_us DESC)` + ); + + const mapping = relMap[relName]; + if (mapping) { + for (const groupKey of Object.keys(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + if (!tableColumns.has(groupCol)) { + tableColumns.add(groupCol); + stmts.push( + `ALTER TABLE ${table} ADD COLUMN ${groupCol} INTEGER NOT NULL DEFAULT 0` + ); + } + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${groupCol} ON ${table}(${groupCol} DESC, time_us DESC)` + ); + } + } + } + } + return stmts; +} + +function buildFeedTables(config: ContrailConfig, dialect: SqlDialect): string[] { + if (!config.feeds || Object.keys(config.feeds).length === 0) return []; + const stmts = [ + `CREATE TABLE IF NOT EXISTS feed_items ( + actor TEXT NOT NULL, + uri TEXT NOT NULL, + collection TEXT NOT NULL, + time_us ${dialect.bigintType} NOT NULL, + PRIMARY KEY (actor, uri) + )`, + `CREATE INDEX IF NOT EXISTS idx_feed_actor_coll_time ON feed_items(actor, collection, time_us DESC)`, + `CREATE INDEX IF NOT EXISTS idx_feed_actor_time ON feed_items(actor, time_us DESC)`, + `CREATE TABLE IF NOT EXISTS feed_backfills ( + actor TEXT NOT NULL, + feed TEXT NOT NULL, + completed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (actor, feed) + )`, + ]; + + const followCollections = new Set(Object.values(config.feeds).map((f) => f.follow)); + for (const col of followCollections) { + const table = recordsTableName(col); + const safe = sanitizeName(col); + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${safe}_subject ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', 'subject'))})` + ); + } + + return stmts; +} + +export function buildFtsTables( + config: ContrailConfig, + dialect: SqlDialect, + opts: BuilderOpts = {} +): string[] { + const stmts: string[] = []; + for (const [collection, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const fields = getSearchableFields(collection, colConfig); + if (!fields || fields.length === 0) continue; + const table = tableFor(collection, opts); + stmts.push(...buildFtsSchema(dialect, table, fields)); + } + return stmts; +} + +const MIGRATIONS = [ + "ALTER TABLE backfills ADD COLUMN retries INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE backfills ADD COLUMN last_error TEXT", + "ALTER TABLE spaces_invites ADD COLUMN kind TEXT NOT NULL DEFAULT 'join'", +]; + +async function runMigrations(db: Database): Promise<void> { + for (const sql of MIGRATIONS) { + try { + await db.prepare(sql).run(); + } catch { + // Column already exists — ignore + } + } +} + +/** Pluggable schema applier — passed in by extension packages (community, + * third-party plugins) to install their own tables alongside contrail's. */ +export type SchemaModule = (db: Database) => Promise<void>; + +export interface InitSchemaOptions { + /** Separate DB for the spaces tables. Defaults to the main `db`. */ + spacesDb?: Database; + /** Extra schema modules to apply after contrail's own DDL. Used by the + * community package to install its tables — contrail core no longer + * imports community schema directly. */ + extraSchemas?: SchemaModule[]; +} + +async function applySpacesSchema( + target: Database, + config: ContrailConfig, + dialect: SqlDialect +): Promise<void> { + const base = buildSpacesBaseSchema(dialect); + const perCollection = buildCollectionTables(config, dialect, { forSpaces: true }); + const indexes = buildDynamicIndexes(config, dialect, { forSpaces: true }); + await target.batch([...base, ...perCollection, ...indexes].map((s) => target.prepare(s))); + + const ftsStmts = buildFtsTables(config, dialect, { forSpaces: true }); + for (const stmt of ftsStmts) { + try { await target.prepare(stmt).run(); } catch { /* FTS5 unavailable */ } + } + for (const stmt of buildCountColumns(config, { forSpaces: true })) { + try { await target.prepare(stmt).run(); } catch { /* already exists */ } + } +} + +export async function initSchema( + db: Database, + config: ContrailConfig, + options: InitSchemaOptions = {} +): Promise<void> { + const dialect = getDialect(db); + const baseStatements = buildBaseSchema(dialect).split(";") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + const collectionStatements = buildCollectionTables(config, dialect); + const indexStatements = buildDynamicIndexes(config, dialect); + const ftsStatements = buildFtsTables(config, dialect); + const feedStatements = buildFeedTables(config, dialect); + + const spacesDb = options.spacesDb; + const spacesSharesMainDb = !spacesDb || spacesDb === db; + + const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...feedStatements]; + + await db.batch(all.map((s) => db.prepare(s))); + + if (config.spaces?.authority || config.spaces?.recordHost) { + await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); + } + + // Extension schemas (e.g. community) — applied to the spacesDb when one's + // configured separately, since they typically reference space_uri. The + // caller is responsible for routing the schema to the right db; we just + // hand it the spaces-or-main DB as a sensible default. + const extensionTarget = spacesSharesMainDb ? db : spacesDb!; + for (const apply of options.extraSchemas ?? []) { + await apply(extensionTarget); + } + + if (config.labels) { + // Labels tables live on the main DB — they're keyed by at-URI / DID and + // are read alongside public records during hydration. + const labelsStmts = buildLabelsSchema(dialect); + await db.batch(labelsStmts.map((s) => db.prepare(s))); + } + + // FTS5 may not be available (e.g. node:sqlite) — skip gracefully + for (const stmt of ftsStatements) { + try { + await db.prepare(stmt).run(); + } catch { + // FTS5 not supported in this environment + } + } + await runMigrations(db); + + // Add count columns (ALTER TABLE — may already exist) + for (const stmt of buildCountColumns(config)) { + try { + await db.prepare(stmt).run(); + } catch { + // Column/index already exists — ignore + } + } +} diff --git a/packages/contrail-appview/src/core/dialect.ts b/packages/contrail-appview/src/core/dialect.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/dialect.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/identity.ts b/packages/contrail-appview/src/core/identity.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/identity.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/invite/community-handler.ts b/packages/contrail-appview/src/core/invite/community-handler.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/invite/community-handler.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/invite/index.ts b/packages/contrail-appview/src/core/invite/index.ts new file mode 100644 index 0000000..8dae0df --- /dev/null +++ b/packages/contrail-appview/src/core/invite/index.ts @@ -0,0 +1,3 @@ +export { generateInviteToken, hashInviteToken, mintInviteToken } from "./token"; +export { registerInviteRoutes } from "./router"; +export type { InviteRoutesOptions } from "./router"; diff --git a/packages/contrail-appview/src/core/invite/router.ts b/packages/contrail-appview/src/core/invite/router.ts new file mode 100644 index 0000000..920123f --- /dev/null +++ b/packages/contrail-appview/src/core/invite/router.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-authority"; diff --git a/packages/contrail-appview/src/core/invite/token.ts b/packages/contrail-appview/src/core/invite/token.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/invite/token.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/jetstream.ts b/packages/contrail-appview/src/core/jetstream.ts new file mode 100644 index 0000000..d42a5e8 --- /dev/null +++ b/packages/contrail-appview/src/core/jetstream.ts @@ -0,0 +1,280 @@ +import { JetstreamSubscription } from "@atcute/jetstream"; +import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; +import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS } from "./types"; +import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; +import { refreshStaleIdentities } from "./identity"; + +const BATCH_SIZE = 50; +const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour + +/** Mutable state that persists across ingest cycles within the same process. */ +export interface IngestState { + cachedKnownDids?: Set<string>; + schemaInitialized: boolean; + lastFeedPruneMs: number; +} + +export function createIngestState(): IngestState { + return { schemaInitialized: false, lastFeedPruneMs: 0 }; +} + +function getLogger(config: ContrailConfig): Logger { + return config.logger ?? console; +} + +export async function ingestEvents( + config: ContrailConfig, + cursor: number | null, + safetyTimeoutMs: number = 25_000, + knownDids?: Set<string> +): Promise<{ events: IngestEvent[]; lastCursor: number | null }> { + const log = getLogger(config); + const startTimeUs = Date.now() * 1000; + const deadline = Date.now() + safetyTimeoutMs; + const collected: IngestEvent[] = []; + + const collections = getCollectionNsids(config); + const dependentCollections = new Set(getDependentNsids(config)); + const urls = config.jetstreams ?? []; + + let totalCommits = 0; + let filteredUnknownDid = 0; + const filteredDidSamples = new Set<string>(); + let lastYieldedTimeUs: number | null = null; + let firstYieldedTimeUs: number | null = null; + let connectCount = 0; + const seenUris = new Map<string, number>(); // uri -> time_us of first occurrence + const duplicateUris: string[] = []; + + const subscription = new JetstreamSubscription({ + url: urls, + wantedCollections: collections, + ...(cursor !== null ? { cursor } : {}), + onConnectionOpen() { + connectCount++; + log.log( + `[ingest] connected to Jetstream #${connectCount} (url=${urls.join("|")}, cursor=${cursor ?? "none"}, wanted=${collections.join(",")})` + ); + }, + onConnectionClose(event) { + log.log( + `[ingest] disconnected from Jetstream: ${event.code} ${event.reason}` + ); + }, + onConnectionError(event) { + log.error("[ingest] Jetstream error:", event.error); + }, + }); + + for await (const event of subscription) { + if (firstYieldedTimeUs === null) firstYieldedTimeUs = event.time_us; + lastYieldedTimeUs = event.time_us; + if (event.kind === "commit") { + const { commit } = event; + totalCommits++; + + const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; + + if (dependentCollections.has(commit.collection) && knownDids) { + if (!knownDids.has(event.did)) { + filteredUnknownDid++; + if (filteredDidSamples.size < 10) filteredDidSamples.add(event.did); + continue; + } + } + + const prev = seenUris.get(uri); + if (prev !== undefined) { + duplicateUris.push(uri); + log.warn( + `[ingest] DUPLICATE in cycle: ${uri} first time_us=${prev}, again=${event.time_us}, delta=${event.time_us - prev}us` + ); + } else { + seenUris.set(uri, event.time_us); + } + + const now = Date.now(); + + collected.push({ + uri, + did: event.did, + time_us: event.time_us, + collection: commit.collection, + operation: commit.operation as "create" | "update" | "delete", + rkey: commit.rkey, + cid: commit.operation === "delete" ? null : commit.cid, + record: + commit.operation === "delete" + ? null + : JSON.stringify(commit.record), + indexed_at: now * 1000, + }); + + log.log( + `[ingest] keep: ${commit.operation} ${uri} time_us=${event.time_us}` + ); + + if (knownDids && !dependentCollections.has(commit.collection)) { + knownDids.add(event.did); + } + } + + if (event.time_us >= startTimeUs) { + log.log( + `[ingest] caught up to present, stopping (last time_us=${event.time_us}, startTimeUs=${startTimeUs})` + ); + break; + } + + if (Date.now() >= deadline) { + log.log( + `[ingest] safety timeout reached, stopping (deadline=${deadline}, collected=${collected.length})` + ); + break; + } + } + + if (filteredUnknownDid > 0) { + const sample = [...filteredDidSamples].join(", "); + log.log( + `[ingest] ${filteredUnknownDid} events filtered (unknown did). sample dids: ${sample}` + ); + } + const lastCursor = subscription.cursor || null; + + const cursorGap = + lastCursor !== null && lastYieldedTimeUs !== null + ? lastCursor - lastYieldedTimeUs + : null; + + // Detect the library's internal cursor rollback (picks a different URL → rolls + // back 10s → first event comes in BEFORE the cursor we asked it to start from). + const rolledBackUs = + cursor !== null && firstYieldedTimeUs !== null && firstYieldedTimeUs < cursor + ? cursor - firstYieldedTimeUs + : 0; + + log.log( + `[ingest] jetstream loop done. commits_seen=${totalCommits}, filtered=${filteredUnknownDid}, kept=${collected.length}, dupes=${duplicateUris.length}, connects=${connectCount}, first_yielded=${firstYieldedTimeUs ?? "none"}, last_yielded=${lastYieldedTimeUs ?? "none"}, subscription_cursor=${lastCursor ?? "none"}, cursor_gap=${cursorGap ?? "n/a"}us, rolled_back=${rolledBackUs}us` + ); + + if (cursorGap !== null && cursorGap > 1000) { + log.warn( + `[ingest] CURSOR GAP: subscription cursor is ${cursorGap}us (${Math.floor( + cursorGap / 1000 + )}ms) ahead of last yielded event — buffered events may be dropped` + ); + } + + if (connectCount > 1) { + log.warn( + `[ingest] RECONNECTED ${connectCount} times during cycle — each reconnect picks a URL at random and rolls cursor back 10s` + ); + } + + return { events: collected, lastCursor }; +} + +// Run a full ingest cycle: init schema, load cursor, ingest, apply, save cursor +export async function runIngestCycle( + db: Database, + config: ContrailConfig, + timeoutMs: number = 25_000, + state?: IngestState, + pubsub?: import("./realtime/types").PubSub +): Promise<void> { + const log = getLogger(config); + const s = state ?? createIngestState(); + + if (!s.schemaInitialized) { + await initSchema(db, config); + s.schemaInitialized = true; + } + + const cursor = await getLastCursor(db); + const collections = getCollectionNsids(config); + const nowUs = Date.now() * 1000; + const lagMs = cursor !== null ? Math.floor((nowUs - cursor) / 1000) : null; + + log.log( + `[ingest] starting cycle. cursor=${cursor ?? "none"}${ + lagMs !== null ? ` (lag=${lagMs}ms)` : "" + }, timeout=${timeoutMs}ms, collections=${collections.join(", ")}` + ); + + // Load known DIDs for filtering dependent collections + const dependentCollections = getDependentNsids(config); + let knownDids: Set<string> | undefined; + + if (dependentCollections.length > 0) { + if (s.cachedKnownDids) { + knownDids = s.cachedKnownDids; + log.log(`Using cached known DIDs (${knownDids.size} users)`); + } else { + const result = await db + .prepare("SELECT did FROM identities") + .all<{ did: string }>(); + knownDids = new Set((result.results ?? []).map((r) => r.did)); + s.cachedKnownDids = knownDids; + log.log(`Loaded ${knownDids.size} known DIDs from database`); + } + } + + const { events, lastCursor } = await ingestEvents( + config, + cursor, + timeoutMs, + knownDids + ); + + if (events.length > 0) { + const breakdown: Record<string, number> = {}; + for (const e of events) { + const key = `${e.collection}:${e.operation}`; + breakdown[key] = (breakdown[key] ?? 0) + 1; + } + log.log( + `[ingest] received ${events.length} events. breakdown=${JSON.stringify(breakdown)}` + ); + } else { + log.log(`[ingest] received 0 events from Jetstream`); + } + + for (let i = 0; i < events.length; i += BATCH_SIZE) { + const batch = events.slice(i, i + BATCH_SIZE); + await applyEvents(db, batch, config, { pubsub }); + } + + // Refresh stale/missing identities for DIDs in this batch + const uniqueDids = [...new Set(events.map((e) => e.did))]; + if (uniqueDids.length > 0) { + try { + await refreshStaleIdentities(db, uniqueDids); + } catch (err) { + log.warn(`Identity refresh failed: ${err}`); + } + } + + if (lastCursor !== null) { + await saveCursor(db, lastCursor); + log.log( + `[ingest] saved cursor=${lastCursor} (advanced ${ + cursor !== null ? lastCursor - cursor : "n/a" + }us)` + ); + } else { + log.log(`[ingest] no cursor returned from subscription; not saving`); + } + + // Prune feed items hourly + if (config.feeds && Date.now() - s.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { + const maxItems = Math.max( + ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) + ); + const pruned = await pruneFeedItems(db, maxItems); + if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); + s.lastFeedPruneMs = Date.now(); + } + + log.log(`[ingest] cycle complete. stored=${events.length}`); +} diff --git a/packages/contrail-appview/src/core/labels/apply.ts b/packages/contrail-appview/src/core/labels/apply.ts new file mode 100644 index 0000000..cbf5558 --- /dev/null +++ b/packages/contrail-appview/src/core/labels/apply.ts @@ -0,0 +1,64 @@ +import type { Database, Statement } from "../types"; + +/** Wire shape of a single `com.atproto.label.defs#label` entry. Field names + * match the spec exactly. We accept the spec's ISO-8601 strings and + * convert to unix seconds at the storage boundary. */ +export interface IncomingLabel { + src: string; + uri: string; + val: string; + cid?: string; + neg?: boolean; + exp?: string; + cts: string; + sig?: Uint8Array; +} + +/** Upsert a batch of labels. Idempotent on `(src, uri, val, cts)`. Bad rows + * (missing required fields, unparseable timestamps) are dropped silently; + * we don't want one malformed label to abort an entire labeler frame. */ +export async function applyLabels( + db: Database, + labels: IncomingLabel[], +): Promise<number> { + if (labels.length === 0) return 0; + const stmts: Statement[] = []; + let kept = 0; + for (const l of labels) { + if (!l.src || !l.uri || !l.val || !l.cts) continue; + const cts = isoToUnixSec(l.cts); + if (cts == null) continue; + const exp = l.exp ? isoToUnixSec(l.exp) : null; + stmts.push( + db + .prepare( + `INSERT INTO labels (src, uri, val, cid, neg, exp, cts, sig) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(src, uri, val, cts) DO UPDATE SET + cid = excluded.cid, + neg = excluded.neg, + exp = excluded.exp, + sig = excluded.sig`, + ) + .bind( + l.src, + l.uri, + l.val, + l.cid ?? null, + l.neg ? 1 : 0, + exp, + cts, + l.sig ?? null, + ), + ); + kept++; + } + if (stmts.length > 0) await db.batch(stmts); + return kept; +} + +function isoToUnixSec(iso: string): number | null { + const ms = Date.parse(iso); + if (!Number.isFinite(ms)) return null; + return Math.floor(ms / 1000); +} diff --git a/packages/contrail-appview/src/core/labels/hydrate.ts b/packages/contrail-appview/src/core/labels/hydrate.ts new file mode 100644 index 0000000000000000000000000000000000000000..02b947a44ef764f5c31f64fefd468727f76fd7d1 GIT binary patch literal 3547 zcmd1IEyyn_Q7EY_NL8p-a7iplOiC<HRj5@+E6UGRP}0-W2T2txS#v>ERx2bWmLzAS zrg-KBmZla}A}cM*FD*$e($7fEDM&2>>EY7X(ozV|EK1ca&PXgsRY=P(Qb<(Ds7xtJ zEJ;mK$Vp5}%~5#NFhwC3Y-X`SLUMktUSdf>QGQ9j9#~p0B{i*B8O%)JQqWRRNXtyk zNzqNqFVY3G6pHf|lJoP5OLJ3;iWQO*^Arjai;ER9OB9MriV`!^GfEUnGK%s`(=!xG z@)e*)D+IfE7wai#>2sx4fc>ADSCU$kmYAHX-~n}<57=$hTnY-sMafnm!!q;It+^Bw zN{cd)xn+qt$lT=26nkVoC;<HPaw-)Hic*VH^GX!TGg9*uN{UKT!PYA%<fW$DTY&{Y zhJ%!Q2K(z;m>C%ADCFm6mVm+utS2vDA+@3)v#3%bvsj@xwFIg^wW0vSLYN5($wjG& zC7Jno3MHAjsl_FUxdl*N$tA@ISJrZIA^abdSPu4jP=2`rIlhM(t5BnmSDKRp@>E`G zx|Kp+X>L+#5r~^wQ2^s2DFV9)rl^(+oO0Y!OOi7nsiYX1c8XI=6!Oy)ic6ESQj<%H zbrjMvb4pT+KyjO#n3I!Qq+YC$n4Fwi017v7s!1&>)&VD+<oukR#Dd~fg#?Y_qGTOV zMCvG%CFW=*D3p{I<fIlWBvmTpB$lKWmnbA8mlP-HC?w{kD5MnS7r+#hXXF>Bf(?VJ z%Pdw%R7l86O}7O_X@WvgYDrOIGAPLP6oOJqN{jM93JOw-bfGR*$W1KJ<5JK98xjol z1vuFzr79$r=!OP)7AxczDI_Hpr7F01x)g)6M@dGiLUCelszOO(QckKuMt)98u|iUQ zNd`DNQz2<MvA8lXSs|@74{Sdymw_X*Sc6MJAtlKQp22lM=^AP~EXJZ^L87q0LlrGb zP0lY$an4L}sthg4w6{|5O)Rh}E-A{)OV@$s4QO22>2PUUDFhVd=VlhC+6001!BwJU zuIN}hJ8%-qOjFQ+22ZhGPHJ9yNrr;0t*wHALQM@kGEpQnK>=M_l&4T#Yt6-_prEg> z;F6kBT967)7^uPK;_0H0n^>ukSWu9fSfr4dr;w16nNpkpj^M;%1yJ}x0v}>nVrfZ6 zevvg;931WtNw68o`MIf((lIBqxI{r47SMWJ3gCjMM4>b<vk)G&whGaDdU|=O<qE;6 zC2((Q#)5JJsN8}mcZ4NbROPV904awXUyzfSoC-<;paRbpY#U6wUT$K6hK8nst(}6B zy^^M0R(@ulhLVnwCc?J7{PN(`WCdFV-^7v(y|kSC{2~pP#FA9Ky!>(vO$B`gLjwZ? zO<1TpW#*-1=A{=alw~HO7yOAusa9am6%?hWWmf1Y=;`StXO!k;$Lc8rl!A)gRE31p zih=}4m_jqER#IwOeo?9xIBXN*<30W2;}aB+`~)h{LGh6Uv05R&C?&NBRzX0F0!ar4 z`k?3nD?`LVQGR)`f~`Vgd17V>n!|07ia0w^#sFmvP|{aWNC<ZIadi#>75RwrK}R7u zGerkfQ0st7U>$|zl41q7Ab($I$;71q0pT95L9QUJ3Z8xn8Y<ODek!ijgsD<caP)Ig z&`7N)Q1A>^@C)_vQSc7}nX6!@V2`G%v?vp779^B36CjQRr6*7lgGIWI0whks*;fIa z9$={tqyZk60h!6!;82H_T}Aoj3I(Y}n8l%<LP%z6u|j?xq!?5vN-fUMDN9vIs)PoA zVs>U;I@l<X86_EsB?^f}MVV#bDjifBXXb%r3&9P9qSWHjoDxVumI6{$q??jjkeZj0 znpXlbFsZaep$JsqW#%aqfXmP1RBMIeg4AS%l>F4<JoOTVl+?7$yi|pf%v9Z^qExU0 zK#>h<oG5swRw~#+OCdx;hxpQ|5+V!AfS@wVJGD|r0VM%|^9HD4q5;W{ATQ;oDS#4? zCb)P~fM)A#ur&!P)kXQ`dc{S_wG0quX;CJOS(cbnn*eeGOiw{kW`2=^EwowzJ4P=( zwL~L3wGwQvf<k7Rf`%eQSxpVdRy|Piva?kHi-9<rD7uRwx^+NG!A8`A3P6a{^Gi#t z6bKZ#3bx=vQvqC`qIwxA#PrG%b4pW-H8f#C0CFbC-+8I&nxOC~$;>NFwFVWu&?+xA z1>_EJQS9vLqEL{TXQhw`$sr2h6q1*in+mE7LD8C-mYJ*osZ$g{suK$eav(`Ar&6IL zADmb8^&$GoQj3Z+^Yio+Jk!953R2C3TY0G|3W-o-bdxhvKy@p)L<f}*Mfv6G#V~_F zcBEt$C#DyrrWWf!s`ShfcpU+5C%}^$$T1*SfNg`?4l}4cRlO(`TKQz=rRym;CMV~Y z=9TE?q?V=TK)hS5paH6Hpf#GNLULkJQKdp&KFFsqa|$x^K(!Dev6rVR<b#@t$r-81 z+0aHJsM3ej(F)0+%D+6bBtsz)WDF?jLy`_CPC#B)Q-cNstd53794sY5LfbhrMZp$L zB{)HX%qcC(1g9uaaR!cWaODlsqG$`MXCQenMFH6^1&|a-b8==1Qo;cH6Pz+(_JhON zN&%$>07{EsmC!~u$Q)493@Qe0X@kT{i!wna8OUAW<~B&IEHMWr25vf*6o=${7Uye# z<dRE@H6fC<kjx17O@6r^*pXnDfS90=fC!?h0+r2*w&3O~DDNeu=IDWhY!#60NUbP< z29`#CX-PE5+R~!TSOt4~TZQOYO}&EB;tUP2RbXY{uqXnz2o>^6ORTwSxwzm>HJG`j wd6^Z#smZX`u_mky1}=u6nn48_sJ_;KDpt^f)Yy6@`JTc4!QdvDh9<~-0O=5WfdBvi literal 0 HcmV?d00001 diff --git a/packages/contrail-appview/src/core/labels/resolve.ts b/packages/contrail-appview/src/core/labels/resolve.ts new file mode 100644 index 0000000..694bfec --- /dev/null +++ b/packages/contrail-appview/src/core/labels/resolve.ts @@ -0,0 +1,134 @@ +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, +} from "@atcute/identity-resolver"; +import type { Did } from "@atcute/lexicons"; +import type { Database } from "../types"; + +/** Reject endpoint URLs that point to private/internal addresses or non-HTTPS. + * Mirrors the validator in core/client.ts — labeler endpoints should be + * publicly reachable for the same reasons PDS endpoints should. */ +function validateEndpointUrl(url: string): boolean { + try { + const parsed = new URL(url); + if (parsed.protocol !== "https:") return false; + const host = parsed.hostname; + if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; + if (host.startsWith("10.")) return false; + if (host.startsWith("192.168.")) return false; + if (host.startsWith("169.254.")) return false; + if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; + return true; + } catch { + return false; + } +} + +const didResolver = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, +}); + +/** Look up the labeler service endpoint from a DID. + * Reads the DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ +export async function resolveLabelerEndpoint(did: string): Promise<string | null> { + if (!did.startsWith("did:plc:") && !did.startsWith("did:web:")) return null; + try { + const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); + const endpoint = doc.service + ?.find((s) => s.id === "#atproto_labeler") + ?.serviceEndpoint?.toString(); + if (!endpoint) return null; + if (!validateEndpointUrl(endpoint)) return null; + return endpoint; + } catch { + return null; + } +} + +/** State row for a labeler — the per-DID equivalent of the singleton + * jetstream `cursor` table, with cached endpoint to avoid repeated DID-doc + * fetches. */ +export interface LabelerState { + did: string; + cursor: number; + endpoint: string | null; + resolved_at: number | null; +} + +const ENDPOINT_TTL_MS = 6 * 60 * 60 * 1000; // 6h, matches the recommended client cache for label-defs + +/** Get cached `(endpoint, cursor)` for a labeler. Resolves endpoint on + * cache miss or staleness; persists endpoint + resolved_at back to the DB + * so subsequent ingest cycles avoid the network round-trip. */ +export async function getLabelerState( + db: Database, + did: string, + endpointOverride: string | undefined, +): Promise<LabelerState | null> { + const row = await db + .prepare( + "SELECT did, cursor, endpoint, resolved_at FROM labeler_cursors WHERE did = ?", + ) + .bind(did) + .first<LabelerState>(); + + let endpoint = endpointOverride ?? row?.endpoint ?? null; + const stale = + !row?.resolved_at || Date.now() - row.resolved_at > ENDPOINT_TTL_MS; + + if (!endpoint || (!endpointOverride && stale)) { + endpoint = await resolveLabelerEndpoint(did); + if (!endpoint) return null; + const now = Date.now(); + await db + .prepare( + `INSERT INTO labeler_cursors (did, cursor, endpoint, resolved_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(did) DO UPDATE SET endpoint = excluded.endpoint, resolved_at = excluded.resolved_at`, + ) + .bind(did, row?.cursor ?? 0, endpoint, now) + .run(); + return { + did, + cursor: row?.cursor ?? 0, + endpoint, + resolved_at: now, + }; + } + + return row ?? { did, cursor: 0, endpoint, resolved_at: null }; +} + +/** Persist the highest seen seq number for a labeler. Idempotent; + * the next ingest cycle resumes from `cursor + 1` via the `?cursor=` param. */ +export async function saveLabelerCursor( + db: Database, + did: string, + cursor: number, +): Promise<void> { + await db + .prepare( + `INSERT INTO labeler_cursors (did, cursor) + VALUES (?, ?) + ON CONFLICT(did) DO UPDATE SET cursor = excluded.cursor`, + ) + .bind(did, cursor) + .run(); +} + +/** Reset cursor to 0 — used in response to `#info { name: "OutdatedCursor" }` + * frames, which signal that the labeler's seq history was rewound. */ +export async function resetLabelerCursor(db: Database, did: string): Promise<void> { + await db + .prepare( + `INSERT INTO labeler_cursors (did, cursor) + VALUES (?, 0) + ON CONFLICT(did) DO UPDATE SET cursor = 0`, + ) + .bind(did) + .run(); +} diff --git a/packages/contrail-appview/src/core/labels/schema.ts b/packages/contrail-appview/src/core/labels/schema.ts new file mode 100644 index 0000000..486ce27 --- /dev/null +++ b/packages/contrail-appview/src/core/labels/schema.ts @@ -0,0 +1,30 @@ +import type { SqlDialect } from "../dialect"; + +/** DDL for the labels module. Single `labels` table covers record-level + * (uri starts with `at://`) and account-level (uri is a bare DID) entries — + * the spec collapses both into the same row shape. `labeler_cursors` + * mirrors the role of the singleton `cursor` table for jetstream, but + * per-labeler. */ +export function buildLabelsSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS labels ( + src TEXT NOT NULL, + uri TEXT NOT NULL, + val TEXT NOT NULL, + cid TEXT, + neg INTEGER NOT NULL DEFAULT 0, + exp ${dialect.bigintType}, + cts ${dialect.bigintType} NOT NULL, + sig BLOB, + PRIMARY KEY (src, uri, val, cts) + )`, + `CREATE INDEX IF NOT EXISTS idx_labels_uri ON labels(uri)`, + `CREATE INDEX IF NOT EXISTS idx_labels_src_cts ON labels(src, cts DESC)`, + `CREATE TABLE IF NOT EXISTS labeler_cursors ( + did TEXT PRIMARY KEY, + cursor ${dialect.bigintType} NOT NULL DEFAULT 0, + endpoint TEXT, + resolved_at ${dialect.bigintType} + )`, + ]; +} diff --git a/packages/contrail-appview/src/core/labels/select.ts b/packages/contrail-appview/src/core/labels/select.ts new file mode 100644 index 0000000..da89d9c --- /dev/null +++ b/packages/contrail-appview/src/core/labels/select.ts @@ -0,0 +1,64 @@ +import type { LabelsConfig } from "./types"; +import { DEFAULT_LABELS_MAX_PER_REQUEST } from "./types"; + +/** Pick which labelers to honor for this request. + * + * Order of precedence: + * 1. `atproto-accept-labelers` header (atproto spec) + * 2. `?labelers=` query param (fallback for SSE/WS where headers are awkward) + * 3. `config.defaults` (operator policy) + * 4. every entry in `config.sources` + * + * Each candidate DID is checked against `config.sources`. Unknowns are + * dropped — we only have rows for labelers we've subscribed to. + * + * Header values can carry `;param` modifiers (e.g. `did:plc:...;redact`); + * v1 strips and ignores those — only the bare DID is honored. */ +export interface SelectedLabelers { + /** DIDs to use for hydration this request. */ + accepted: string[]; +} + +export function selectAcceptedLabelers( + headerValue: string | null | undefined, + paramValue: string | null | undefined, + cfg: LabelsConfig, +): SelectedLabelers { + const cap = cfg.maxPerRequest ?? DEFAULT_LABELS_MAX_PER_REQUEST; + const known = new Set(cfg.sources.map((s) => s.did)); + + const fromCaller = parseLabelerList(headerValue) ?? parseLabelerList(paramValue); + + let candidates: string[]; + if (fromCaller && fromCaller.length > 0) { + candidates = fromCaller; + } else { + candidates = (cfg.defaults ?? cfg.sources.map((s) => s.did)).slice(); + } + + const accepted: string[] = []; + const seen = new Set<string>(); + for (const did of candidates) { + if (seen.has(did)) continue; + seen.add(did); + if (known.has(did)) accepted.push(did); + if (accepted.length >= cap) break; + } + + return { accepted }; +} + +/** Parse a comma-separated DID list. Returns null when the input is empty + * or undefined so callers can distinguish "absent" from "empty list" (the + * latter — `atproto-accept-labelers: ` — is technically valid and means + * "no labelers"; we treat it the same as absent for ergonomics). */ +function parseLabelerList(value: string | null | undefined): string[] | null { + if (!value) return null; + const out: string[] = []; + for (const raw of value.split(",")) { + // Drop `;param` modifiers from the spec (e.g. `;redact`). v1 ignores them. + const head = raw.split(";")[0]!.trim(); + if (head.startsWith("did:")) out.push(head); + } + return out.length > 0 ? out : null; +} diff --git a/packages/contrail-appview/src/core/labels/subscribe.ts b/packages/contrail-appview/src/core/labels/subscribe.ts new file mode 100644 index 0000000..c80ed2b --- /dev/null +++ b/packages/contrail-appview/src/core/labels/subscribe.ts @@ -0,0 +1,315 @@ +import { decodeFirst } from "@atcute/cbor"; +import type { ContrailConfig, Database, Logger } from "../types"; +import type { LabelerSource } from "./types"; +import { applyLabels, type IncomingLabel } from "./apply"; +import { + getLabelerState, + resetLabelerCursor, + saveLabelerCursor, +} from "./resolve"; + +const DEFAULT_CYCLE_TIMEOUT_MS = 25_000; +const DEFAULT_BATCH_SIZE = 100; +const DEFAULT_FLUSH_INTERVAL_MS = 5_000; + +function getLogger(config: ContrailConfig): Logger { + return config.logger ?? console; +} + +/** One catch-up cycle for every configured labeler. Designed to fit inside a + * Cloudflare Workers cron tick — we drain frames until the labeler has no + * more buffered events for us, or `timeoutMs` is reached, then save cursor + * and disconnect. Mirrors the shape of `runIngestCycle` for jetstream. */ +export async function runLabelIngestCycle( + db: Database, + config: ContrailConfig, + timeoutMs = DEFAULT_CYCLE_TIMEOUT_MS, +): Promise<void> { + if (!config.labels) return; + const log = getLogger(config); + const deadline = Date.now() + timeoutMs; + + for (const source of config.labels.sources) { + if (Date.now() >= deadline) { + log.log(`[labels] cycle deadline hit before processing ${source.did}`); + break; + } + const remaining = Math.max(2_000, deadline - Date.now()); + try { + await pumpOneLabeler(db, source, log, remaining, /* persistent */ false); + } catch (err) { + log.warn(`[labels] cycle for ${source.did} failed: ${err}`); + } + } +} + +export interface PersistentLabelsOptions { + signal?: AbortSignal; + batchSize?: number; + flushIntervalMs?: number; + logger?: Logger; +} + +/** Long-lived equivalent — keeps one socket per labeler open forever, with + * exponential backoff reconnect. Mirrors `runPersistent` for jetstream. */ +export async function runPersistentLabels( + db: Database, + config: ContrailConfig, + options: PersistentLabelsOptions = {}, +): Promise<void> { + if (!config.labels) return; + const log = options.logger ?? config.logger ?? console; + const signal = options.signal; + + const tasks = config.labels.sources.map((source) => + runOneLabelerForever(db, source, log, signal, options), + ); + await Promise.all(tasks); +} + +async function runOneLabelerForever( + db: Database, + source: LabelerSource, + log: Logger, + signal: AbortSignal | undefined, + options: PersistentLabelsOptions, +): Promise<void> { + let attempts = 0; + while (!signal?.aborted) { + try { + await pumpOneLabeler(db, source, log, /* timeoutMs */ Infinity, true, { + signal, + batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE, + flushIntervalMs: options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, + }); + attempts = 0; + } catch (err) { + if (signal?.aborted) break; + log.error(`[labels] ${source.did} stream error: ${err}`); + const delay = Math.min(1_000 * 2 ** attempts, 30_000); + attempts++; + log.log(`[labels] ${source.did} reconnecting in ${delay}ms (attempt ${attempts})`); + await new Promise((r) => setTimeout(r, delay)); + } + } +} + +interface PumpOptions { + signal?: AbortSignal; + batchSize?: number; + flushIntervalMs?: number; +} + +/** Open a `subscribeLabels` WebSocket, drain frames into a buffer, flush + * the buffer to `labels` in batches, and persist the seq cursor. Returns + * when: + * - the labeler closes the socket cleanly (caught up + no more events) + * - `timeoutMs` is reached (cron mode) + * - `signal` is aborted (persistent mode) + * - an error tears the socket down (caller may retry) */ +async function pumpOneLabeler( + db: Database, + source: LabelerSource, + log: Logger, + timeoutMs: number, + persistent: boolean, + pumpOpts: PumpOptions = {}, +): Promise<void> { + const state = await getLabelerState(db, source.did, source.endpoint); + if (!state) { + log.warn(`[labels] could not resolve labeler endpoint for ${source.did}; skipping`); + return; + } + + // First-time policy: cursor 0 = "from the beginning" if backfill is on + // (default), null = "from now" otherwise. After the first cycle we always + // resume from the saved cursor — `backfill` only flips the start point. + const isFirstRun = state.cursor === 0 && state.resolved_at === null; + const backfill = source.backfill !== false; + const startCursor = isFirstRun && !backfill ? null : state.cursor; + + const url = buildWsUrl(state.endpoint!, startCursor); + log.log(`[labels] connecting to ${source.did} (cursor=${startCursor ?? "now"})`); + + const ws = new WebSocket(url); + ws.binaryType = "arraybuffer"; + + const buffer: IncomingLabel[] = []; + let highestSeq = state.cursor; + let flushing = false; + let resolveDone!: () => void; + let rejectDone!: (err: unknown) => void; + const done = new Promise<void>((res, rej) => { + resolveDone = res; + rejectDone = rej; + }); + + const flush = async () => { + if (buffer.length === 0 || flushing) return; + flushing = true; + const batch = buffer.splice(0); + try { + const kept = await applyLabels(db, batch); + if (highestSeq > state.cursor) { + await saveLabelerCursor(db, source.did, highestSeq); + state.cursor = highestSeq; + } + log.log( + `[labels] ${source.did} flushed ${kept}/${batch.length} labels, cursor=${highestSeq}`, + ); + } catch (err) { + log.error(`[labels] ${source.did} flush failed: ${err}`); + } finally { + flushing = false; + } + }; + + const batchSize = pumpOpts.batchSize ?? DEFAULT_BATCH_SIZE; + const flushInterval = pumpOpts.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS; + const flushTimer = setInterval(() => { + flush().catch(() => {}); + }, flushInterval); + + const cleanup = () => { + clearInterval(flushTimer); + try { + ws.close(); + } catch { + /* already closed */ + } + }; + + // External abort (persistent mode) — close socket gracefully. + const abortHandler = () => { + cleanup(); + flush().finally(() => resolveDone()); + }; + pumpOpts.signal?.addEventListener("abort", abortHandler, { once: true }); + + // Cron-mode time budget — close socket gracefully when reached. + let deadlineTimer: ReturnType<typeof setTimeout> | undefined; + if (Number.isFinite(timeoutMs)) { + deadlineTimer = setTimeout(() => { + log.log(`[labels] ${source.did} cycle deadline reached, closing`); + cleanup(); + flush().finally(() => resolveDone()); + }, timeoutMs); + } + + ws.addEventListener("error", (ev) => { + cleanup(); + if (deadlineTimer) clearTimeout(deadlineTimer); + pumpOpts.signal?.removeEventListener("abort", abortHandler); + rejectDone(new Error(`WebSocket error: ${(ev as ErrorEvent)?.message ?? "unknown"}`)); + }); + + ws.addEventListener("close", () => { + if (deadlineTimer) clearTimeout(deadlineTimer); + pumpOpts.signal?.removeEventListener("abort", abortHandler); + flush().finally(() => { + clearInterval(flushTimer); + resolveDone(); + }); + }); + + ws.addEventListener("message", async (ev) => { + let bytes: Uint8Array; + if (ev.data instanceof ArrayBuffer) { + bytes = new Uint8Array(ev.data); + } else if (ev.data instanceof Uint8Array) { + bytes = ev.data; + } else { + // Binary-only protocol — text frames shouldn't arrive. + return; + } + const frame = decodeFrame(bytes); + if (!frame) return; + + if (frame.t === "#labels") { + const seq = Number(frame.payload?.seq ?? 0); + const rawLabels = Array.isArray(frame.payload?.labels) ? frame.payload.labels : []; + for (const raw of rawLabels) { + const lab = normalizeLabel(raw, source.did); + if (lab) buffer.push(lab); + } + if (Number.isFinite(seq) && seq > highestSeq) highestSeq = seq; + if (buffer.length >= batchSize) { + flush().catch(() => {}); + } + } else if (frame.t === "#info") { + const name = String(frame.payload?.name ?? ""); + log.log(`[labels] ${source.did} info: ${name}`); + if (name === "OutdatedCursor") { + // Labeler rewound its log — discard our cursor and let the next + // run start from the beginning. We don't reconnect here; the + // caller (or the persistent loop) will pick up the reset on retry. + await resetLabelerCursor(db, source.did); + cleanup(); + } + } else if (frame.op === -1) { + log.warn(`[labels] ${source.did} error frame: ${JSON.stringify(frame.payload)}`); + cleanup(); + } + }); + + // Workers WebSocket doesn't always emit `open`; just await `done` directly. + await done; +} + +function buildWsUrl(httpEndpoint: string, cursor: number | null): string { + const u = new URL("/xrpc/com.atproto.label.subscribeLabels", httpEndpoint); + // wss:// for HTTPS endpoints — the protocol on the labeler service is + // expected to be HTTPS already (validated at resolution time). + u.protocol = u.protocol === "https:" ? "wss:" : "ws:"; + if (cursor !== null) u.searchParams.set("cursor", String(cursor)); + return u.toString(); +} + +interface DecodedFrame { + op: number; + t: string | undefined; + payload: Record<string, unknown>; +} + +/** Decode an atproto subscription frame: two consecutive CBOR objects. + * Header `{ op, t? }`, payload — shape depends on `t`. Returns null on + * decode failure or non-object frames. */ +function decodeFrame(bytes: Uint8Array): DecodedFrame | null { + try { + const [header, rest] = decodeFirst(bytes); + if (!header || typeof header !== "object") return null; + const op = typeof (header as { op?: number }).op === "number" ? (header as { op: number }).op : 1; + const t = typeof (header as { t?: string }).t === "string" ? (header as { t: string }).t : undefined; + const [payload] = decodeFirst(rest); + if (!payload || typeof payload !== "object") return null; + return { op, t, payload: payload as Record<string, unknown> }; + } catch { + return null; + } +} + +/** Coerce a wire `Label` object into our `IncomingLabel` shape. Returns + * null when required fields are missing — we'd rather skip a row than + * insert one with placeholder values. */ +function normalizeLabel(raw: unknown, expectedSrc: string): IncomingLabel | null { + if (!raw || typeof raw !== "object") return null; + const r = raw as Record<string, unknown>; + const src = typeof r.src === "string" ? r.src : null; + const uri = typeof r.uri === "string" ? r.uri : null; + const val = typeof r.val === "string" ? r.val : null; + const cts = typeof r.cts === "string" ? r.cts : null; + if (!src || !uri || !val || !cts) return null; + // A labeler shouldn't emit labels under a different `src` than its own + // DID — drop them rather than poison our table with cross-issuer rows. + if (src !== expectedSrc) return null; + return { + src, + uri, + val, + cts, + cid: typeof r.cid === "string" ? r.cid : undefined, + neg: r.neg === true, + exp: typeof r.exp === "string" ? r.exp : undefined, + sig: r.sig instanceof Uint8Array ? r.sig : undefined, + }; +} diff --git a/packages/contrail-appview/src/core/labels/types.ts b/packages/contrail-appview/src/core/labels/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/labels/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/persistent.ts b/packages/contrail-appview/src/core/persistent.ts new file mode 100644 index 0000000..c1e1c18 --- /dev/null +++ b/packages/contrail-appview/src/core/persistent.ts @@ -0,0 +1,243 @@ +import type { JetstreamSubscription } from "@atcute/jetstream"; +import type { ContrailConfig, IngestEvent, Database, Logger, ResolvedContrailConfig } from "./types"; +import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS, resolveConfig } from "./types"; +import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; +import { refreshStaleIdentities } from "./identity"; +import { createIngestState } from "./jetstream"; +import type { IngestState } from "./jetstream"; + +const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; + +export interface PersistentIngestOptions { + batchSize?: number; + flushIntervalMs?: number; + signal?: AbortSignal; + /** Override subscription creation for testing */ + createSubscription?: (cursor: number | null) => JetstreamSubscription; + logger?: Logger; + /** Publish `collection:<nsid>` / `actor:<did>` events for each applied + * public record. Usually supplied by the Contrail instance. */ + pubsub?: import("./realtime/types").PubSub; +} + +function getLogger(config: ContrailConfig, options?: PersistentIngestOptions): Logger { + return options?.logger ?? config.logger ?? console; +} + +export async function runPersistent( + db: Database, + config: ContrailConfig, + options?: PersistentIngestOptions, +): Promise<void> { + // Internals (applyEvents, count updates, query planning) read `_resolved` + // and silently skip features when it's missing. The Contrail class resolves + // in its constructor; callers using this raw export must also get a resolved + // config, so do it defensively here. resolveConfig is idempotent. + if (!(config as ResolvedContrailConfig)._resolved) { + config = resolveConfig(config); + } + const log = getLogger(config, options); + const batchSize = options?.batchSize ?? 50; + const flushIntervalMs = options?.flushIntervalMs ?? 5_000; + const signal = options?.signal; + const state = createIngestState(); + + // Init schema once + if (!state.schemaInitialized) { + await initSchema(db, config); + state.schemaInitialized = true; + } + + // Load known DIDs for dependent collection filtering + const dependentCollections: Set<string> = new Set(getDependentNsids(config)); + let knownDids: Set<string> | undefined; + if (dependentCollections.size > 0) { + const result = await db + .prepare("SELECT did FROM identities") + .all<{ did: string }>(); + knownDids = new Set((result.results ?? []).map((r) => r.did)); + state.cachedKnownDids = knownDids; + log.log(`Loaded ${knownDids.size} known DIDs from database`); + } + + const collections = getCollectionNsids(config); + let reconnectAttempts = 0; + + while (!signal?.aborted) { + const cursor = await getLastCursor(db); + log.log(`Starting persistent ingestion. Cursor: ${cursor ?? "none"}, Collections: ${collections.join(", ")}`); + + try { + await streamAndFlush(db, config, cursor, { + batchSize, + flushIntervalMs, + signal, + collections, + dependentCollections, + knownDids, + state, + log, + createSubscription: options?.createSubscription, + pubsub: options?.pubsub, + }); + reconnectAttempts = 0; + } catch (err) { + if (signal?.aborted) break; + log.error(`Jetstream connection error: ${err}`); + const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30_000); + reconnectAttempts++; + log.log(`Reconnecting in ${delay}ms (attempt ${reconnectAttempts})...`); + await new Promise((r) => setTimeout(r, delay)); + } + } + + log.log("Persistent ingestion stopped"); +} + +interface StreamOptions { + batchSize: number; + flushIntervalMs: number; + signal?: AbortSignal; + collections: string[]; + dependentCollections: Set<string>; + knownDids?: Set<string>; + state: IngestState; + log: Logger; + createSubscription?: (cursor: number | null) => any; + pubsub?: import("./realtime/types").PubSub; +} + +async function streamAndFlush( + db: Database, + config: ContrailConfig, + cursor: number | null, + opts: StreamOptions, +): Promise<void> { + const { batchSize, flushIntervalMs, signal, collections, dependentCollections, knownDids, state, log } = opts; + + const subscription = opts.createSubscription + ? opts.createSubscription(cursor) + : new (await import("@atcute/jetstream")).JetstreamSubscription({ + url: config.jetstreams ?? [], + wantedCollections: collections, + ...(cursor !== null ? { cursor } : {}), + onConnectionOpen() { log.log("Connected to Jetstream"); }, + onConnectionClose(event: any) { log.log(`Disconnected: ${event.code} ${event.reason}`); }, + onConnectionError(event: any) { log.error("Jetstream error:", event.error); }, + }); + + const buffer: IngestEvent[] = []; + // Guards against overlap between the periodic timer flush and a main-loop + // batchSize-driven flush. The main loop only ever awaits flush() sequentially, + // but the setInterval callback is a second entry point on another tick. + let flushing = false; + + const flush = async () => { + if (buffer.length === 0 || flushing) return; + flushing = true; + const batch = buffer.splice(0); + + try { + await applyEvents(db, batch, config, { pubsub: opts.pubsub }); + + const lastTimeUs = Math.max(...batch.map((e) => e.time_us)); + await saveCursor(db, lastTimeUs); + + const uniqueDids = [...new Set(batch.map((e) => e.did))]; + if (uniqueDids.length > 0) { + try { + await refreshStaleIdentities(db, uniqueDids); + } catch (err) { + log.warn(`Identity refresh failed: ${err}`); + } + } + + if (config.feeds && Date.now() - state.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { + const maxItems = Math.max( + ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) + ); + const pruned = await pruneFeedItems(db, maxItems); + if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); + state.lastFeedPruneMs = Date.now(); + } + + log.log(`Flushed ${batch.length} events. Cursor: ${lastTimeUs}`); + } finally { + flushing = false; + } + }; + + // Periodic flush decoupled from the main loop. Runs even when Jetstream is + // idle, which is the whole point — without it, buffered events strand until + // the next event or abort. Errors log and retry next interval rather than + // propagate, so transient DB hiccups don't force a reconnect. + const flushTimer = setInterval(() => { + flush().catch((err) => log.error(`Timer flush failed: ${err}`)); + }, flushIntervalMs); + + const onAbort = () => { + clearInterval(flushTimer); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + + const iterator = subscription[Symbol.asyncIterator](); + + try { + while (!signal?.aborted) { + // Per-iteration abort race so the handler can be removed synchronously + // after the race settles — otherwise addEventListener calls accumulate on + // the signal across the streamAndFlush lifetime. + let abortHandler!: () => void; + const abortPromise = new Promise<IteratorResult<any>>((resolve) => { + abortHandler = () => resolve({ value: undefined, done: true }); + signal?.addEventListener("abort", abortHandler, { once: true }); + }); + + let result: IteratorResult<any>; + try { + result = await Promise.race([iterator.next(), abortPromise]); + } finally { + signal?.removeEventListener("abort", abortHandler); + } + + if (result.done) break; + const event = result.value; + + if (event.kind === "commit") { + const { commit } = event; + + if (dependentCollections.has(commit.collection) && knownDids) { + if (!knownDids.has(event.did)) continue; + } + + const now = Date.now(); + const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; + + buffer.push({ + uri, + did: event.did, + time_us: event.time_us, + collection: commit.collection, + operation: commit.operation as "create" | "update" | "delete", + rkey: commit.rkey, + cid: commit.operation === "delete" ? null : commit.cid, + record: commit.operation === "delete" ? null : JSON.stringify(commit.record), + indexed_at: now * 1000, + }); + + if (knownDids && !dependentCollections.has(commit.collection)) { + knownDids.add(event.did); + } + } + + if (buffer.length >= batchSize) { + await flush(); + } + } + } finally { + clearInterval(flushTimer); + signal?.removeEventListener("abort", onAbort); + await iterator.return?.({ value: undefined, done: true }); + await flush(); + } +} diff --git a/packages/contrail-appview/src/core/realtime/durable-object.ts b/packages/contrail-appview/src/core/realtime/durable-object.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/durable-object.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/in-memory.ts b/packages/contrail-appview/src/core/realtime/in-memory.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/in-memory.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/index.ts b/packages/contrail-appview/src/core/realtime/index.ts new file mode 100644 index 0000000..05e39ec --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/index.ts @@ -0,0 +1,37 @@ +export { registerRealtimeRoutes } from "./router"; +export type { RealtimeRoutesOptions } from "./router"; +export { InMemoryPubSub } from "./in-memory"; +export { DurableObjectPubSub, RealtimePubSubDO } from "./durable-object"; +export type { + DurableObjectId, + DurableObjectNamespace, + DurableObjectStub, + DurableObjectState, +} from "./durable-object"; +export { TicketSigner } from "./ticket"; +export type { TicketPayload } from "./ticket"; +export { wrapWithPublishing } from "./publishing-adapter"; +export { sseResponse } from "./sse"; +export { pumpWebSocket } from "./websocket"; +export type { WebSocketLike } from "./websocket"; +export { mergeAsyncIterables } from "./merge"; +export { resolveTopicForCaller } from "./resolve"; +export type { TopicResolution, TopicResolutionContext, TopicResolutionError } from "./resolve"; +export type { + PubSub, + RealtimeConfig, + RealtimeEvent, + RealtimeEventKind, +} from "./types"; +export { + actorTopic, + collectionTopic, + communityTopic, + parseCommunityTopic, + parseSpaceTopic, + spaceTopic, + isCommunityTopic, + DEFAULT_KEEPALIVE_MS, + DEFAULT_QUEUE_BOUND, + DEFAULT_TICKET_TTL_MS, +} from "./types"; diff --git a/packages/contrail-appview/src/core/realtime/merge.ts b/packages/contrail-appview/src/core/realtime/merge.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/merge.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/publishing-adapter.ts b/packages/contrail-appview/src/core/realtime/publishing-adapter.ts new file mode 100644 index 0000000..c5dc61f --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/publishing-adapter.ts @@ -0,0 +1,157 @@ +/** Decorator that wraps a spaces StorageAdapter and publishes realtime events + * after successful writes. Spaces and community modules stay unaware of + * realtime; the decorator is the only integration seam. */ + +import type { StorageAdapter, SpaceMemberRow } from "../spaces/types"; +import type { PubSub, RealtimeEvent } from "./types"; +import { communityTopic, spaceTopic } from "./types"; + +export interface PublishingAdapterOptions { + /** Optional lookup: given a space's ownerDid, return true if that DID is a + * community in the local `communities` table. When provided, writes also + * publish to `community:<ownerDid>` so subscribers who expanded that alias + * at ticket-mint time receive the event. + * + * The lookup is expected to be cheap (cached in the caller) — the decorator + * calls it on every write. */ + isCommunityDid?: (did: string) => Promise<boolean> | boolean; +} + +export function wrapWithPublishing( + inner: StorageAdapter, + pubsub: PubSub, + opts: PublishingAdapterOptions = {} +): StorageAdapter { + const publishSpaceAndCommunity = async ( + spaceUri: string, + ownerDid: string | null, + build: (topic: string) => RealtimeEvent + ): Promise<void> => { + await pubsub.publish(build(spaceTopic(spaceUri))); + if (ownerDid && opts.isCommunityDid && (await opts.isCommunityDid(ownerDid))) { + await pubsub.publish(build(communityTopic(ownerDid))); + } + }; + + const ownerOf = async (spaceUri: string): Promise<string | null> => { + const s = await inner.getSpace(spaceUri); + return s?.ownerDid ?? null; + }; + + const wrapped: StorageAdapter = { + ...inner, + createSpace: inner.createSpace.bind(inner), + getSpace: inner.getSpace.bind(inner), + listSpaces: inner.listSpaces.bind(inner), + deleteSpace: inner.deleteSpace.bind(inner), + updateSpaceAppPolicy: inner.updateSpaceAppPolicy.bind(inner), + getMember: inner.getMember.bind(inner), + listMembers: inner.listMembers.bind(inner), + createInvite: inner.createInvite.bind(inner), + listInvites: inner.listInvites.bind(inner), + revokeInvite: inner.revokeInvite.bind(inner), + getInvite: inner.getInvite.bind(inner), + redeemInvite: inner.redeemInvite.bind(inner), + enroll: inner.enroll.bind(inner), + getEnrollment: inner.getEnrollment.bind(inner), + listEnrollments: inner.listEnrollments.bind(inner), + removeEnrollment: inner.removeEnrollment.bind(inner), + getRecord: inner.getRecord.bind(inner), + listRecords: inner.listRecords.bind(inner), + listCollections: inner.listCollections.bind(inner), + putBlobMeta: inner.putBlobMeta.bind(inner), + getBlobMeta: inner.getBlobMeta.bind(inner), + listBlobMeta: inner.listBlobMeta.bind(inner), + deleteBlobMeta: inner.deleteBlobMeta.bind(inner), + findOrphanBlobs: inner.findOrphanBlobs.bind(inner), + + async addMember(spaceUri, did, addedBy) { + await inner.addMember(spaceUri, did, addedBy); + const owner = await ownerOf(spaceUri); + const now = Date.now(); + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.added", + payload: { space: spaceUri, did }, + ts: now, + })); + }, + + async removeMember(spaceUri, did) { + await inner.removeMember(spaceUri, did); + const owner = await ownerOf(spaceUri); + const now = Date.now(); + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.removed", + payload: { space: spaceUri, did }, + ts: now, + })); + }, + + async applyMembershipDiff(spaceUri, adds, removes, addedBy) { + await inner.applyMembershipDiff(spaceUri, adds, removes, addedBy); + if (adds.length === 0 && removes.length === 0) return; + const owner = await ownerOf(spaceUri); + const now = Date.now(); + for (const did of adds) { + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.added", + payload: { space: spaceUri, did }, + ts: now, + })); + } + for (const did of removes) { + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.removed", + payload: { space: spaceUri, did }, + ts: now, + })); + } + }, + + async putRecord(record) { + await inner.putRecord(record); + const owner = await ownerOf(record.spaceUri); + const now = Date.now(); + // Space records use ms timestamps; listRecords surface uses microseconds + // (time_us). Convert here so subscribers can render a row identically. + const time_us = record.createdAt * 1000; + const uri = `at://${record.authorDid}/${record.collection}/${record.rkey}`; + await publishSpaceAndCommunity(record.spaceUri, owner, (topic) => ({ + topic, + kind: "record.created", + payload: { + uri, + did: record.authorDid, + collection: record.collection, + rkey: record.rkey, + cid: record.cid, + record: record.record, + time_us, + space: record.spaceUri, + }, + ts: now, + })); + }, + + async deleteRecord(spaceUri, collection, authorDid, rkey) { + await inner.deleteRecord(spaceUri, collection, authorDid, rkey); + const owner = await ownerOf(spaceUri); + const now = Date.now(); + const uri = `at://${authorDid}/${collection}/${rkey}`; + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "record.deleted", + payload: { uri, did: authorDid, collection, rkey, space: spaceUri }, + ts: now, + })); + }, + }; + return wrapped; +} + +// Keep this import hint for types that downstream code might pull from here. +export type { SpaceMemberRow }; diff --git a/packages/contrail-appview/src/core/realtime/query-filter.ts b/packages/contrail-appview/src/core/realtime/query-filter.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/query-filter.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/resolve.ts b/packages/contrail-appview/src/core/realtime/resolve.ts new file mode 100644 index 0000000..c16abe5 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/resolve.ts @@ -0,0 +1,92 @@ +/** Resolve a raw topic request (as given by a caller) to the concrete set of + * delivery topics they are authorized to subscribe to. + * + * Rules (v1): + * - `space:<uri>` → allowed iff caller is owner or a member of the space. + * - `community:<did>` → expanded to `space:<uri>` for every space in the + * community reachable by the caller (direct grants + * or via delegation). `resolveReachableSpaces` + * already has exactly this semantics. + * - `actor:<did>` → self-only in v1. + * - `collection:<nsid>` → rejected unless the deployment opts in + * (not yet implemented). */ + +import type { StorageAdapter } from "../spaces/types"; +import type { CommunityProbe } from "../community-integration"; +import { spaceTopic, parseCommunityTopic, parseSpaceTopic } from "./types"; + +export interface TopicResolutionContext { + /** May be null when the deployment has no spaces module — in that case + * `space:` and `community:` topics are NotSupported. Public topics + * (`collection:`, `actor:`) still resolve. */ + spaces: StorageAdapter | null; + /** May be null if the community module is not enabled. */ + community: CommunityProbe | null; +} + +export interface TopicResolution { + ok: true; + topics: string[]; +} + +export interface TopicResolutionError { + ok: false; + error: "Forbidden" | "InvalidRequest" | "NotFound" | "NotSupported"; + reason: string; +} + +export async function resolveTopicForCaller( + rawTopic: string, + callerDid: string, + ctx: TopicResolutionContext +): Promise<TopicResolution | TopicResolutionError> { + // space:<uri> + const spaceUri = parseSpaceTopic(rawTopic); + if (spaceUri) { + if (!ctx.spaces) { + return { ok: false, error: "NotSupported", reason: "spaces-module-disabled" }; + } + const space = await ctx.spaces.getSpace(spaceUri); + if (!space) return { ok: false, error: "NotFound", reason: "space-not-found" }; + if (space.ownerDid === callerDid) return { ok: true, topics: [rawTopic] }; + const member = await ctx.spaces.getMember(spaceUri, callerDid); + if (!member) return { ok: false, error: "Forbidden", reason: "not-member" }; + return { ok: true, topics: [rawTopic] }; + } + + // community:<did> + const communityDid = parseCommunityTopic(rawTopic); + if (communityDid) { + if (!ctx.community || !ctx.spaces) { + return { ok: false, error: "NotSupported", reason: "community-module-disabled" }; + } + const row = await ctx.community.getCommunity(communityDid); + if (!row) return { ok: false, error: "NotFound", reason: "community-not-found" }; + const reachable = await ctx.community.resolveReachableSpaces(callerDid); + // Filter to spaces owned by THIS community — reachable may include spaces + // from other communities via cross-community delegation. + const ownedList = await ctx.spaces.listSpaces({ ownerDid: communityDid, limit: 1000 }); + const owned: Set<string> = new Set(ownedList.spaces.map((s) => s.uri)); + const topics: string[] = []; + for (const uri of reachable) { + if (owned.has(uri)) topics.push(spaceTopic(uri)); + } + if (topics.length === 0) { + return { ok: false, error: "Forbidden", reason: "no-reachable-spaces-in-community" }; + } + return { ok: true, topics }; + } + + // actor:<did> — public stream of records authored by this DID. + // Any caller can subscribe (parallels listRecords with an `actor` filter). + if (rawTopic.startsWith("actor:")) { + return { ok: true, topics: [rawTopic] }; + } + + // collection:<nsid> — public firehose for this collection. + if (rawTopic.startsWith("collection:")) { + return { ok: true, topics: [rawTopic] }; + } + + return { ok: false, error: "InvalidRequest", reason: "unknown-topic" }; +} diff --git a/packages/contrail-appview/src/core/realtime/router.ts b/packages/contrail-appview/src/core/realtime/router.ts new file mode 100644 index 0000000..d10e45e --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/router.ts @@ -0,0 +1,255 @@ +/** Realtime XRPC routes: ticket mint + subscribe (SSE | WS). */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { ContrailConfig } from "../types"; +import type { ServiceAuth } from "../spaces/auth"; +import type { StorageAdapter } from "../spaces/types"; +import type { CommunityProbe } from "../community-integration"; +import { InMemoryPubSub } from "./in-memory"; +import { TicketSigner } from "./ticket"; +import { sseResponse } from "./sse"; +import { pumpWebSocket, type WebSocketLike } from "./websocket"; +import { mergeAsyncIterables } from "./merge"; +import { resolveTopicForCaller } from "./resolve"; +import type { PubSub, RealtimeEvent } from "./types"; +import { DEFAULT_TICKET_TTL_MS, DEFAULT_KEEPALIVE_MS } from "./types"; + +export interface RealtimeRoutesOptions { + /** Auth middleware for `<ns>.realtime.ticket` and for JWT-based bot + * subscriptions to private topics. Null when no JWT verifier is available + * (deployments without a spaces config) — in that case, private-topic + * subscribe paths return NotSupported and public topics still work without + * auth. */ + authMiddleware: MiddlewareHandler | null; + pubsub?: PubSub; +} + +/** Public topics: subscribable without any auth. Mirrors listRecords + * semantics — no JWT means "public records only". */ +function isPublicTopic(topic: string): boolean { + return topic.startsWith("collection:") || topic.startsWith("actor:"); +} + +/** WebSocketPair exists on Cloudflare Workers; on Node/Bun it's absent. + * When absent, a platform-provided WebSocket accept hook is used instead. */ +interface WebSocketPairCtor { + new (): { 0: WebSocketLike & { accept?: () => void }; 1: WebSocketLike & { accept?: () => void } }; +} + +export function registerRealtimeRoutes( + app: Hono, + config: ContrailConfig, + spaces: StorageAdapter | null, + community: CommunityProbe | null, + options: RealtimeRoutesOptions +): void { + const cfg = config.realtime; + if (!cfg) return; + + const pubsub: PubSub = options.pubsub ?? cfg.pubsub ?? new InMemoryPubSub({ queueBound: cfg.queueBound }); + const signer = new TicketSigner(cfg.ticketSecret); + const ticketTtl = cfg.ticketTtlMs ?? DEFAULT_TICKET_TTL_MS; + const keepaliveMs = cfg.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + + const NS = `${config.namespace}.realtime`; + + // POST /<ns>.realtime.ticket — { topic } → { ticket, topics, expiresAt } + // Ticket-minting exists so browsers (which can't set Authorization on + // EventSource) can subscribe to *private* topics. Public topics + // (collection:, actor:) don't need tickets — subscribe with `?topic=` directly. + if (options.authMiddleware) { + const authMw = options.authMiddleware; + app.post(`/xrpc/${NS}.ticket`, authMw, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { topic?: string } | null; + if (!body?.topic) { + return c.json({ error: "InvalidRequest", message: "topic required" }, 400); + } + const resolved = await resolveTopicForCaller(body.topic, sa.issuer, { spaces, community }); + if (!resolved.ok) { + const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; + return c.json({ error: resolved.error, reason: resolved.reason }, status); + } + const ticket = await signer.sign({ + topics: resolved.topics, + did: sa.issuer, + ttlMs: ticketTtl, + }); + return c.json({ + ticket, + topics: resolved.topics, + expiresAt: Date.now() + ticketTtl, + }); + }); + } + + // GET /<ns>.realtime.subscribe — SSE or WS. + // + // Three access paths, all land on the same stream: + // - `?topic=collection:<nsid>` or `?topic=actor:<did>` — *public*, no auth. + // Mirrors listRecords semantics. + // - `?ticket=<jwt>` — presented by browsers, minted via `.ticket` after + // a JWT-authenticated call. Only used for private topics. + // - `Authorization: Bearer <jwt>` + `?topic=space:<uri>` — server-side + // bots can skip the ticket dance and go straight to subscribe. + app.get(`/xrpc/${NS}.subscribe`, async (c) => { + const url = new URL(c.req.url); + const ticketParam = url.searchParams.get("ticket"); + const collectionFilter = url.searchParams.get("collection"); + const topicParam = url.searchParams.get("topic"); + + let callerDid: string | null = null; + let topics: string[]; + + if (ticketParam) { + const payload = await signer.verify(ticketParam); + if (!payload) { + return c.json({ error: "AuthRequired", reason: "invalid-or-expired-ticket" }, 401); + } + callerDid = payload.did; + topics = payload.topics; + // Optional: narrow to topics the query explicitly requests. + if (topicParam) { + if (!payload.topics.includes(topicParam)) { + return c.json({ error: "Forbidden", reason: "topic-not-in-ticket" }, 403); + } + topics = [topicParam]; + } + } else if (topicParam && isPublicTopic(topicParam)) { + // Public subscribe — no auth required. Jetstream ingestion publishes + // record events to collection:/actor: topics directly. + topics = [topicParam]; + } else { + // JWT path for private-topic bots. If no auth middleware is available + // (deployment has no spaces config), private topics aren't offered. + if (!options.authMiddleware) { + return c.json( + { + error: "InvalidRequest", + reason: "private-topic-without-auth", + message: + "Subscribing to space:/community: topics requires a JWT verifier; only public topics (collection:, actor:) are available on this deployment.", + }, + 400 + ); + } + let authed = false; + await options.authMiddleware(c, async () => { + authed = true; + }); + if (!authed) return c.res; // middleware already responded with 401 + const sa = getAuth(c); + callerDid = sa.issuer; + if (!topicParam) { + return c.json({ error: "InvalidRequest", message: "topic required" }, 400); + } + const resolved = await resolveTopicForCaller(topicParam, callerDid, { spaces, community }); + if (!resolved.ok) { + const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; + return c.json({ error: resolved.error, reason: resolved.reason }, status); + } + topics = resolved.topics; + } + + if (topics.length === 0) { + return c.json({ error: "InvalidRequest", reason: "no-topics" }, 400); + } + + // Build the merged iterable, with an inline filter that closes the stream + // on a matching `member.removed` event (self-kick on revocation). + const ac = new AbortController(); + const signals: AbortSignal[] = [ac.signal]; + const reqSignal = c.req.raw.signal; + if (reqSignal) signals.push(reqSignal); + const combined = anySignal(signals); + + const sources = topics.map((t) => pubsub.subscribe(t, combined)); + const merged = withSelfKickAndFilter( + mergeAsyncIterables(sources, combined), + callerDid, + collectionFilter, + ac + ); + + // Content negotiation: Upgrade: websocket → WS, else SSE. + if (c.req.header("Upgrade")?.toLowerCase() === "websocket") { + const Pair = (globalThis as unknown as { WebSocketPair?: WebSocketPairCtor }) + .WebSocketPair; + if (!Pair) { + return c.json( + { error: "NotSupported", reason: "websockets-require-worker-or-ws-adapter" }, + 426 + ); + } + const pair = new Pair(); + const clientWs = pair[0]; + const serverWs = pair[1]; + serverWs.accept?.(); + // Pump in the background; don't await. + void pumpWebSocket(serverWs, merged, combined, { keepaliveMs }); + return new Response(null, { + status: 101, + // Hono/undici-compat: some runtimes honor `webSocket` on the init. + // @ts-expect-error - Workers-specific init field + webSocket: clientWs, + }); + } + + return sseResponse(merged, combined, { keepaliveMs }); + }); +} + +// ============================================================================ +// Helpers +// ============================================================================ + +function getAuth(c: Context): ServiceAuth { + const a = c.get("serviceAuth") as ServiceAuth | undefined; + if (!a) throw new Error("service auth not set"); + return a; +} + +/** Merge multiple AbortSignals into one. Aborts when any source aborts. */ +function anySignal(signals: AbortSignal[]): AbortSignal { + const ac = new AbortController(); + for (const s of signals) { + if (s.aborted) { + ac.abort(); + return ac.signal; + } + s.addEventListener("abort", () => ac.abort(), { once: true }); + } + return ac.signal; +} + +/** Wrap an iterable: drop events that don't pass the collection filter (if + * any), and close the outer controller as soon as we see a `member.removed` + * for the caller's own DID. `callerDid` may be null on public subscriptions + * (anonymous) — in that case self-kick is not applicable. */ +function withSelfKickAndFilter( + source: AsyncIterable<RealtimeEvent>, + callerDid: string | null, + collectionFilter: string | null, + ac: AbortController +): AsyncIterable<RealtimeEvent> { + return { + async *[Symbol.asyncIterator]() { + for await (const event of source) { + if (event.kind === "member.removed" && event.payload.did === callerDid) { + // Deliver the kick event so the client sees why, then close. + yield event; + ac.abort(); + return; + } + if ( + collectionFilter && + (event.kind === "record.created" || event.kind === "record.deleted") && + event.payload.collection !== collectionFilter + ) { + continue; + } + yield event; + } + }, + }; +} diff --git a/packages/contrail-appview/src/core/realtime/sse.ts b/packages/contrail-appview/src/core/realtime/sse.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/sse.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/ticket.ts b/packages/contrail-appview/src/core/realtime/ticket.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/ticket.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/types.ts b/packages/contrail-appview/src/core/realtime/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/websocket.ts b/packages/contrail-appview/src/core/realtime/websocket.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/websocket.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/refresh.ts b/packages/contrail-appview/src/core/refresh.ts new file mode 100644 index 0000000..7b2fd21 --- /dev/null +++ b/packages/contrail-appview/src/core/refresh.ts @@ -0,0 +1,267 @@ +import type {} from "@atcute/atproto"; +/** + * Fresh refresh: re-walk every known DID's PDS for every configured collection + * and reconcile against what's in our DB. Unlike `backfillPending`, this + * ignores the `backfills` state machine — it's a "check what we might have + * missed" pass, not a resumable bulk load. + * + * Two categories of delta are counted: + * - missing — the PDS has a record we don't + * - staleUpdates — we have the same URI but a different CID, *and* our + * copy's `indexed_at` is older than `ignoreWindowMs` + * + * The ignore window exists because Jetstream can run ~seconds behind the + * PDS; without the window, "in-sync but racy" writes would show up as + * misses every run. Records inside the window are still applied (they + * might be legit updates), just not counted toward stats. + * + * Typical uses: + * - dev: "I ran backfillAll on Monday, haven't touched it for a week, + * how much did jetstream miss?" + * - prod: "we had jetstream outage yesterday, what did we drop?" + */ +import { type Did, type Nsid } from "@atcute/lexicons"; +import { isDid, isNsid } from "@atcute/lexicons/syntax"; + +import type { Client } from "@atcute/client"; +import type { ContrailConfig, Database, IngestEvent } from "./types.js"; +import { applyEvents, lookupExistingRecords } from "./db/records.js"; +import { getClient } from "./client.js"; + +const PAGE_SIZE = 100; +const REQUEST_TIMEOUT_MS = 10_000; + +async function withTimeout<T>(fn: () => Promise<T>, ms: number): Promise<T> { + return Promise.race([ + fn(), + new Promise<never>((_, rej) => + setTimeout(() => rej(new Error(`timeout after ${ms}ms`)), ms) + ), + ]); +} + +export interface CollectionStats { + /** Record exists on PDS but was absent from our DB. */ + missing: number; + /** Record exists in our DB with a different CID than the PDS, and our + * copy was written before the ignore window. */ + staleUpdates: number; + /** Record is present and matches (same CID, or within ignore window). */ + inSync: number; +} + +export interface RefreshProgress { + usersComplete: number; + usersTotal: number; + usersFailed: number; + recordsScanned: number; +} + +export interface RefreshResult { + /** Per-NSID stats. */ + byCollection: Record<string, CollectionStats>; + /** Sum across every NSID. */ + total: CollectionStats; + usersScanned: number; + usersFailed: number; + /** Effective ignore window used for classification, in ms. */ + ignoreWindowMs: number; + /** Wall-clock runtime, in ms. */ + elapsedMs: number; +} + +export interface RefreshOptions { + /** How many DIDs to fan out against in parallel. Default: 50. */ + concurrency?: number; + /** Records whose local `indexed_at` is within this window of `now` are + * still upserted but excluded from `staleUpdates` counts — guards + * against jetstream being briefly behind the PDS. Default: 60_000 ms. */ + ignoreWindowMs?: number; + /** Override which NSIDs to walk. Default: every `config.collections[*].collection`. */ + nsids?: string[]; + /** Optional progress callback (fires per completed DID). */ + onProgress?: (p: RefreshProgress) => void; + /** Max attempts per listRecords request. Default: 3. */ + maxRetries?: number; + /** Per-request timeout in ms. Default: 10000. */ + requestTimeout?: number; +} + +function emptyStats(): CollectionStats { + return { missing: 0, staleUpdates: 0, inSync: 0 }; +} + +export async function refresh( + db: Database, + config: ContrailConfig, + options?: RefreshOptions +): Promise<RefreshResult> { + const concurrency = options?.concurrency ?? 50; + const ignoreWindowMs = options?.ignoreWindowMs ?? 60_000; + const requestTimeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; + const maxRetries = options?.maxRetries ?? 3; + const startedAt = Date.now(); + + // Default to every configured collection NSID. Profiles are already + // included because `resolveConfig` adds them to `config.collections`. + const nsids = + options?.nsids ?? + Object.values(config.collections).map((c) => c.collection); + + const byCollection: Record<string, CollectionStats> = {}; + for (const nsid of nsids) byCollection[nsid] = emptyStats(); + const total: CollectionStats = emptyStats(); + + // Known DIDs = every author we've ever written for. `backfills` is a + // superset (it also includes failed/pending users that we never got + // records from), which is actually what we want — if we tried and + // failed before, we might succeed now. + const didRows = await db + .prepare("SELECT DISTINCT did FROM backfills") + .all<{ did: string }>(); + const dids = (didRows.results ?? []) + .map((r) => r.did) + .filter((d) => isDid(d)); + + const usersTotal = dids.length; + let usersComplete = 0; + let usersFailed = 0; + let recordsScanned = 0; + + const ignoreBeforeUs = (Date.now() - ignoreWindowMs) * 1000; + + const processDid = async (did: string): Promise<void> => { + let client: Client; + try { + client = await withTimeout( + () => getClient(did as Did, db), + requestTimeout + ); + } catch { + usersFailed++; + return; + } + + for (const nsid of nsids) { + if (!isNsid(nsid)) continue; + let cursor: string | undefined; + while (true) { + let pageRecords: Array<{ uri: string; cid: string; value: unknown }>; + let nextCursor: string | undefined; + try { + // Retry listRecords: transient PDS failures are expected during refresh + let attempt = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + try { + const res = await withTimeout( + () => + client.get("com.atproto.repo.listRecords", { + params: { + repo: did as Did, + collection: nsid as Nsid, + limit: PAGE_SIZE, + cursor, + }, + }), + requestTimeout + ); + if (!res.ok) { + // 400s on a collection the user doesn't have are fine; stop + // paging this collection for this user. + pageRecords = []; + nextCursor = undefined; + break; + } + pageRecords = res.data.records; + nextCursor = res.data.cursor ?? undefined; + break; + } catch (err) { + if (attempt >= maxRetries) throw err; + attempt++; + await new Promise((r) => setTimeout(r, 500 * 2 ** attempt)); + } + } + } catch { + // Give up on this collection for this user; keep going. + break; + } + + if (pageRecords.length === 0) break; + + const now = Date.now(); + const events: IngestEvent[] = pageRecords.map((r) => ({ + uri: r.uri, + did, + collection: nsid, + rkey: r.uri.split("/").pop()!, + operation: "create" as const, + cid: r.cid, + record: JSON.stringify(r.value), + time_us: now * 1000, + indexed_at: now * 1000, + })); + + const existing = await lookupExistingRecords( + db, + events.map((e) => ({ uri: e.uri, collection: e.collection })), + false, + config + ); + + for (const ev of events) { + const ex = existing.get(ev.uri); + if (!ex) { + byCollection[nsid].missing++; + total.missing++; + } else if (ex.cid !== ev.cid) { + const inWindow = + ex.indexed_at !== null && ex.indexed_at >= ignoreBeforeUs; + if (inWindow) { + byCollection[nsid].inSync++; + total.inSync++; + } else { + byCollection[nsid].staleUpdates++; + total.staleUpdates++; + } + } else { + byCollection[nsid].inSync++; + total.inSync++; + } + } + + // Upsert everything — even records "inside the ignore window" + // might genuinely have a new CID; we just don't count them as a + // miss-signal. Skip feed fanout since this is a catch-up, not a + // user-visible write. + await applyEvents(db, events, config, { skipFeedFanout: true }); + recordsScanned += events.length; + + cursor = nextCursor; + if (!cursor) break; + } + } + + usersComplete++; + options?.onProgress?.({ + usersComplete, + usersTotal, + usersFailed, + recordsScanned, + }); + }; + + for (let i = 0; i < dids.length; i += concurrency) { + const batch = dids.slice(i, i + concurrency); + await Promise.allSettled(batch.map(processDid)); + } + + return { + byCollection, + total, + usersScanned: usersComplete, + usersFailed, + ignoreWindowMs, + elapsedMs: Date.now() - startedAt, + }; +} diff --git a/packages/contrail-appview/src/core/router/admin.ts b/packages/contrail-appview/src/core/router/admin.ts new file mode 100644 index 0000000..a3b754b --- /dev/null +++ b/packages/contrail-appview/src/core/router/admin.ts @@ -0,0 +1,44 @@ +import type { Hono } from "hono"; +import type { ContrailConfig, Database } from "../types"; +import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../types"; +import { getLastCursor } from "../db"; + +export function registerAdminRoutes( + app: Hono, + db: Database, + config: ContrailConfig +): void { + const ns = config.namespace; + + app.get(`/xrpc/${ns}.getCursor`, async (c) => { + const cursor = await getLastCursor(db); + if (cursor === null) return c.json({ cursor: null }); + + const dateMs = Math.floor(cursor / 1000); + return c.json({ + time_us: cursor, + date: new Date(dateMs).toISOString(), + seconds_ago: Math.floor((Date.now() - dateMs) / 1000), + }); + }); + + app.get(`/xrpc/${ns}.getOverview`, async (c) => { + const collections: { collection: string; records: number; unique_users: number }[] = []; + + for (const short of getCollectionShortNames(config)) { + const table = recordsTableName(short); + const nsid = nsidForShortName(config, short) ?? short; + const row = await db + .prepare(`SELECT COUNT(*) as records, COUNT(DISTINCT did) as unique_users FROM ${table}`) + .first<{ records: number; unique_users: number }>(); + if (row) { + collections.push({ collection: nsid, records: row.records, unique_users: row.unique_users }); + } + } + + return c.json({ + total_records: collections.reduce((sum, col) => sum + col.records, 0), + collections, + }); + }); +} diff --git a/packages/contrail-appview/src/core/router/collection.ts b/packages/contrail-appview/src/core/router/collection.ts new file mode 100644 index 0000000..6d43d00 --- /dev/null +++ b/packages/contrail-appview/src/core/router/collection.ts @@ -0,0 +1,1225 @@ +import type { Context, Hono } from "hono"; +import type { ContrailConfig, ResolvedContrailConfig, Database, RecordRow, QueryableField, RecordSource, RelationConfig } from "../types"; +import { + getCollectionShortNames, + countColumnName, + groupedCountColumnName, + recordsTableName, + nsidForShortName, + getCollectionMethods, +} from "../types"; +import { queryRecords, queryAcrossSources } from "../db"; +import type { SortOption } from "../db/records"; +import { backfillUser } from "../backfill"; +import { resolveHydrates, resolveReferences, parseHydrateParams } from "./hydrate"; +import { resolveProfiles, collectDids } from "./profiles"; +import { resolveActor } from "../identity"; +import type { FormattedRecord } from "./helpers"; +import { formatRecord, parseIntParam, fieldToParam } from "./helpers"; +import { selectAcceptedLabelers } from "../labels/select"; +import { hydrateLabels } from "../labels/hydrate"; +import { verifyServiceAuthRequest, extractInviteToken, checkInviteReadGrant } from "../spaces/auth"; +import { checkAccess } from "../spaces/acl"; +import { hashInviteToken } from "../invite/token"; +import type { SpacesContext } from "."; +import type { Nsid } from "@atcute/lexicons"; +import type { RealtimeEvent } from "../realtime/types"; +import { sseResponse } from "../realtime/sse"; +import { spaceTopic, communityTopic, parseSpaceTopic } from "../realtime/types"; +import type { SubscriberQuerySpec } from "../realtime/durable-object"; +import { DurableObjectPubSub } from "../realtime/durable-object"; +import { TicketSigner, type TicketQuerySpec } from "../realtime/ticket"; +import { resolveTopicForCaller } from "../realtime/resolve"; +import { mergeAsyncIterables } from "../realtime/merge"; +import type { CommunityProbe } from "../community-integration"; +import { getRelationField, getNestedValue } from "../types"; + +/** Scope of a watch stream. + * - `space`: single permissioned space — one `space:<uri>` topic. + * - `actor`: records authored by `actor` across multiple spaces — the + * resolver expanded these to a per-caller subset of space topics (plus + * `actor:<did>` for public records). Events outside `allowedSpaces` + * are filtered out. */ +type WatchScope = + | { kind: "space"; spaceUri: string } + | { + kind: "actor"; + actor: string; + /** Concrete pubsub topics to subscribe to (from resolveTopicForCaller). */ + topics: string[]; + /** Space URIs the caller can see. Events with `space` outside this + * set are dropped. Undefined `space` on an event (public record) + * is allowed only when `actor` topic is in `topics`. */ + allowedSpaces: Set<string>; + }; + +/** Shared implementation of the watchRecords snapshot+live loop. Called by + * both transport branches (SSE and Worker-terminated WS). The caller owns + * the actual socket/stream and provides a `send(kind, data)` closure. */ +async function runQueryStream(opts: { + send: (kind: string, data: unknown) => void; + abort: AbortController; + scope: WatchScope; + callerDid: string | undefined; + params: URLSearchParams; + db: Database; + config: ContrailConfig; + collection: string; + colNsid: string; + pubsub: import("../realtime/types").PubSub; + relations: Record<string, import("../types").RelationConfig>; + references: Record<string, import("../types").ReferenceConfig>; + childCollectionMap: Map< + string, + { relName: string; matchField: string; matchMode: "uri" | "did" } + >; +}): Promise<void> { + const { + send, + abort, + scope, + callerDid, + params, + db, + config, + collection, + colNsid, + pubsub, + relations, + references, + childCollectionMap + } = opts; + + // Predicate: does this event belong in the caller's scope? + const inScope = (space: string | undefined): boolean => { + if (scope.kind === "space") return space === scope.spaceUri; + if (space == null) return false; // actor mode: require space for now (app topic) + return scope.allowedSpaces.has(space); + }; + + const hydrateSpec = parseHydrateParams(params, relations, references); + const trackHydration = Object.keys(hydrateSpec.relations).length > 0; + const parentUris = new Set<string>(); + const parentDids = new Set<string>(); + const childToParent = new Map<string, { parentUri: string; relName: string }>(); + + const primaryUri = (payload: { uri: string }) => payload.uri; + + const handleChildEvent = (event: RealtimeEvent) => { + if (!trackHydration) return; + if (event.kind !== "record.created" && event.kind !== "record.deleted") return; + const meta = childCollectionMap.get(event.payload.collection); + if (!meta) return; + if (!(hydrateSpec.relations as Record<string, number>)[meta.relName]) return; + if (!inScope(event.payload.space)) return; + // Actor mode: additionally require the record's author match our actor + // (the caller might share spaces with other authors — we only surface + // records by the actor under watch). + if (scope.kind === "actor" && event.payload.did !== scope.actor) return; + + if (event.kind === "record.created") { + const matched = getNestedValue(event.payload.record, meta.matchField); + if (matched == null) return; + const parent = + meta.matchMode === "did" + ? parentDids.has(String(matched)) + ? `at://${String(matched)}/${colNsid}/_` + : null + : parentUris.has(String(matched)) + ? String(matched) + : null; + if (!parent) return; + childToParent.set(event.payload.rkey, { + parentUri: parent, + relName: meta.relName + }); + send("hydration.added", { + parentUri: parent, + relation: meta.relName, + child: { + uri: primaryUri(event.payload), + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + value: event.payload.record, + space: event.payload.space + } + }); + } else { + const info = childToParent.get(event.payload.rkey); + if (!info) return; + childToParent.delete(event.payload.rkey); + send("hydration.removed", { + parentUri: info.parentUri, + relation: info.relName, + childRkey: event.payload.rkey, + childDid: event.payload.did + }); + } + }; + + const handleLive = (event: RealtimeEvent) => { + if (abort.signal.aborted) return; + if (event.kind === "member.removed" && event.payload.did === callerDid) { + send("member.removed", event.payload); + abort.abort(); + return; + } + if (event.kind !== "record.created" && event.kind !== "record.deleted") return; + if (!inScope(event.payload.space)) return; + if (scope.kind === "actor" && event.payload.did !== scope.actor) return; + + if (event.payload.collection !== colNsid) { + handleChildEvent(event); + return; + } + + const nowUs = event.ts * 1000; + const uri = primaryUri(event.payload); + if (event.kind === "record.created") { + parentUris.add(uri); + parentDids.add(event.payload.did); + send("record.created", { + record: { + uri, + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + value: event.payload.record, + time_us: nowUs, + indexed_at: event.ts, + space: event.payload.space + } + }); + } else { + parentUris.delete(uri); + send("record.deleted", { + uri, + did: event.payload.did, + rkey: event.payload.rkey + }); + } + }; + + // Subscribe: one topic for space-scoped, merge across all topics for + // actor-scoped. `mergeAsyncIterables` exists for exactly this case. + let iter: AsyncIterable<RealtimeEvent>; + if (scope.kind === "space") { + iter = pubsub.subscribe(spaceTopic(scope.spaceUri), abort.signal); + } else { + const sources = scope.topics.map((t) => pubsub.subscribe(t, abort.signal)); + iter = mergeAsyncIterables(sources, abort.signal); + } + + const buffered: RealtimeEvent[] = []; + let snapshotDone = false; + + const pump = (async () => { + try { + for await (const event of iter) { + if (abort.signal.aborted) break; + if (!snapshotDone) buffered.push(event); + else handleLive(event); + } + } catch { + /* aborted or errored */ + } + })(); + + try { + send( + "snapshot.start", + scope.kind === "space" + ? { spaceUri: scope.spaceUri, collection: colNsid } + : { actor: scope.actor, collection: colNsid } + ); + const snapshotSpaces = + scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); + const result = await runPipeline(db, config, collection, params, undefined, snapshotSpaces); + for (const record of result.records) { + if (abort.signal.aborted) break; + if (typeof record.uri === "string") parentUris.add(record.uri); + if (typeof record.did === "string") parentDids.add(record.did); + for (const [relName] of Object.entries(hydrateSpec.relations)) { + const hydratedGroups = (record as Record<string, unknown>)[relName]; + if (!hydratedGroups) continue; + const flat: Array<{ rkey?: string }> = Array.isArray(hydratedGroups) + ? (hydratedGroups as Array<{ rkey?: string }>) + : (Object.values(hydratedGroups as Record<string, unknown>).flat() as Array<{ + rkey?: string; + }>); + for (const child of flat) { + if (child?.rkey) { + childToParent.set(child.rkey, { + parentUri: record.uri as string, + relName + }); + } + } + } + send("snapshot.record", { record }); + } + send("snapshot.end", { cursor: result.cursor }); + snapshotDone = true; + for (const event of buffered) handleLive(event); + } catch (err) { + send("error", { + message: err instanceof Error ? err.message : String(err) + }); + abort.abort(); + } + + await pump.catch(() => {}); +} + +export async function runPipeline( + db: Database, + config: ContrailConfig, + collection: string, + params: URLSearchParams, + source?: RecordSource, + spaceUris?: string[], + /** Optional headers from the originating request — used for label + * hydration (`atproto-accept-labelers`). Other entry points pass nothing + * and labels are gated by `?labelers=` / config defaults. */ + headers?: Headers +): Promise<{ records: FormattedRecord[]; cursor?: string; profiles?: any[]; labelersApplied?: string[] }> { + const colConfig = config.collections[collection]; + if (!colConfig) throw new Error(`Unknown collection: ${collection}`); + + const relations = colConfig.relations ?? {}; + const references = colConfig.references ?? {}; + const queryableFields: Record<string, QueryableField> = + (config as ResolvedContrailConfig)._resolved?.queryable[collection] ?? colConfig.queryable ?? {}; + + const limit = parseIntParam(params.get("limit"), 50); + const cursor = params.get("cursor") || undefined; + const actor = params.get("actor") || params.get("did") || undefined; + const wantProfiles = params.get("profiles") === "true"; + + let did: string | undefined; + if (actor) { + const resolved = await resolveActor(db, actor); + if (!resolved) throw new Error("Could not resolve actor"); + did = resolved; + // backfillUser expects the record NSID (for PDS calls), not the short name. + const nsid = nsidForShortName(config, collection) ?? collection; + await backfillUser(db, did, nsid, Date.now() + 3_000, config, { + maxRetries: 0, + requestTimeout: 3_000, + }); + } + + const filters: Record<string, string> = {}; + const rangeFilters: Record<string, { min?: string; max?: string }> = {}; + for (const [field, fieldConfig] of Object.entries(queryableFields)) { + const param = fieldToParam(field); + if (fieldConfig.type === "range") { + const min = params.get(`${param}Min`); + const max = params.get(`${param}Max`); + if (min || max) { + rangeFilters[field] = {}; + if (min) rangeFilters[field].min = min; + if (max) rangeFilters[field].max = max; + } + } else { + const value = params.get(param); + if (value) filters[field] = value; + } + } + + const countFilters: Record<string, number> = {}; + const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; + for (const [relName, rel] of Object.entries(relations)) { + const totalMin = parseIntParam(params.get(`${relName}CountMin`)); + if (totalMin != null) countFilters[rel.collection] = totalMin; + const mapping = relMap[relName]; + if (mapping) { + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + for (const [shortName, fullToken] of Object.entries(mapping.groups)) { + const val = parseIntParam(params.get(`${relName}${capitalize(shortName)}CountMin`)); + if (val != null) countFilters[fullToken] = val; + } + } + } + + let sort: SortOption | undefined; + const sortParam = params.get("sort"); + if (sortParam) { + const orderParam = params.get("order"); + + const fieldEntry = Object.entries(queryableFields).find( + ([field]) => fieldToParam(field) === sortParam + ); + if (fieldEntry) { + const defaultDir = fieldEntry[1].type === "range" ? "desc" : "asc"; + const direction = orderParam === "asc" ? "asc" as const : orderParam === "desc" ? "desc" as const : defaultDir as "asc" | "desc"; + sort = { recordField: fieldEntry[0], direction }; + } else { + const direction = orderParam === "asc" ? "asc" as const : "desc" as const; + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + for (const [relName, rel] of Object.entries(relations)) { + if (sortParam === `${relName}Count`) { + sort = { countType: rel.collection, direction }; + break; + } + const mapping = relMap[relName]; + if (mapping) { + for (const [shortName, fullToken] of Object.entries(mapping.groups)) { + if (sortParam === `${relName}${capitalize(shortName)}Count`) { + sort = { countType: fullToken, direction }; + break; + } + } + if (sort) break; + } + } + } + } + + const search = params.get("search") || undefined; + const spaceUri = params.get("spaceUri") || undefined; + + const queryOpts = { + collection, + did, + limit, + cursor, + filters, + rangeFilters, + countFilters, + sort, + search, + source, + spaceUri, + }; + const result = spaceUris && spaceUris.length > 0 && !spaceUri + ? await queryAcrossSources(db, config, queryOpts, spaceUris) + : await queryRecords(db, config, queryOpts); + + const rows = result.records; + const hydrateRequested = parseHydrateParams(params, relations, references); + const hydrates = await resolveHydrates( + db, + relations, + hydrateRequested.relations, + rows, + config + ); + const refs = await resolveReferences( + db, + references, + hydrateRequested.references, + rows, + config + ); + + const formattedRecords: FormattedRecord[] = rows.map((row) => { + const formatted = formatRecord(row); + flattenCounts(formatted, row.counts, relations); + const h = hydrates[row.uri]; + if (h) { + for (const [relName, groups] of Object.entries(h)) { + formatted[relName] = groups; + } + } + const r = refs[row.uri]; + if (r) { + for (const [refName, record] of Object.entries(r)) { + formatted[refName] = record; + } + } + return formatted; + }); + + const allDids = collectDids(rows, hydrates); + const profileMap = wantProfiles + ? await resolveProfiles(db, config, allDids) + : undefined; + + let labelersApplied: string[] | undefined; + if (config.labels) { + const sel = selectAcceptedLabelers( + headers?.get("atproto-accept-labelers") ?? null, + params.get("labelers"), + config.labels, + ); + if (sel.accepted.length > 0) { + const subjects: string[] = [ + ...formattedRecords.map((r) => r.uri), + ...allDids, + ]; + const cidByUri = new Map<string, string | null>(); + for (const r of formattedRecords) cidByUri.set(r.uri, r.cid); + const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); + for (const fr of formattedRecords) { + const ls = labelsByUri[fr.uri]; + if (ls && ls.length > 0) fr.labels = ls; + } + if (profileMap) { + for (const entries of Object.values(profileMap)) { + for (const entry of entries) { + const ls = labelsByUri[entry.did]; + if (ls && ls.length > 0) entry.labels = ls; + } + } + } + labelersApplied = sel.accepted; + } + } + + return { + records: formattedRecords, + cursor: result.cursor, + ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), + ...(labelersApplied ? { labelersApplied } : {}), + }; +} + +/** Serialize a runPipeline result as JSON, echoing + * `atproto-content-labelers` when labels were applied. The result's + * `labelersApplied` field never appears in the response body — it's a + * side channel for the route to read and turn into a header. */ +function jsonWithLabelers(c: Context, result: { labelersApplied?: string[] } & Record<string, unknown>) { + const { labelersApplied, ...body } = result; + if (labelersApplied && labelersApplied.length > 0) { + c.header("atproto-content-labelers", labelersApplied.join(",")); + } + return c.json(body); +} + +export function registerCollectionRoutes( + app: Hono, + db: Database, + config: ContrailConfig, + spacesCtx?: SpacesContext | null, + options: { + pubsub?: import("../realtime/types").PubSub | null; + community?: CommunityProbe | null; + } = {} +): void { + const ns = config.namespace; + const pubsub = options.pubsub ?? null; + const community = options.community ?? null; + + /** When a per-collection endpoint receives `?spaceUri=...`, verify the JWT, + * resolve membership, run the space ACL, and return the caller DID if allowed. + * Returns null if the spaces subsystem isn't available; the handler should + * then treat the spaceUri as invalid. + * Throws by returning a Response (caller checks via `instanceof Response`). */ + async function gateSpaceAccess( + c: Context, + spaceUri: string, + op: "read" + ): Promise<Response | { callerDid?: string; clientId?: string; viaInviteToken?: boolean }> { + if (!spacesCtx) { + return c.json( + { error: "InvalidRequest", message: "spaces not configured on this service" }, + 501 + ); + } + + // Read-token path: anonymous bearer access via `?inviteToken=...` (or + // `Authorization: Bearer atmo-invite:<token>`). Token must exist, be + // unexpired/unrevoked, scoped to this space, and have a kind that grants + // read (`read` or `read-join`). Token kind cannot grant write — caller must + // separately redeem to become a member for any non-read op. + if (op === "read") { + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant( + spacesCtx.adapter, + rawToken, + spaceUri, + hashInviteToken + ); + if (ok) { + const space = await spacesCtx.adapter.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + return { viaInviteToken: true }; + } + return c.json( + { error: "Forbidden", reason: "invalid-invite-token" }, + 403 + ); + } + } + + const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; + const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); + if (!auth) { + return c.json( + { error: "AuthRequired", message: "spaceUri requires a valid service-auth JWT or read-grant invite token" }, + 401 + ); + } + const space = await spacesCtx.adapter.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await spacesCtx.adapter.getMember(spaceUri, auth.issuer); + const result = checkAccess({ + op, + space, + callerDid: auth.issuer, + member, + clientId: auth.clientId, + }); + if (!result.allow) { + return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + return { callerDid: auth.issuer, clientId: auth.clientId }; + } + + for (const collection of getCollectionShortNames(config)) { + const colConfig = config.collections[collection]; + const methods = getCollectionMethods(colConfig); + + if (methods.includes("listRecords")) { + app.get(`/xrpc/${ns}.${collection}.listRecords`, async (c) => { + const params = new URL(c.req.url).searchParams; + const spaceUri = params.get("spaceUri") || undefined; + + if (spaceUri) { + const gated = await gateSpaceAccess(c, spaceUri, "read"); + if (gated instanceof Response) return gated; + // Route through runPipeline with a single-element space list so the + // full filter / sort / hydrate / reference surface works on per-space + // queries too, not just on the cross-space union path. + try { + const result = await runPipeline(db, config, collection, params, undefined, [spaceUri], c.req.raw.headers); + return jsonWithLabelers(c, result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + } + + // Union path: when the caller is authenticated, fold in records from + // spaces they're a member of. Anonymous callers just get public results. + let spaceUris: string[] | undefined; + const hasAuthHeader = !!c.req.header("Authorization"); + if (spacesCtx) { + const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; + const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); + if (auth) { + const { spaces } = await spacesCtx.adapter.listSpaces({ + memberDid: auth.issuer, + limit: 200, + }); + spaceUris = spaces.map((s) => s.uri); + } else if (hasAuthHeader) { + // Had an auth header but it was invalid — reject rather than + // silently downgrading to public results. + return c.json( + { error: "AuthRequired", message: "invalid service-auth JWT" }, + 401 + ); + } + } + + try { + const result = await runPipeline(db, config, collection, params, undefined, spaceUris, c.req.raw.headers); + return jsonWithLabelers(c, result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + }); + + // Streaming variant — same query shape, SSE'd forever. Opted in via the + // presence of the realtime module; no explicit method config needed. + if (pubsub && spacesCtx) { + const colNsid = colConfig.collection; + const relations = colConfig.relations ?? {}; + const references = colConfig.references ?? {}; + // Map child-NSID → { relName, matchField } so we can route child + // events to hydration deltas without re-parsing config per event. + const childCollectionMap = new Map< + string, + { relName: string; matchField: string; matchMode: "uri" | "did" } + >(); + for (const [relName, rel] of Object.entries(relations)) { + const childNsid = nsidForShortName(config, rel.collection) ?? rel.collection; + childCollectionMap.set(childNsid, { + relName, + matchField: getRelationField(rel), + matchMode: rel.match ?? "uri" + }); + } + + // TicketSigner for watch-scoped tickets. Minted on `mode=ws` handshake + // so the subsequent WS upgrade can auth with just `?ticket=...` (no + // cookie or JWT needed — enables cross-origin + stateless clients). + const ticketSigner = config.realtime?.ticketSecret + ? new TicketSigner(config.realtime.ticketSecret) + : null; + const ticketTtl = config.realtime?.ticketTtlMs ?? 120_000; + + app.get(`/xrpc/${ns}.${collection}.watchRecords`, async (c) => { + const params = new URL(c.req.url).searchParams; + const spaceUri = params.get("spaceUri"); + const actorParam = params.get("actor"); + + if (!spaceUri && !actorParam) { + return c.json( + { error: "InvalidRequest", message: "spaceUri or actor required" }, + 400 + ); + } + + // Resolve the caller and their scope. Two parallel paths: + // - space-scoped: single `space:<uri>` topic, per-space ACL gate. + // - actor-scoped: caller's reachable spaces in the actor's + // community (v1 only supports community DIDs as the actor). + // Events are delivered via N `space:<uri>` topics and filtered + // to `did === actor`. + let callerDid: string | undefined; + let scope: WatchScope; + let scopeTopics: string[]; // for ticket signing + let ticketSpec: TicketQuerySpec | null = null; + + const providedTicket = params.get("ticket"); + if (providedTicket && ticketSigner) { + const payload = await ticketSigner.verify(providedTicket); + if (payload?.querySpec && payload.querySpec.collection === colNsid) { + const ts = payload.querySpec; + if (spaceUri && ts.spaceUri === spaceUri) { + if (payload.topics.includes(spaceTopic(spaceUri))) { + callerDid = payload.did; + ticketSpec = { + collection: ts.collection, + spaceUri: ts.spaceUri, + ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) + }; + } + } else if (actorParam && ts.actor === actorParam) { + callerDid = payload.did; + ticketSpec = { + collection: ts.collection, + actor: ts.actor, + ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) + }; + } + } + } + + const hydrateSpec = parseHydrateParams(params, relations, references); + const hydrateForSpec = Object.keys(hydrateSpec.relations).length > 0 + ? Object.fromEntries( + Object.entries(hydrateSpec.relations).map(([relName]) => { + const rel = relations[relName]!; + const childNsid = + nsidForShortName(config, rel.collection) ?? rel.collection; + return [ + relName, + { childCollection: childNsid, matchField: getRelationField(rel) } + ]; + }) + ) + : undefined; + + if (spaceUri) { + if (!ticketSpec) { + const gated = await gateSpaceAccess(c, spaceUri, "read"); + if (gated instanceof Response) return gated; + callerDid = "callerDid" in gated ? gated.callerDid : undefined; + } + scope = { kind: "space", spaceUri }; + scopeTopics = [spaceTopic(spaceUri)]; + } else { + // Actor-scoped path — v1 only supports community DIDs. + const actor = actorParam!; + if (!community || !spacesCtx) { + return c.json( + { error: "NotSupported", reason: "community-module-disabled" }, + 400 + ); + } + const isCommunity = !!(await community.getCommunity(actor)); + if (!isCommunity) { + return c.json( + { error: "InvalidRequest", reason: "actor-must-be-community-did", message: "cross-space watch currently only supports community DIDs as actor" }, + 400 + ); + } + + if (!ticketSpec) { + // Verify the caller via the same JWT/in-process path used for + // per-space queries, then resolve the community topic to the + // caller's accessible space topics. + const nsidLxm = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; + const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsidLxm); + if (!auth) { + return c.json( + { error: "AuthRequired", message: "service-auth JWT or in-process principal required" }, + 401 + ); + } + callerDid = auth.issuer; + } + const resolved = await resolveTopicForCaller(communityTopic(actor), callerDid!, { + spaces: spacesCtx.adapter, + community + }); + if (!resolved.ok) { + const status = + resolved.error === "NotFound" ? 404 : + resolved.error === "Forbidden" ? 403 : 400; + return c.json({ error: resolved.error, reason: resolved.reason }, status); + } + const allowedSpaces = new Set<string>(); + for (const t of resolved.topics) { + const uri = parseSpaceTopic(t); + if (uri) allowedSpaces.add(uri); + } + scope = { kind: "actor", actor, topics: resolved.topics, allowedSpaces }; + scopeTopics = resolved.topics; + } + + const querySpec: TicketQuerySpec = ticketSpec ?? { + collection: colNsid, + ...(spaceUri ? { spaceUri } : { actor: actorParam! }), + ...(hydrateForSpec ? { hydrate: hydrateForSpec } : {}) + }; + + // Upgrade-to-WS path — forward directly to the DO with the spec, + // so the DO terminates the socket and hibernates when idle. + // Requires snapshot to be fetched separately (see `mode=ws` JSON + // handshake below) or accepted as lossy-on-connect for a plain WS + // upgrade. + const isUpgrade = c.req.header("Upgrade")?.toLowerCase() === "websocket"; + const isWsMode = params.get("mode") === "ws"; + + if (isWsMode && !isUpgrade) { + // Handshake: return snapshot + a ticket the client uses to + // upgrade. Ticket carries the (did, topics, querySpec) signed + // so the WS-upgrade route skips any other auth. + try { + const sinceTs = Date.now(); + const snapshotSpaces = + scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); + const result = await runPipeline( + db, + config, + collection, + params, + undefined, + snapshotSpaces, + c.req.raw.headers + ); + let ticket: string | undefined; + if (ticketSigner && callerDid) { + ticket = await ticketSigner.sign({ + topics: scopeTopics, + did: callerDid, + ttlMs: ticketTtl, + querySpec + }); + } + const wsUrl = (() => { + const u = new URL(c.req.url); + u.searchParams.delete("mode"); + if (ticket) u.searchParams.set("ticket", ticket); + u.searchParams.set("sinceTs", String(sinceTs)); + return u.pathname + u.search; + })(); + return c.json({ + transport: "ws", + snapshot: { records: result.records, cursor: result.cursor }, + querySpec, + ticket, + ticketTtlMs: ticketTtl, + sinceTs, + wsUrl + }); + } catch (err) { + return c.json( + { error: "SnapshotFailed", message: err instanceof Error ? err.message : String(err) }, + 500 + ); + } + } + + if (isUpgrade && pubsub instanceof DurableObjectPubSub && scope.kind === "space") { + // Forward the WS upgrade to the DO. The DO owns the socket from + // here and hibernates when idle. Replays any events buffered + // since the handshake `sinceTs` so the client closes the gap. + // + // Actor-scoped queries fall through to the worker-terminated + // path below — the DO binding is single-topic today; extending + // it to fan out over N topics is future work. + const sinceTsParam = params.get("sinceTs"); + const sinceTs = sinceTsParam ? Number(sinceTsParam) : 0; + return pubsub.forwardSubscribe(spaceTopic(scope.spaceUri), c.req.raw, { + did: callerDid, + querySpec: { + collection: querySpec.collection, + spaceUri: scope.spaceUri, + ...(querySpec.hydrate ? { hydrate: querySpec.hydrate } : {}) + }, + sinceTs: Number.isFinite(sinceTs) ? sinceTs : 0 + }); + } + + const ac = new AbortController(); + const reqSignal = c.req.raw.signal; + if (reqSignal) { + if (reqSignal.aborted) ac.abort(); + else reqSignal.addEventListener("abort", () => ac.abort(), { once: true }); + } + + // Worker-terminated WebSocket — used when pubsub isn't DO-backed + // (dev InMemoryPubSub). Same query-filter loop as SSE; different + // transport. Runs in the same isolate so no cost benefit, but + // matches the prod protocol. + if (isUpgrade) { + const WsPair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; + if (!WsPair) { + return c.json( + { error: "NotSupported", reason: "websockets-require-workers-runtime" }, + 426 + ); + } + const pair = new WsPair(); + const clientWs = pair[0] as WebSocket; + const serverWs = pair[1] as WebSocket & { accept?: () => void }; + serverWs.accept?.(); + + const sendWs = (kind: string, data: unknown) => { + try { + serverWs.send(JSON.stringify({ kind, data })); + } catch { + ac.abort(); + } + }; + serverWs.addEventListener?.("close", () => ac.abort()); + serverWs.addEventListener?.("error", () => ac.abort()); + + void runQueryStream({ + send: sendWs, + abort: ac, + scope, + callerDid, + params, + db, + config, + collection, + colNsid, + pubsub, + relations, + references, + childCollectionMap + }).finally(() => { + try { + serverWs.close(); + } catch { + /* ignore */ + } + }); + + return new Response(null, { + status: 101, + webSocket: clientWs + } as ResponseInit & { webSocket: unknown }); + } + + // SSE fallback. + const encoder = new TextEncoder(); + const stream = new ReadableStream<Uint8Array>({ + start(controller) { + let closed = false; + const close = () => { + if (closed) return; + closed = true; + try { + controller.close(); + } catch { + /* already closed */ + } + }; + ac.signal.addEventListener("abort", close, { once: true }); + + const send = (kind: string, data: unknown) => { + if (closed) return; + try { + controller.enqueue( + encoder.encode(`event: ${kind}\ndata: ${JSON.stringify(data)}\n\n`) + ); + } catch { + close(); + } + }; + + const keepalive = setInterval(() => { + if (closed) return; + try { + controller.enqueue(encoder.encode(`: keepalive\n\n`)); + } catch { + close(); + } + }, 15_000); + ac.signal.addEventListener( + "abort", + () => clearInterval(keepalive), + { once: true } + ); + + void runQueryStream({ + send, + abort: ac, + scope, + callerDid, + params, + db, + config, + collection, + colNsid, + pubsub, + relations, + references, + childCollectionMap + }).finally(() => close()); + }, + cancel() { + ac.abort(); + }, + }); + + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + }, + }); + }); + } + } + + if (!methods.includes("getRecord")) { + // Skip getRecord + custom queries unless listRecords-only was explicitly requested. + for (const [queryName, handler] of Object.entries(colConfig.queries ?? {})) { + app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { + const params = new URL(c.req.url).searchParams; + return handler(db, params, config); + }); + } + continue; + } + + app.get(`/xrpc/${ns}.${collection}.getRecord`, async (c) => { + const uri = c.req.query("uri"); + if (!uri) return c.json({ error: "uri parameter required" }, 400); + + // Spaces path — `?spaceUri=` routes to the per-space store + ACL gate. + const spaceUri = c.req.query("spaceUri") || undefined; + if (spaceUri) { + const gated = await gateSpaceAccess(c, spaceUri, "read"); + if (gated instanceof Response) return gated; + + // Parse author + rkey from the record uri `at://<did>/<collection>/<rkey>` + const m = uri.match(/^at:\/\/([^/]+)\/[^/]+\/([^/]+)$/); + if (!m) return c.json({ error: "InvalidRequest", message: "uri must be at://<did>/<collection>/<rkey>" }, 400); + const authorDid = m[1]; + const rkey = m[2]; + + const nsid = colConfig.collection; + const record = await spacesCtx!.adapter.getRecord(spaceUri, nsid, authorDid, rkey); + if (!record) return c.json({ error: "NotFound" }, 404); + return c.json({ record }); + } + + const relations = colConfig.relations ?? {}; + const references = colConfig.references ?? {}; + const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; + + const table = recordsTableName(collection); + const countCols = getRelationCountColumns(relations, relMap); + const selectCols = `uri, did, rkey, cid, record, time_us, indexed_at${countCols.length > 0 ? ", " + countCols.map(c => c.column).join(", ") : ""}`; + const row = await db + .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri = ?`) + .bind(uri) + .first<any>(); + + if (!row) return c.json({ error: "Record not found" }, 404); + + const nsid = nsidForShortName(config, collection) ?? collection; + const formatted = formatRecord({ ...row, collection: nsid }); + const counts = extractCounts(row, relations); + if (counts) flattenCounts(formatted, counts, relations); + + const params = new URL(c.req.url).searchParams; + const wantProfilesSingle = params.get("profiles") === "true"; + + const hydrateRequested = parseHydrateParams(params, relations, references); + const hydrates = await resolveHydrates( + db, + relations, + hydrateRequested.relations, + [row], + config + ); + const refs = await resolveReferences( + db, + references, + hydrateRequested.references, + [row], + config + ); + const h = hydrates[row.uri]; + if (h) { + for (const [relName, groups] of Object.entries(h)) { + (formatted as any)[relName] = groups; + } + } + const r = refs[row.uri]; + if (r) { + for (const [refName, record] of Object.entries(r)) { + (formatted as any)[refName] = record; + } + } + + const allDids = collectDids([row], hydrates); + const profileMap = wantProfilesSingle + ? await resolveProfiles(db, config, allDids) + : undefined; + + let labelersApplied: string[] | undefined; + if (config.labels) { + const sel = selectAcceptedLabelers( + c.req.raw.headers.get("atproto-accept-labelers"), + params.get("labelers"), + config.labels, + ); + if (sel.accepted.length > 0) { + const subjects: string[] = [row.uri, ...allDids]; + const cidByUri = new Map<string, string | null>([[row.uri, row.cid]]); + const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); + const ls = labelsByUri[row.uri]; + if (ls && ls.length > 0) (formatted as Record<string, unknown>).labels = ls; + if (profileMap) { + for (const entries of Object.values(profileMap)) { + for (const entry of entries) { + const els = labelsByUri[entry.did]; + if (els && els.length > 0) entry.labels = els; + } + } + } + labelersApplied = sel.accepted; + } + } + if (labelersApplied) { + c.header("atproto-content-labelers", labelersApplied.join(",")); + } + + return c.json({ + ...formatted, + ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), + }); + }); + + for (const [queryName, handler] of Object.entries( + colConfig.queries ?? {} + )) { + app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { + const params = new URL(c.req.url).searchParams; + return handler(db, params, config); + }); + } + + for (const [queryName, handler] of Object.entries( + colConfig.pipelineQueries ?? {} + )) { + app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { + const params = new URL(c.req.url).searchParams; + try { + const source = await handler(db, params, config); + const result = await runPipeline(db, config, collection, params, source, undefined, c.req.raw.headers); + return jsonWithLabelers(c, result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + }); + } + } +} + +function getRelationCountColumns( + relations: Record<string, RelationConfig>, + relMap: Record<string, any> +): { column: string }[] { + const cols: { column: string }[] = []; + for (const [relName, rel] of Object.entries(relations)) { + if (rel.count === false) continue; + cols.push({ column: countColumnName(rel.collection) }); + const mapping = relMap[relName]; + if (mapping?.groups) { + for (const groupKey of Object.keys(mapping.groups as Record<string, string>)) { + cols.push({ column: groupedCountColumnName(rel.collection, groupKey) }); + } + } + } + return cols; +} + +function extractCounts( + row: any, + relations: Record<string, any> +): Record<string, number> | undefined { + const counts: Record<string, number> = {}; + + for (const [, rel] of Object.entries(relations)) { + if (rel.count === false) continue; + const totalCol = countColumnName(rel.collection); + const val = row[totalCol]; + if (val != null && val !== 0) counts[rel.collection] = val; + + if (rel.groups) { + for (const [groupKey, fullToken] of Object.entries(rel.groups as Record<string, string>)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + const gval = row[groupCol]; + if (gval != null && gval !== 0) counts[fullToken] = gval; + } + } + } + + return Object.keys(counts).length > 0 ? counts : undefined; +} + +function flattenCounts( + formatted: FormattedRecord, + counts: Record<string, number> | undefined, + relations: Record<string, any> +): void { + if (!counts) return; + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + + const collectionToRelName: Record<string, string> = {}; + const tokenToField: Record<string, string> = {}; + for (const [relName, rel] of Object.entries(relations)) { + collectionToRelName[rel.collection] = relName; + if (rel.groups) { + for (const [shortName, fullToken] of Object.entries(rel.groups as Record<string, string>)) { + tokenToField[fullToken] = `${relName}${capitalize(shortName)}Count`; + } + } + } + + for (const [type, count] of Object.entries(counts)) { + if (collectionToRelName[type]) { + formatted[`${collectionToRelName[type]}Count`] = count; + } else if (tokenToField[type]) { + formatted[tokenToField[type]] = count; + } + } +} diff --git a/packages/contrail-appview/src/core/router/feed.ts b/packages/contrail-appview/src/core/router/feed.ts new file mode 100644 index 0000000..0430292 --- /dev/null +++ b/packages/contrail-appview/src/core/router/feed.ts @@ -0,0 +1,134 @@ +import type { Hono } from "hono"; +import type { ContrailConfig, Database, FeedConfig } from "../types"; +import { getDialect } from "../dialect"; +import { DEFAULT_FEED_MAX_ITEMS, recordsTableName } from "../types"; +import { resolveActor } from "../identity"; +import { backfillUser } from "../backfill"; +import { runPipeline } from "./collection"; + +async function maybeBackfillFeed( + db: Database, + config: ContrailConfig, + actor: string, + feedName: string, + feedConfig: FeedConfig +): Promise<void> { + const status = await db + .prepare("SELECT completed FROM feed_backfills WHERE actor = ? AND feed = ?") + .bind(actor, feedName) + .first<{ completed: number }>(); + + if (status?.completed) return; + + // Ensure the user's follow records are backfilled first + await backfillUser(db, actor, feedConfig.follow, Date.now() + 3_000, config, { + maxRetries: 0, + requestTimeout: 3_000, + }); + + // Mark as in-progress (idempotent) + await db + .prepare( + "INSERT INTO feed_backfills (actor, feed, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" + ) + .bind(actor, feedName) + .run(); + + const maxItems = feedConfig.maxItems ?? DEFAULT_FEED_MAX_ITEMS; + + // Populate feed from existing records by followed users + const followTable = recordsTableName(feedConfig.follow); + for (const targetCol of feedConfig.targets) { + const targetTable = recordsTableName(targetCol); + await db + .prepare( + getDialect(db).insertOrIgnore( + `INSERT INTO feed_items (actor, uri, collection, time_us) + SELECT ?, r.uri, ?, r.time_us + FROM ${targetTable} r + WHERE r.did IN ( + SELECT ${getDialect(db).jsonExtract('f.record', 'subject')} + FROM ${followTable} f + WHERE f.did = ? + ) + ORDER BY r.time_us DESC + LIMIT ?` + ) + ) + .bind(actor, targetCol, actor, maxItems) + .run(); + } + + // Prune oldest items beyond the cap + await db + .prepare( + `DELETE FROM feed_items WHERE actor = ? AND uri NOT IN ( + SELECT uri FROM feed_items WHERE actor = ? ORDER BY time_us DESC LIMIT ? + )` + ) + .bind(actor, actor, maxItems) + .run(); + + await db + .prepare("UPDATE feed_backfills SET completed = 1 WHERE actor = ? AND feed = ?") + .bind(actor, feedName) + .run(); +} + +export function registerFeedRoutes( + app: Hono, + db: Database, + config: ContrailConfig +): void { + if (!config.feeds) return; + + const ns = config.namespace; + + app.get(`/xrpc/${ns}.getFeed`, async (c) => { + const params = new URL(c.req.url).searchParams; + const feedName = params.get("feed"); + const actor = params.get("actor"); + + if (!feedName || !actor) { + return c.json({ error: "feed and actor parameters required" }, 400); + } + + const feedConfig = config.feeds![feedName]; + if (!feedConfig) { + return c.json({ error: "Unknown feed" }, 404); + } + + const did = await resolveActor(db, actor); + if (!did) return c.json({ error: "Could not resolve actor" }, 400); + + await maybeBackfillFeed(db, config, did, feedName, feedConfig); + + const collection = params.get("collection") || feedConfig.targets[0]; + if (!feedConfig.targets.includes(collection)) { + return c.json({ error: "Collection not in feed targets" }, 400); + } + + // Strip feed-specific params so runPipeline doesn't misinterpret them + // (e.g. "actor" in feeds means "whose feed", not "filter by record creator") + const pipelineParams = new URLSearchParams(params); + pipelineParams.delete("feed"); + pipelineParams.delete("actor"); + pipelineParams.delete("collection"); + + const source = { + joins: "JOIN feed_items f ON r.uri = f.uri", + conditions: ["f.actor = ?"], + params: [did], + }; + + try { + const result = await runPipeline(db, config, collection, pipelineParams, source); + return c.json(result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + }); +} diff --git a/packages/contrail-appview/src/core/router/helpers.ts b/packages/contrail-appview/src/core/router/helpers.ts new file mode 100644 index 0000000..7ff372e --- /dev/null +++ b/packages/contrail-appview/src/core/router/helpers.ts @@ -0,0 +1,67 @@ +import type { Database, RecordRow } from "../types"; + +export interface FormattedRecord { + uri: string; + cid: string | null; + value: unknown; + did: string; + collection: string; + rkey: string; + time_us: number; + [key: string]: unknown; +} + +export function formatRecord(row: RecordRow): FormattedRecord { + let value: unknown = null; + if (row.record) { + try { + value = JSON.parse(row.record); + } catch { + value = row.record; + } + } + return { + uri: row.uri, + cid: row.cid, + value, + did: row.did, + collection: row.collection, + rkey: row.rkey, + time_us: row.time_us, + ...(row.space ? { space: row.space } : {}), + }; +} + +export function parseIntParam( + value: string | null | undefined, + defaultValue?: number +): number | undefined { + if (!value) return defaultValue; + const parsed = parseInt(value, 10); + return isNaN(parsed) ? defaultValue : parsed; +} + +export function fieldToParam(field: string): string { + return field.replace(/\.(\w)/g, (_, c) => c.toUpperCase()); +} + +const BATCH_SIZE = 50; + +export async function batchedInQuery<T>( + db: Database, + sql: string, + prefixBindings: (string | number)[], + inValues: string[] +): Promise<T[]> { + const results: T[] = []; + for (let i = 0; i < inValues.length; i += BATCH_SIZE) { + const chunk = inValues.slice(i, i + BATCH_SIZE); + const query = sql.replace("__IN__", chunk.map(() => "?").join(",")); + const rows = await db + .prepare(query) + .bind(...prefixBindings, ...chunk) + .all<T>(); + results.push(...(rows.results ?? [])); + } + return results; +} diff --git a/packages/contrail-appview/src/core/router/hydrate.ts b/packages/contrail-appview/src/core/router/hydrate.ts new file mode 100644 index 0000000..8c1ae63 --- /dev/null +++ b/packages/contrail-appview/src/core/router/hydrate.ts @@ -0,0 +1,223 @@ +import type { RelationConfig, ReferenceConfig, RecordRow, Database, ContrailConfig } from "../types"; +import { getDialect } from "../dialect"; +import { + getNestedValue, + getRelationField, + recordsTableName, + spacesRecordsTableName, + nsidForShortName, +} from "../types"; +import { batchedInQuery, formatRecord } from "./helpers"; + +/** Group rows by their origin: public (undefined key) or a specific spaceUri. */ +function groupBySource<T extends { space?: string }>(rows: T[]): Map<string | undefined, T[]> { + const groups = new Map<string | undefined, T[]>(); + for (const r of rows) { + const key = r.space; + const g = groups.get(key); + if (g) g.push(r); + else groups.set(key, [r]); + } + return groups; +} + +// --- Hydration: embed related records --- + +export function parseHydrateParams( + params: URLSearchParams, + relations: Record<string, RelationConfig>, + references: Record<string, ReferenceConfig> +): { relations: Record<string, number>; references: Set<string> } { + const relHydrates: Record<string, number> = {}; + const refHydrates = new Set<string>(); + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + + for (const relName of Object.keys(relations)) { + const val = params.get(`hydrate${capitalize(relName)}`); + if (val) { + const limit = parseInt(val, 10); + if (!isNaN(limit) && limit > 0) { + relHydrates[relName] = Math.min(limit, 50); + } + } + } + + for (const refName of Object.keys(references)) { + const val = params.get(`hydrate${capitalize(refName)}`); + if (val === "true" || val === "1") { + refHydrates.add(refName); + } + } + + return { relations: relHydrates, references: refHydrates }; +} + +// Per-relation hydrate result: array for ungrouped, Record<group, array> for grouped +export type HydrateResult = Record<string, Record<string, any[] | Record<string, any[]>>>; + +export async function resolveHydrates( + db: Database, + relations: Record<string, RelationConfig>, + requested: Record<string, number>, + records: RecordRow[], + config?: ContrailConfig +): Promise<HydrateResult> { + if (Object.keys(requested).length === 0 || records.length === 0) return {}; + + const grouped: Record<string, Record<string, Record<string, any[]>>> = {}; + + const sourceGroups = groupBySource(records); + + for (const [relName, hydrateLimit] of Object.entries(requested)) { + const rel = relations[relName]; + const field = getRelationField(rel); + const matchMode = rel.match ?? "uri"; + + for (const [sourceSpace, sourceRecords] of sourceGroups) { + const matchValues = matchMode === "did" + ? [...new Set(sourceRecords.map((r) => r.did))] + : sourceRecords.map((r) => r.uri); + + if (matchValues.length === 0) continue; + + const groupCount = rel.groupBy ? 10 : 1; + const maxRows = matchValues.length * hydrateLimit * groupCount; + + const table = sourceSpace + ? spacesRecordsTableName(rel.collection) + : recordsTableName(rel.collection); + const where = sourceSpace + ? `space_uri = ? AND ${getDialect(db).jsonExtract('record', field)} IN (__IN__)` + : `${getDialect(db).jsonExtract('record', field)} IN (__IN__)`; + const prefix = sourceSpace ? [sourceSpace] : []; + + const relatedRows = await batchedInQuery<Omit<RecordRow, "collection">>( + db, + `SELECT uri, did, rkey, record, time_us FROM ${table} + WHERE ${where} + ORDER BY time_us DESC + LIMIT ${maxRows}`, + prefix, + matchValues + ); + + for (const row of relatedRows) { + const record = row.record ? JSON.parse(row.record) : null; + const matchedValue = getNestedValue(record, field); + if (!matchedValue) continue; + + const parentUris = matchMode === "did" + ? sourceRecords.filter((r) => r.did === matchedValue).map((r) => r.uri) + : [matchedValue]; + + const groupValue = rel.groupBy + ? String(getNestedValue(record, rel.groupBy) ?? "other") + : "_flat"; + + for (const parentUri of parentUris) { + const targetUri = matchMode === "did" ? parentUri : matchedValue; + + if (!grouped[targetUri]) grouped[targetUri] = {}; + if (!grouped[targetUri][relName]) grouped[targetUri][relName] = {}; + if (!grouped[targetUri][relName][groupValue]) grouped[targetUri][relName][groupValue] = []; + + const group = grouped[targetUri][relName][groupValue]; + if (group.length < hydrateLimit) { + const childNsid = config + ? nsidForShortName(config, rel.collection) ?? rel.collection + : rel.collection; + group.push( + formatRecord({ + ...(row as any), + collection: childNsid, + ...(sourceSpace ? { space: sourceSpace } : {}), + } as RecordRow) + ); + } + } + } + } + } + + const result: HydrateResult = {}; + for (const [uri, rels] of Object.entries(grouped)) { + result[uri] = {}; + for (const [relName, groups] of Object.entries(rels)) { + if (relations[relName].groupBy) { + result[uri][relName] = groups; + } else { + result[uri][relName] = groups["_flat"] ?? []; + } + } + } + + return result; +} + +// --- References: embed records that our records point at --- + +export type ReferenceResult = Record<string, Record<string, any>>; + +export async function resolveReferences( + db: Database, + references: Record<string, ReferenceConfig>, + requested: Set<string>, + records: RecordRow[], + config?: ContrailConfig +): Promise<ReferenceResult> { + if (requested.size === 0 || records.length === 0) return {}; + + const result: ReferenceResult = {}; + + const sourceGroups = groupBySource(records); + + for (const refName of requested) { + const ref = references[refName]; + if (!ref) continue; + + const refNsid = config + ? nsidForShortName(config, ref.collection) ?? ref.collection + : ref.collection; + + for (const [sourceSpace, sourceRecords] of sourceGroups) { + const targetMap = new Map<string, string[]>(); + for (const r of sourceRecords) { + const parsed = r.record ? JSON.parse(r.record) : null; + const targetValue = parsed ? getNestedValue(parsed, ref.field) : null; + if (!targetValue) continue; + if (!targetMap.has(targetValue)) targetMap.set(targetValue, []); + targetMap.get(targetValue)!.push(r.uri); + } + + const targetUris = [...targetMap.keys()]; + if (targetUris.length === 0) continue; + + const table = sourceSpace + ? spacesRecordsTableName(ref.collection) + : recordsTableName(ref.collection); + const where = sourceSpace ? `space_uri = ? AND uri IN (__IN__)` : `uri IN (__IN__)`; + const prefix = sourceSpace ? [sourceSpace] : []; + + const rows = await batchedInQuery<Omit<RecordRow, "collection">>( + db, + `SELECT uri, did, rkey, record, time_us FROM ${table} WHERE ${where}`, + prefix, + targetUris + ); + + for (const row of rows) { + const parentUris = targetMap.get(row.uri) ?? []; + for (const parentUri of parentUris) { + if (!result[parentUri]) result[parentUri] = {}; + result[parentUri][refName] = formatRecord({ + ...(row as any), + collection: refNsid, + ...(sourceSpace ? { space: sourceSpace } : {}), + } as RecordRow); + } + } + } + } + + return result; +} diff --git a/packages/contrail-appview/src/core/router/index.ts b/packages/contrail-appview/src/core/router/index.ts new file mode 100644 index 0000000..0c5e058 --- /dev/null +++ b/packages/contrail-appview/src/core/router/index.ts @@ -0,0 +1,247 @@ +import { Hono } from "hono"; +import { cors } from "hono/cors"; +import type { Database, ContrailConfig } from "../types"; +import { normalizeProfileConfig } from "../types"; +import { registerAdminRoutes } from "./admin"; +import { registerCollectionRoutes } from "./collection"; +import { registerFeedRoutes } from "./feed"; +import { registerNotifyRoute } from "./notify"; +import { registerSpacesRoutes } from "../spaces/router"; +import type { SpacesRoutesOptions } from "../spaces/router"; +import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; +import { HostedAdapter } from "../spaces/adapter"; +import type { StorageAdapter } from "../spaces/types"; +import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import type { CommunityIntegration } from "../community-integration"; +import { registerRealtimeRoutes } from "../realtime/router"; +import type { RealtimeRoutesOptions } from "../realtime/router"; +import { registerInviteRoutes } from "../invite/router"; +import { InMemoryPubSub } from "../realtime/in-memory"; +import { wrapWithPublishing } from "../realtime/publishing-adapter"; +import type { PubSub } from "../realtime/types"; +import { resolveActor } from "../identity"; +import { resolveProfiles } from "./profiles"; +import { backfillUser } from "../backfill"; +import { selectAcceptedLabelers } from "../labels/select"; +import { hydrateLabels } from "../labels/hydrate"; +import type { MiddlewareHandler } from "hono"; + +export interface SpacesContext { + adapter: StorageAdapter; + verifier: ServiceJwtVerifier; +} + +export interface CreateAppOptions { + spaces?: SpacesRoutesOptions; + /** Pre-built community integration. Construct via the community package's + * `createCommunityIntegration({ ... })`. When set, contrail wires + * community whoami extension, invite handler, route registration, etc. + * When omitted, deployment runs without community features. */ + community?: CommunityIntegration | null; + /** Auth middleware override for community routes (rare — mostly for tests). */ + communityAuthMiddleware?: MiddlewareHandler; + realtime?: Partial<RealtimeRoutesOptions>; + /** Separate DB for the spaces tables. Defaults to `db`. */ + spacesDb?: Database; + /** Full spaces context override (escape hatch for tests). */ + spacesCtx?: SpacesContext | null; + /** Lexicon JSONs to serve at `/lexicons` so consumer apps can fetch + + * typegen against this deployment. Emit with `contrail-lex generate` — + * its `lexicons/generated/index.ts` exports the right shape. If omitted, + * the endpoint returns `404`. */ + lexicons?: object[]; +} + +export function createApp( + db: Database, + config: ContrailConfig, + options: CreateAppOptions = {} +): Hono { + const app = new Hono(); + app.use("*", cors()); + + app.get("/", (c) => c.json({ status: "ok" })); + app.get("/health", (c) => c.json({ status: "ok" })); + app.get("/xrpc/_health", (c) => c.json({ status: "ok" })); + + const ns = config.namespace; + + // Lexicon manifest — lets consumer apps fetch every lexicon this + // deployment speaks (generated + pulled + custom) over HTTP and + // typegen clients, without needing a PDS or DNS resolution. Only + // registered when the caller passed bundled lexicons at build time + // via `contrail-lex generate`. + if (options.lexicons && options.lexicons.length > 0) { + const lexicons = options.lexicons; + app.get(`/xrpc/${ns}.lexicons`, (c) => c.json({ lexicons })); + } + + app.get(`/xrpc/${ns}.getProfile`, async (c) => { + const actor = c.req.query("actor"); + if (!actor) return c.json({ error: "actor parameter required" }, 400); + + const did = await resolveActor(db, actor); + if (!did) return c.json({ error: "Could not resolve actor" }, 400); + + // Ensure profile records are backfilled + const profileConfigs = (config.profiles ?? []).map(normalizeProfileConfig); + for (const pc of profileConfigs) { + await backfillUser(db, did, pc.collection, Date.now() + 3_000, config, { + maxRetries: 0, + requestTimeout: 3_000, + }); + } + + const profileMap = await resolveProfiles(db, config, [did]); + const profiles = profileMap[did]; + if (!profiles || profiles.length === 0) return c.json({ error: "Profile not found" }, 404); + + if (config.labels) { + const params = new URL(c.req.url).searchParams; + const sel = selectAcceptedLabelers( + c.req.raw.headers.get("atproto-accept-labelers"), + params.get("labelers"), + config.labels, + ); + if (sel.accepted.length > 0) { + const labelsByUri = await hydrateLabels(db, [did], sel.accepted); + const ls = labelsByUri[did]; + if (ls && ls.length > 0) { + for (const entry of profiles) { + entry.labels = ls; + } + } + c.header("atproto-content-labelers", sel.accepted.join(",")); + } + } + + return c.json({ profiles }); + }); + + // Shared spaces context — verifier + adapter — reused by both the per-collection + // routes (for `?spaceUri=...` dispatch) and the `<ns>.space.*` routes. + // Built when an authority is configured (spaces are gated on the authority, + // not the record host — a record-host-only deployment still needs an + // authority somewhere, just possibly external). + const spacesDb = options.spacesDb ?? db; + let spacesCtx: SpacesContext | null = + options.spacesCtx !== undefined + ? options.spacesCtx + : config.spaces?.authority + ? { + adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), + verifier: buildVerifier(config.spaces.authority), + } + : null; + + // Community is provided as a pre-built integration — contrail core never + // imports from the community package. The integration object is opaque; + // we just pass through its probe / whoamiExtension / inviteHandler / + // registerRoutes hooks at the right wiring points. + const community = options.community ?? null; + + // Realtime pubsub is built whenever realtime is configured — independent of + // spaces. With spaces, the spaces adapter is wrapped so private record/member + // events publish to space:/community: topics. Without spaces, only public + // topics (collection:/actor:) see traffic — those are published from + // applyEvents (jetstream ingestion), not from here. + let realtimePubsub: PubSub | null = null; + if (config.realtime) { + realtimePubsub = + options.realtime?.pubsub ?? config.realtime.pubsub ?? new InMemoryPubSub({ + queueBound: config.realtime.queueBound, + }); + if (spacesCtx) { + const isCommunityDid = community + ? cachedIsCommunityDid(community.probe) + : undefined; + spacesCtx = { + ...spacesCtx, + adapter: wrapWithPublishing(spacesCtx.adapter, realtimePubsub, { isCommunityDid }), + }; + } + } + + registerAdminRoutes(app, db, config); + + registerCollectionRoutes(app, db, config, spacesCtx, { + pubsub: realtimePubsub, + community: community?.probe ?? null, + }); + registerFeedRoutes(app, db, config); + registerNotifyRoute(app, db, config); + + // Spaces routes — get a whoami extension from the community integration + // when one's wired so community-owned spaces get an `accessLevel` field. + const spacesOptions = { + ...options.spaces, + whoamiExtension: + options.spaces?.whoamiExtension ?? community?.whoamiExtension, + }; + registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); + + if (community && spacesCtx) { + // Community routes reuse the spaces service-auth middleware (same JWT verifier). + const authMiddleware = + options.communityAuthMiddleware ?? + options.spaces?.authMiddleware ?? + createServiceAuthMiddleware(spacesCtx.verifier); + community.registerRoutes(app, { authMiddleware }); + } + + if (config.spaces?.authority && spacesCtx) { + // Unified invite surface: one `<ns>.invite.*` family that dispatches on + // space ownership (user-owned → addMember; community-owned → grant via + // the integration's invite handler). + const authMiddleware = + options.spaces?.authMiddleware ?? + createServiceAuthMiddleware(spacesCtx.verifier); + registerInviteRoutes( + app, + config, + spacesCtx.adapter, + community?.inviteHandler ?? null, + { authMiddleware } + ); + } + + if (config.realtime && realtimePubsub) { + // The ticket endpoint still needs a JWT verifier — but that verifier only + // exists when spaces is configured. Without spaces, private-topic ticket + // minting simply isn't offered; public subscriptions (collection:/actor:) + // require no auth and still work. + const authMiddleware = spacesCtx + ? options.realtime?.authMiddleware ?? + options.spaces?.authMiddleware ?? + createServiceAuthMiddleware(spacesCtx.verifier) + : null; + registerRealtimeRoutes( + app, + config, + spacesCtx?.adapter ?? null, + community?.probe ?? null, + { + authMiddleware, + pubsub: realtimePubsub, + } + ); + } + + return app; +} + +function cachedIsCommunityDid( + probe: import("../community-integration").CommunityProbe +): (did: string) => Promise<boolean> { + const TTL = 60_000; + const cache = new Map<string, { value: boolean; expires: number }>(); + return async (did: string) => { + const now = Date.now(); + const hit = cache.get(did); + if (hit && hit.expires > now) return hit.value; + const row = await probe.getCommunity(did); + const value = row != null; + cache.set(did, { value, expires: now + TTL }); + return value; + }; +} diff --git a/packages/contrail-appview/src/core/router/notify.ts b/packages/contrail-appview/src/core/router/notify.ts new file mode 100644 index 0000000..5ff630b --- /dev/null +++ b/packages/contrail-appview/src/core/router/notify.ts @@ -0,0 +1,179 @@ +import type { Hono } from "hono"; +import type { Database, ContrailConfig, IngestEvent } from "../types"; +import { shortNameForNsid } from "../types"; +import { applyEvents, lookupExistingRecords } from "../db/records"; +import { getPDS } from "../client"; +import type { Did } from "@atcute/lexicons"; + +/** Parse an AT URI into its components. */ +export function parseAtUri(uri: string): { did: string; collection: string; rkey: string } | null { + const match = uri.match(/^at:\/\/(did:[^/]+)\/([^/]+)\/([^/]+)$/); + if (!match) return null; + return { did: match[1], collection: match[2], rkey: match[3] }; +} + +/** + * Fetch a single record from the user's PDS. + * Returns the record + cid on success, null if not found. + */ +async function fetchRecordFromPDS( + pds: string, + did: string, + collection: string, + rkey: string +): Promise<{ value: unknown; cid: string } | null> { + const url = new URL(`/xrpc/com.atproto.repo.getRecord`, pds); + url.searchParams.set("repo", did); + url.searchParams.set("collection", collection); + url.searchParams.set("rkey", rkey); + + const res = await fetch(url.toString()); + if (!res.ok) return null; + + const data = (await res.json()) as { value?: unknown; cid?: string }; + if (!data.value || !data.cid) return null; + return { value: data.value, cid: data.cid }; +} + +export interface NotifyResult { + indexed: number; + deleted: number; + errors?: string[]; +} + +/** + * Process notify URIs: fetch from PDS, detect changes, apply events. + * Shared by both the Hono route and the Contrail.notify() method. + */ +export async function processNotifyUris( + db: Database, + config: ContrailConfig, + uris: string[] +): Promise<NotifyResult> { + const events: IngestEvent[] = []; + const errors: string[] = []; + + // Validate and parse all URIs first + const validUris: { uri: string; parsed: { did: string; collection: string; rkey: string } }[] = []; + for (const uri of uris) { + const parsed = parseAtUri(uri); + if (!parsed) { + errors.push(`invalid AT URI: ${uri}`); + continue; + } + // `parsed.collection` is an NSID; look up the matching short name. + if (!shortNameForNsid(config, parsed.collection)) { + errors.push(`collection not tracked: ${parsed.collection}`); + continue; + } + validUris.push({ uri, parsed }); + } + + // Single batch lookup for all existing records (cid + record in one query) + const existing = await lookupExistingRecords( + db, + validUris.map(({ uri, parsed }) => ({ uri, collection: parsed.collection })), + true, + config + ); + + for (const { uri, parsed } of validUris) { + const pds = await getPDS(parsed.did as Did, db); + if (!pds) { + errors.push(`could not resolve PDS for ${parsed.did}`); + continue; + } + + const result = await fetchRecordFromPDS( + pds, + parsed.did, + parsed.collection, + parsed.rkey + ); + + const now = Date.now() * 1000; // microseconds + const existingInfo = existing.get(uri); + + if (result) { + if (existingInfo?.cid === result.cid) { + // Same CID — nothing changed + continue; + } + + events.push({ + uri, + did: parsed.did, + collection: parsed.collection, + rkey: parsed.rkey, + operation: existingInfo ? "update" : "create", + cid: result.cid, + record: JSON.stringify(result.value), + time_us: now, + indexed_at: now, + }); + } else if (existingInfo) { + // Record gone from PDS but exists locally — delete it. + events.push({ + uri, + did: parsed.did, + collection: parsed.collection, + rkey: parsed.rkey, + operation: "delete", + cid: null, + record: existingInfo.record, + time_us: now, + indexed_at: now, + }); + } + } + + if (events.length > 0) { + // Pass pre-fetched existing records so applyEvents skips re-querying + await applyEvents(db, events, config, { existing }); + } + + return { + indexed: events.filter((e) => e.operation === "create" || e.operation === "update").length, + deleted: events.filter((e) => e.operation === "delete").length, + errors: errors.length > 0 ? errors : undefined, + }; +} + +export function registerNotifyRoute( + app: Hono, + db: Database, + config: ContrailConfig +) { + // Endpoint is off by default. Set config.notify to true or a secret string to enable. + if (!config.notify) return; + + const ns = config.namespace; + const secret = typeof config.notify === "string" ? config.notify : null; + + app.post(`/xrpc/${ns}.notifyOfUpdate`, async (c) => { + if (secret) { + const auth = c.req.header("Authorization"); + if (auth !== `Bearer ${secret}`) { + return c.json({ error: "unauthorized" }, 401); + } + } + + const body = await c.req.json<{ uri?: string; uris?: string[] }>().catch(() => null); + const uris: string[] = []; + + if (body?.uris && Array.isArray(body.uris)) { + uris.push(...body.uris); + } else if (body?.uri) { + uris.push(body.uri); + } else { + return c.json({ error: "uri or uris required" }, 400); + } + + if (uris.length > 25) { + return c.json({ error: "max 25 URIs per request" }, 400); + } + + const result = await processNotifyUris(db, config, uris); + return c.json(result); + }); +} diff --git a/packages/contrail-appview/src/core/router/profiles.ts b/packages/contrail-appview/src/core/router/profiles.ts new file mode 100644 index 0000000..5552357 --- /dev/null +++ b/packages/contrail-appview/src/core/router/profiles.ts @@ -0,0 +1,181 @@ +import type { Database, ContrailConfig, RecordRow, ProfileConfig } from "../types"; +import { recordsTableName, normalizeProfileConfig } from "../types"; +import { resolveIdentities } from "../identity"; +import { getPDS } from "../client"; +import type { Did } from "@atcute/lexicons"; +import { batchedInQuery } from "./helpers"; + +export interface ProfileEntry { + did: string; + handle: string | null; + uri?: string; + cid?: string | null; + value?: unknown; + collection?: string; + rkey?: string; + /** Hydrated by the labels module when the caller has accepted-labelers + * active and there are matching labels on this DID. */ + labels?: unknown; +} + +export function collectDids( + records: RecordRow[], + hydrates: Record<string, Record<string, any[] | Record<string, any[]>>> +): string[] { + const dids = new Set(records.map((r) => r.did)); + for (const rels of Object.values(hydrates)) { + for (const value of Object.values(rels)) { + const items = Array.isArray(value) + ? value + : Object.values(value).flat(); + for (const item of items) { + if (item.did) dids.add(item.did); + } + } + } + return [...dids]; +} + +export async function resolveProfiles( + db: Database, + config: ContrailConfig, + dids: string[] +): Promise<Record<string, ProfileEntry[]>> { + if (dids.length === 0 || !config.profiles || config.profiles.length === 0) { + return {}; + } + + const profileConfigs = config.profiles.map(normalizeProfileConfig); + const result: Record<string, ProfileEntry[]> = {}; + + // Batch-lookup profile records for each configured profile collection + for (const pc of profileConfigs) { + const { collection, rkey: configRkey, shortName } = pc; + const rkey = configRkey ?? "self"; + const table = recordsTableName(shortName ?? collection); + const uris = dids.map((did) => `at://${did}/${collection}/${rkey}`); + + const rows = await batchedInQuery<Omit<RecordRow, "collection">>( + db, + `SELECT uri, did, rkey, cid, record FROM ${table} WHERE uri IN (__IN__)`, + [], + uris + ); + + for (const row of rows) { + let value: unknown = null; + if (row.record) { + try { + value = JSON.parse(row.record); + } catch { + value = row.record; + } + } + if (!result[row.did]) result[row.did] = []; + result[row.did].push({ + did: row.did, + handle: null, // filled below + uri: row.uri, + collection, + rkey: row.rkey, + cid: row.cid, + value, + }); + } + } + + // Resolve identities for all DIDs + const identities = await resolveIdentities(db, dids); + + // Fetch missing profile records from PDS on demand + const missingDids = dids.filter((d) => !result[d]); + if (missingDids.length > 0 && profileConfigs.length > 0) { + const fetched = await fetchMissingProfiles(db, config, missingDids); + for (const [did, entries] of Object.entries(fetched)) { + if (!result[did]) result[did] = []; + result[did].push(...entries); + } + } + + // Fill in handles and create entries for DIDs without profile records + for (const did of dids) { + const identity = identities.get(did); + const handle = identity?.handle ?? null; + + if (result[did]) { + for (const entry of result[did]) { + entry.handle = handle; + } + } else { + result[did] = [{ did, handle }]; + } + } + + return result; +} + +/** + * Fetch profile records from PDS for DIDs not yet in the index. + * Fetches in parallel across all configured profile collections, + * indexes the results into D1 for future requests. + */ +async function fetchMissingProfiles( + db: Database, + config: ContrailConfig, + dids: string[] +): Promise<Record<string, ProfileEntry[]>> { + const result: Record<string, ProfileEntry[]> = {}; + const profileConfigs = config.profiles!.map(normalizeProfileConfig); + + await Promise.all( + dids.flatMap((did) => + profileConfigs.map(async (pc) => { + const { collection, rkey: configRkey, shortName } = pc; + const rkey = configRkey ?? "self"; + const table = recordsTableName(shortName ?? collection); + try { + const pds = await getPDS(did as Did, db); + if (!pds) return; + + const url = new URL("/xrpc/com.atproto.repo.getRecord", pds); + url.searchParams.set("repo", did); + url.searchParams.set("collection", collection); + url.searchParams.set("rkey", rkey); + + const res = await fetch(url.toString()); + if (!res.ok) return; + + const data = (await res.json()) as { uri?: string; value?: unknown; cid?: string }; + if (!data.value || !data.cid) return; + + const uri = data.uri ?? `at://${did}/${collection}/${rkey}`; + const record = data.value; + const cid = data.cid; + + // Index into D1 for future requests + await db + .prepare( + `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, indexed_at = excluded.indexed_at` + ) + .bind(uri, did, rkey, cid, JSON.stringify(record), Date.now() * 1000, Date.now()) + .run(); + + if (!result[did]) result[did] = []; + result[did].push({ + did, + handle: null, + uri, + collection, + rkey, + cid, + value: record, + }); + } catch { + // Skip failures silently + } + }) + ) + ); + + return result; +} diff --git a/packages/contrail-appview/src/core/search.ts b/packages/contrail-appview/src/core/search.ts new file mode 100644 index 0000000..a33dc5e --- /dev/null +++ b/packages/contrail-appview/src/core/search.ts @@ -0,0 +1,31 @@ +import type { CollectionConfig } from "./types"; +import { getNestedValue } from "./types"; + +/** + * Resolve which fields are searchable for a collection. + * Returns null if search is disabled or no fields found. + */ +export function getSearchableFields( + collection: string, + colConfig: CollectionConfig +): string[] | null { + if (!Array.isArray(colConfig.searchable)) return null; + return colConfig.searchable.length > 0 ? colConfig.searchable : null; +} + +/** Sanitized FTS table name for a collection. */ +export function ftsTableName(collection: string): string { + return `fts_${collection.replace(/[^a-zA-Z0-9]/g, "_")}`; +} + +/** Extract searchable field values from a record and join them into a single string. */ +export function buildFtsContent(record: unknown, fields: string[]): string | null { + const parts: string[] = []; + for (const field of fields) { + const value = getNestedValue(record, field); + if (typeof value === "string" && value.length > 0) { + parts.push(value); + } + } + return parts.length > 0 ? parts.join(" ") : null; +} diff --git a/packages/contrail-appview/src/core/spaces/acl.ts b/packages/contrail-appview/src/core/spaces/acl.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/acl.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/adapter.ts b/packages/contrail-appview/src/core/spaces/adapter.ts new file mode 100644 index 0000000..50fb33d --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/adapter.ts @@ -0,0 +1,505 @@ +/** Contrail's all-in-one default adapter — extends the authority package's + * {@link HostedAuthorityAdapter} (which owns space lifecycle, member list, + * invites) and adds the record-host methods (records, blobs, enrollment). + * + * Phase 7a step 3 will lift the record-host methods into a separate + * HostedRecordHostAdapter, at which point this class becomes a thin + * composition / re-export. For now we keep both roles in one class so + * consumers can wire a single object that satisfies the full StorageAdapter. */ + +import type { ContrailConfig, RelationConfig, ResolvedContrailConfig } from "../types"; +import { + shortNameForNsid, + spacesRecordsTableName, + countColumnName, + groupedCountColumnName, + getRelationField, + getNestedValue, +} from "../types"; +import { getDialect } from "../dialect"; +import type { + BlobMetaRow, + CollectionCount, + EnrollmentRow, + ListBlobsOptions, + ListBlobsResult, + ListOptions, + ListResult, + StorageAdapter, + StoredRecord, +} from "./types"; +import type { Database } from "../types"; +import { buildRecordUri } from "./uri"; +import { HostedAuthorityAdapter, parseJson, toNum } from "@atmo-dev/contrail-authority"; + +function mapBlobMetaRow(row: any): BlobMetaRow { + return { + spaceUri: row.space_uri, + cid: row.cid, + mimeType: row.mime_type, + size: Number(row.size), + authorDid: row.author_did, + createdAt: toNum(row.created_at), + }; +} + +function mapEnrollmentRow(row: any): EnrollmentRow { + return { + spaceUri: row.space_uri, + authorityDid: row.authority_did, + enrolledAt: toNum(row.enrolled_at), + enrolledBy: row.enrolled_by, + }; +} + +/** Row mapper for per-collection spaces_records_<short> tables. + * `collection` is injected by the caller (known from the table name). */ +function mapRecordRow(row: any, collection: string): StoredRecord { + return { + spaceUri: row.space_uri, + collection, + authorDid: row.did, + rkey: row.rkey, + cid: row.cid ?? null, + record: parseJson<Record<string, unknown>>(row.record) ?? {}, + createdAt: toNum(row.time_us), + }; +} + +export class HostedAdapter extends HostedAuthorityAdapter implements StorageAdapter { + /** Resolve the per-collection spaces table name, or throw if the collection + * isn't configured (and therefore has no table). */ + private tableFor(collection: string): string { + if (!this.config) { + throw new Error( + `HostedAdapter: config not provided; cannot resolve table for collection ${collection}` + ); + } + const short = shortNameForNsid(this.config, collection); + if (!short) { + throw new Error( + `HostedAdapter: collection ${collection} is not configured in this deployment` + ); + } + return spacesRecordsTableName(short); + } + + // ---- Enrollment ---- + + async enroll(input: EnrollmentRow): Promise<void> { + await this.db + .prepare( + `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri) DO UPDATE SET + authority_did = excluded.authority_did, + enrolled_at = excluded.enrolled_at, + enrolled_by = excluded.enrolled_by` + ) + .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) + .run(); + } + + async getEnrollment(spaceUri: string): Promise<EnrollmentRow | null> { + const row = await this.db + .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .first<any>(); + return row ? mapEnrollmentRow(row) : null; + } + + async listEnrollments( + options: { authorityDid?: string; limit?: number } = {} + ): Promise<EnrollmentRow[]> { + const limit = Math.min(options.limit ?? 200, 1000); + if (options.authorityDid) { + const { results } = await this.db + .prepare( + `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` + ) + .bind(options.authorityDid, limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + const { results } = await this.db + .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) + .bind(limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + + async removeEnrollment(spaceUri: string): Promise<void> { + await this.db + .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .run(); + } + + // ---- Records ---- + + async putRecord(record: StoredRecord): Promise<void> { + const table = this.tableFor(record.collection); + const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); + + const childShort = this.config ? shortNameForNsid(this.config, record.collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(record.spaceUri, record.authorDid, record.rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `INSERT INTO ${table} (space_uri, uri, did, rkey, cid, record, time_us, indexed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, did, rkey) DO UPDATE SET + uri = excluded.uri, + cid = excluded.cid, + record = excluded.record, + time_us = excluded.time_us, + indexed_at = excluded.indexed_at` + ) + .bind( + record.spaceUri, + uri, + record.authorDid, + record.rkey, + record.cid, + JSON.stringify(record.record), + record.createdAt, + Date.now() + ) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace( + record.spaceUri, + childShort, + beforeRecord, + record.record, + record.authorDid + ); + } + } + + async getRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<StoredRecord | null> { + const table = this.tableFor(collection); + const row = await this.db + .prepare( + `SELECT * FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .first<any>(); + return row ? mapRecordRow(row, collection) : null; + } + + async listRecords( + spaceUri: string, + collection: string, + options: ListOptions = {} + ): Promise<ListResult> { + const table = this.tableFor(collection); + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("time_us < ?"); + params.push(Number(options.cursor)); + } + + const sql = `SELECT * FROM ${table} + WHERE ${clauses.join(" AND ")} + ORDER BY time_us DESC + LIMIT ?`; + params.push(limit + 1); + + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const records = results.map((r) => mapRecordRow(r, collection)); + let cursor: string | undefined; + if (records.length > limit) { + const next = records.pop()!; + cursor = String(next.createdAt); + } + return { records, cursor }; + } + + async deleteRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<void> { + const table = this.tableFor(collection); + + const childShort = this.config ? shortNameForNsid(this.config, collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(spaceUri, authorDid, rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `DELETE FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace(spaceUri, childShort, beforeRecord, null, authorDid); + } + } + + /** Recompute count columns on parent records in the same space, scoped to the + * targets derived from before/after versions of the written/deleted child record. */ + private async recountParentsForSpace( + spaceUri: string, + childShort: string, + before: Record<string, unknown> | null, + after: Record<string, unknown> | null, + childDid: string + ): Promise<void> { + if (!this.config) return; + const config = this.config; + const resolved = (config as ResolvedContrailConfig)._resolved; + const childTable = spacesRecordsTableName(childShort); + + type Inbound = { parentShort: string; relationName: string; rel: RelationConfig }; + const inbound: Inbound[] = []; + for (const [parentShort, parentCfg] of Object.entries(config.collections)) { + if (parentCfg.allowInSpaces === false) continue; + for (const [relName, rel] of Object.entries(parentCfg.relations ?? {})) { + if (rel.count === false) continue; + if (rel.collection !== childShort) continue; + inbound.push({ parentShort, relationName: relName, rel }); + } + } + if (inbound.length === 0) return; + + // Deduplicate (parent, relation, target) across before/after. + const keyed = new Map<string, { parentShort: string; relationName: string; rel: RelationConfig; target: string }>(); + for (const { parentShort, relationName, rel } of inbound) { + const field = getRelationField(rel); + const collectTarget = (rec: Record<string, unknown> | null) => { + if (!rec) return; + if (rel.match === "did") { + keyed.set(`${parentShort}:${relationName}:${childDid}`, { + parentShort, relationName, rel, target: childDid, + }); + return; + } + const v = getNestedValue(rec, field); + if (typeof v === "string" && v.length > 0) { + keyed.set(`${parentShort}:${relationName}:${v}`, { + parentShort, relationName, rel, target: v, + }); + } + }; + collectTarget(before); + collectTarget(after); + } + if (keyed.size === 0) return; + + const dialect = getDialect(this.db); + const stmts: ReturnType<Database["prepare"]>[] = []; + + for (const { parentShort, relationName, rel, target } of keyed.values()) { + const parentTable = spacesRecordsTableName(parentShort); + const matchColumn = rel.match === "did" ? "did" : "uri"; + const field = getRelationField(rel); + const countExpr = rel.countDistinct + ? `COUNT(DISTINCT ${rel.countDistinct})` + : "COUNT(*)"; + + const setClauses: string[] = []; + const binds: (string | number)[] = []; + + const totalCol = countColumnName(rel.collection); + setClauses.push( + `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ?)` + ); + binds.push(spaceUri, target); + + if (rel.groupBy) { + const mapping = resolved?.relations[parentShort]?.[relationName]; + if (mapping?.groups) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + setClauses.push( + `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ? AND ${dialect.jsonExtract("record", rel.groupBy)} = ?)` + ); + binds.push(spaceUri, target, fullToken); + } + } + } + + binds.push(spaceUri, target); + stmts.push( + this.db + .prepare( + `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE space_uri = ? AND ${matchColumn} = ?` + ) + .bind(...binds) + ); + } + + if (stmts.length > 0) await this.db.batch(stmts); + } + + async listCollections( + spaceUri: string, + options: { byUser?: string } = {} + ): Promise<CollectionCount[]> { + if (!this.config) return []; + const results: CollectionCount[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + const table = spacesRecordsTableName(short); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + try { + const row = await this.db + .prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${clauses.join(" AND ")}`) + .bind(...params) + .first<{ count: number }>(); + const count = Number(row?.count ?? 0); + if (count > 0) results.push({ collection: colConfig.collection, count }); + } catch { + // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip + } + } + return results; + } + + // ---- Blobs ---- + + async putBlobMeta(row: BlobMetaRow): Promise<void> { + const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, cid) DO NOTHING`; + await this.db + .prepare(sql) + .bind(row.spaceUri, row.cid, row.mimeType, row.size, row.authorDid, row.createdAt) + .run(); + } + + async getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null> { + const r = await this.db + .prepare(`SELECT * FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .first<any>(); + return r ? mapBlobMetaRow(r) : null; + } + + async listBlobMeta( + spaceUri: string, + options: ListBlobsOptions = {} + ): Promise<ListBlobsResult> { + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("author_did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("created_at < ?"); + params.push(Number(options.cursor)); + } + const sql = `SELECT * FROM spaces_blobs + WHERE ${clauses.join(" AND ")} + ORDER BY created_at DESC + LIMIT ?`; + params.push(limit + 1); + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const blobs = results.map(mapBlobMetaRow); + let cursor: string | undefined; + if (blobs.length > limit) { + const next = blobs.pop()!; + cursor = String(next.createdAt); + } + return { blobs, cursor }; + } + + async deleteBlobMeta(spaceUri: string, cid: string): Promise<void> { + await this.db + .prepare(`DELETE FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .run(); + } + + async findOrphanBlobs( + spaceUri: string, + cutoff: number, + limit: number + ): Promise<BlobMetaRow[]> { + if (!this.config) return []; + // Gather candidate blobs older than cutoff, then filter out any whose CID + // appears in any record JSON in this space. We use a cheap substring probe + // (LIKE) per collection — false positives are OK because an orphan that + // survives GC just gets collected next cycle; false negatives (deleting + // a referenced blob) would be a bug, and substring search over the full + // CID is safe enough for that. + const { results } = await this.db + .prepare( + `SELECT * FROM spaces_blobs + WHERE space_uri = ? AND created_at < ? + ORDER BY created_at ASC + LIMIT ?` + ) + .bind(spaceUri, cutoff, limit) + .all<any>(); + const candidates = results.map(mapBlobMetaRow); + if (candidates.length === 0) return []; + + const tables: string[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + tables.push(spacesRecordsTableName(short)); + } + + const orphans: BlobMetaRow[] = []; + for (const blob of candidates) { + let referenced = false; + const pattern = `%${blob.cid}%`; + for (const table of tables) { + try { + const row = await this.db + .prepare( + `SELECT 1 FROM ${table} WHERE space_uri = ? AND record LIKE ? LIMIT 1` + ) + .bind(spaceUri, pattern) + .first<any>(); + if (row) { + referenced = true; + break; + } + } catch { + // table missing — ignore + } + } + if (!referenced) orphans.push(blob); + } + return orphans; + } +} diff --git a/packages/contrail-appview/src/core/spaces/auth.ts b/packages/contrail-appview/src/core/spaces/auth.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/auth.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/binding.ts b/packages/contrail-appview/src/core/spaces/binding.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/binding.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/blob-adapter.ts b/packages/contrail-appview/src/core/spaces/blob-adapter.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/blob-adapter.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/blob-gc.ts b/packages/contrail-appview/src/core/spaces/blob-gc.ts new file mode 100644 index 0000000..d523fcd --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/blob-gc.ts @@ -0,0 +1,38 @@ +import type { BlobAdapter } from "./blob-adapter"; +import { blobKey } from "./blob-adapter"; +import type { StorageAdapter } from "./types"; + +export interface BlobGcOptions { + /** Orphan rows created before this timestamp are eligible for deletion. */ + olderThan: number; + /** Maximum number of blobs to delete in this pass. Defaults to 500. */ + batchSize?: number; +} + +export interface BlobGcResult { + deleted: number; + cids: string[]; +} + +/** Delete blob bytes + metadata for any blob older than `olderThan` that + * is not referenced by any record in the space. Safe to run periodically. */ +export async function gcOrphanBlobs( + storage: StorageAdapter, + blobs: BlobAdapter, + spaceUri: string, + options: BlobGcOptions +): Promise<BlobGcResult> { + const batchSize = options.batchSize ?? 500; + const orphans = await storage.findOrphanBlobs(spaceUri, options.olderThan, batchSize); + if (orphans.length === 0) return { deleted: 0, cids: [] }; + + const keys: string[] = []; + for (const row of orphans) { + keys.push(await blobKey(row.spaceUri, row.cid)); + } + await blobs.delete(keys); + for (const row of orphans) { + await storage.deleteBlobMeta(row.spaceUri, row.cid); + } + return { deleted: orphans.length, cids: orphans.map((o) => o.cid) }; +} diff --git a/packages/contrail-appview/src/core/spaces/blob-refs.ts b/packages/contrail-appview/src/core/spaces/blob-refs.ts new file mode 100644 index 0000000..62cf693 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/blob-refs.ts @@ -0,0 +1,27 @@ +/** + * Walk a record JSON and collect every atproto blob ref. + * + * Blob refs look like: + * { "$type": "blob", "ref": { "$link": "<cid>" }, "mimeType": "...", "size": N } + * + * We return the CID strings. + */ +export function collectBlobCids(value: unknown, out: Set<string> = new Set()): Set<string> { + if (value == null) return out; + if (Array.isArray(value)) { + for (const v of value) collectBlobCids(v, out); + return out; + } + if (typeof value !== "object") return out; + + const obj = value as Record<string, unknown>; + if (obj["$type"] === "blob") { + const ref = obj["ref"] as { $link?: unknown } | undefined; + if (ref && typeof ref["$link"] === "string") out.add(ref["$link"]); + // Don't descend — a blob ref's own shape has no nested blobs. + return out; + } + + for (const v of Object.values(obj)) collectBlobCids(v, out); + return out; +} diff --git a/packages/contrail-appview/src/core/spaces/credentials.ts b/packages/contrail-appview/src/core/spaces/credentials.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/credentials.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/in-process.ts b/packages/contrail-appview/src/core/spaces/in-process.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/in-process.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/router.ts b/packages/contrail-appview/src/core/spaces/router.ts new file mode 100644 index 0000000..943f6a9 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/router.ts @@ -0,0 +1,94 @@ +/** Umbrella that wires the authority + record-host route registrations from + * their respective packages onto a single Hono app. The actual route + * handlers live in `@atmo-dev/contrail-authority` and + * `@atmo-dev/contrail-record-host`. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { ContrailConfig, Database } from "../types"; +import { HostedAdapter } from "./adapter"; +import { + buildVerifier, + createBindingCredentialVerifier, + createCompositeBindingResolver, + createEnrollmentBindingResolver, + createLocalBindingResolver, + createLocalKeyResolver, + createServiceAuthMiddleware, +} from "@atmo-dev/contrail-base"; +import type { + CredentialVerifier, + StorageAdapter, + WhoamiExtension, +} from "@atmo-dev/contrail-base"; +import { registerAuthorityRoutes } from "@atmo-dev/contrail-authority"; +import { registerRecordHostRoutes } from "@atmo-dev/contrail-record-host"; + +// Re-export the route-registration functions and WhoamiExtension type so +// existing consumers of `@atmo-dev/contrail` keep their imports working +// without switching to the new packages. +export { registerAuthorityRoutes, registerRecordHostRoutes }; +export type { WhoamiExtension }; + +export interface SpacesRoutesOptions { + /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ + authMiddleware?: MiddlewareHandler; + /** Storage adapter override. Defaults to HostedAdapter(db). */ + adapter?: StorageAdapter; + /** Optional whoami extension; see {@link WhoamiExtension}. */ + whoamiExtension?: WhoamiExtension; + /** Optional credential verifier for the record host. */ + credentialVerifier?: CredentialVerifier; +} + +/** Umbrella registration: wires both the authority and the record-host + * routes against the same adapter. Today's deployments enable both via + * `config.spaces.authority` and `config.spaces.recordHost`. */ +export function registerSpacesRoutes( + app: Hono, + db: Database, + config: ContrailConfig, + options: SpacesRoutesOptions = {}, + ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null +): void { + const spacesConfig = config.spaces; + if (!spacesConfig) return; + const authorityConfig = spacesConfig.authority; + if (!authorityConfig) return; + + const adapter = options.adapter ?? ctx?.adapter ?? new HostedAdapter(db, config); + const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); + const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); + + const localRecordHost = spacesConfig.recordHost ? adapter : null; + registerAuthorityRoutes( + app, + adapter, + authorityConfig, + config, + auth, + options.whoamiExtension, + localRecordHost + ); + + if (spacesConfig.recordHost) { + // Default in-process verifier: enrollment is the canonical binding + // source; Local-binding is a fallback for spaces created but not yet + // enrolled. Caller overrides via `options.credentialVerifier` to + // accept external authorities. + const credentialVerifier = + options.credentialVerifier ?? + (authorityConfig.signing + ? createBindingCredentialVerifier({ + bindings: createCompositeBindingResolver([ + createEnrollmentBindingResolver({ recordHost: adapter }), + createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), + ]), + keys: createLocalKeyResolver({ + authorityDid: authorityConfig.serviceDid, + publicKey: authorityConfig.signing.publicKey, + }), + }) + : undefined); + registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, credentialVerifier); + } +} diff --git a/packages/contrail-appview/src/core/spaces/schema.ts b/packages/contrail-appview/src/core/spaces/schema.ts new file mode 100644 index 0000000..ae33a7c --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/schema.ts @@ -0,0 +1,103 @@ +import type { ContrailConfig, Database } from "../types"; +import type { SqlDialect } from "../dialect"; +import { getDialect } from "../dialect"; +import { + buildCollectionTables, + buildDynamicIndexes, + buildFtsTables, + buildCountColumns, +} from "../db/schema"; + +/** Spaces metadata tables — spaces, members, invites. No per-collection tables. */ +export function buildSpacesBaseSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS spaces ( + uri TEXT PRIMARY KEY, + owner_did TEXT NOT NULL, + type TEXT NOT NULL, + key TEXT NOT NULL, + service_did TEXT NOT NULL, + app_policy_ref TEXT, + app_policy ${dialect.recordColumnType}, + created_at ${dialect.bigintType} NOT NULL, + deleted_at ${dialect.bigintType} + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_owner ON spaces(owner_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_type ON spaces(type)`, + + `CREATE TABLE IF NOT EXISTS spaces_members ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + added_at ${dialect.bigintType} NOT NULL, + added_by TEXT, + PRIMARY KEY (space_uri, did) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_members_did ON spaces_members(did)`, + + `CREATE TABLE IF NOT EXISTS spaces_blobs ( + space_uri TEXT NOT NULL, + cid TEXT NOT NULL, + mime_type TEXT NOT NULL, + size INTEGER NOT NULL, + author_did TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + PRIMARY KEY (space_uri, cid) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_author ON spaces_blobs(space_uri, author_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_created ON spaces_blobs(space_uri, created_at)`, + + `CREATE TABLE IF NOT EXISTS spaces_invites ( + token_hash TEXT PRIMARY KEY, + space_uri TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'join', + expires_at ${dialect.bigintType}, + max_uses INTEGER, + used_count INTEGER NOT NULL DEFAULT 0, + created_by TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + revoked_at ${dialect.bigintType}, + note TEXT + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, + + // Record-host enrollment table — the host's local cache of "spaces I + // accept records for, and which authority signs credentials for each." + // Filled by the recordHost.enroll endpoint, or auto-populated by the + // authority's createSpace when both roles run in the same process. + `CREATE TABLE IF NOT EXISTS record_host_enrollments ( + space_uri TEXT PRIMARY KEY, + authority_did TEXT NOT NULL, + enrolled_at ${dialect.bigintType} NOT NULL, + enrolled_by TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, + ]; +} + +/** Full spaces schema (base + per-collection tables + indexes). For callers + * that need a single array of statements. Note: this does NOT include FTS + * virtual tables or ALTER TABLE count columns — those must be applied with + * try/catch fallbacks and are handled by `initSchema`. */ +export function buildSpacesSchema(db: Database, config?: ContrailConfig): string[] { + const dialect = getDialect(db); + const base = buildSpacesBaseSchema(dialect); + if (!config) return base; + return [ + ...base, + ...buildCollectionTables(config, dialect, { forSpaces: true }), + ...buildDynamicIndexes(config, dialect, { forSpaces: true }), + ]; +} + +export async function initSpacesSchema(db: Database, config?: ContrailConfig): Promise<void> { + const dialect = getDialect(db); + const stmts = buildSpacesSchema(db, config); + await db.batch(stmts.map((s) => db.prepare(s))); + if (!config) return; + for (const stmt of buildFtsTables(config, dialect, { forSpaces: true })) { + try { await db.prepare(stmt).run(); } catch { /* ignore */ } + } + for (const stmt of buildCountColumns(config, { forSpaces: true })) { + try { await db.prepare(stmt).run(); } catch { /* ignore */ } + } +} diff --git a/packages/contrail-appview/src/core/spaces/tid.ts b/packages/contrail-appview/src/core/spaces/tid.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/tid.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/types.ts b/packages/contrail-appview/src/core/spaces/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/uri.ts b/packages/contrail-appview/src/core/spaces/uri.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/uri.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/types.ts b/packages/contrail-appview/src/core/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/index.ts b/packages/contrail-appview/src/index.ts new file mode 100644 index 0000000..1aad674 --- /dev/null +++ b/packages/contrail-appview/src/index.ts @@ -0,0 +1,65 @@ +/** @atmo-dev/contrail-appview — public-records appview for contrail. + * + * Owns: jetstream ingestion, backfill, refresh, query layer, per-collection + * XRPC routes, feeds, profiles, labels, the umbrella `createApp` that wires + * authority + record-host integrations, plus the `HostedAdapter` + * composition for in-process deployments. + * + * Re-exported wholesale from each module so consumers don't need to know + * internal path layout. */ + +// Forward base + authority + record-host so a consumer that imports +// `@atmo-dev/contrail-appview` (or a shim that re-exports it) sees the full +// shared surface in one place. Contrail bundle re-exports from here to keep +// its public API surface unchanged. +export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-authority"; +export * from "@atmo-dev/contrail-record-host"; + +// Indexing pipeline (jetstream, persistent, backfill, refresh, ingest helpers) +export * from "./core/jetstream"; +export * from "./core/persistent"; +export * from "./core/backfill"; +export * from "./core/refresh"; +export * from "./core/search"; + +// DB +export * from "./core/db/schema"; +export * from "./core/db/records"; +// note: ./core/db/index is implicitly covered by the wildcard if we export it +// — but we don't, since both schema and records may export overlapping names. +// Tests can import the specifics they need. + +// Router (createApp, registerCollectionRoutes, admin, feed, notify, profiles, hydrate) +export * from "./core/router"; +export * from "./core/router/notify"; +export * from "./core/router/profiles"; +export * from "./core/router/feed"; +export * from "./core/router/admin"; +export * from "./core/router/collection"; +export * from "./core/router/hydrate"; +export * from "./core/router/helpers"; + +// Spaces — re-exports + the bundle's HostedAdapter composition +export { HostedAdapter } from "./core/spaces/adapter"; +export { + registerSpacesRoutes, + registerAuthorityRoutes, + registerRecordHostRoutes, +} from "./core/spaces/router"; +export type { + SpacesRoutesOptions, + WhoamiExtension, +} from "./core/spaces/router"; + +// Realtime +export * from "./core/realtime"; + +// Labels +export * from "./core/labels/types"; +export * from "./core/labels/hydrate"; +export * from "./core/labels/select"; +export * from "./core/labels/apply"; +export * from "./core/labels/subscribe"; +export * from "./core/labels/resolve"; +export * from "./core/labels/schema"; diff --git a/packages/contrail-appview/tsconfig.build.json b/packages/contrail-appview/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-appview/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-appview/tsconfig.json b/packages/contrail-appview/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-appview/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-appview/tsup.config.ts b/packages/contrail-appview/tsup.config.ts new file mode 100644 index 0000000..eb81354 --- /dev/null +++ b/packages/contrail-appview/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: [ + "@atmo-dev/contrail-base", + "@atmo-dev/contrail-authority", + "@atmo-dev/contrail-record-host", + ], +}); diff --git a/packages/contrail-authority/src/index.ts b/packages/contrail-authority/src/index.ts index 638b762..8bd56ef 100644 --- a/packages/contrail-authority/src/index.ts +++ b/packages/contrail-authority/src/index.ts @@ -1,8 +1,7 @@ /** @atmo-dev/contrail-authority — default space-authority implementation. * * Owns the authority-side adapter (member list, invites, app policy, space - * lifecycle) and DDL. Route registration currently lives in - * @atmo-dev/contrail and will move here in a subsequent extraction pass. */ + * lifecycle, credential issuance), DDL, and route registration. */ export { HostedAuthorityAdapter, @@ -17,3 +16,8 @@ export { buildAuthoritySchema, applyAuthoritySchema, } from "./schema"; + +export { registerAuthorityRoutes } from "./routes"; + +export { registerInviteRoutes } from "./invite-routes"; +export type { InviteRoutesOptions } from "./invite-routes"; diff --git a/packages/contrail-authority/src/invite-routes.ts b/packages/contrail-authority/src/invite-routes.ts new file mode 100644 index 0000000..dd0fc04 --- /dev/null +++ b/packages/contrail-authority/src/invite-routes.ts @@ -0,0 +1,240 @@ +/** Unified invite surface: a single `<ns>.invite.*` family serving both + * user-owned spaces (handled here directly via the authority adapter) and + * community-owned spaces (delegated to a {@link CommunityInviteHandler}). + * + * Storage stays separate (`spaces_invites` vs `community_invites` tables) — + * schemas differ enough that unifying them would be net-negative. The token + * primitive and HTTP dance are shared. */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { + CommunityInviteHandler, + ContrailConfig, + HandlerResponse, + InviteKind, + InviteRow, + ServiceAuth, + SpaceAuthority, +} from "@atmo-dev/contrail-base"; +import { hashInviteToken, mintInviteToken } from "@atmo-dev/contrail-base"; + +export interface InviteRoutesOptions { + authMiddleware: MiddlewareHandler; +} + +interface PublicInviteView { + tokenHash: string; + spaceUri: string; + kind?: InviteKind; + createdBy: string; + createdAt: number; + expiresAt: number | null; + maxUses: number | null; + usedCount: number; + revokedAt: number | null; + note: string | null; +} + +function toSpacesView(row: InviteRow): PublicInviteView { + return { + tokenHash: row.tokenHash, + spaceUri: row.spaceUri, + kind: row.kind, + createdBy: row.createdBy, + createdAt: row.createdAt, + expiresAt: row.expiresAt, + maxUses: row.maxUses, + usedCount: row.usedCount, + revokedAt: row.revokedAt, + note: row.note, + }; +} + +export function registerInviteRoutes( + app: Hono, + config: ContrailConfig, + authority: SpaceAuthority, + community: CommunityInviteHandler | null, + options: InviteRoutesOptions +): void { + if (!config.spaces?.authority) return; + + const NS = `${config.namespace}.invite`; + const auth = options.authMiddleware; + + const classifySpace = async (spaceUri: string) => { + const space = await authority.getSpace(spaceUri); + if (!space) return null; + const isCommunity = community ? await community.isCommunityOwned(spaceUri) : false; + return { space, isCommunity }; + }; + + app.post(`/xrpc/${NS}.create`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { + spaceUri?: string; + kind?: string; + accessLevel?: string; + expiresAt?: number; + maxUses?: number; + note?: string; + } + | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + if (body.kind && body.accessLevel) { + return c.json( + { error: "InvalidRequest", reason: "kind-or-accessLevel", message: "pass kind OR accessLevel, not both" }, + 400 + ); + } + + const classified = await classifySpace(body.spaceUri); + if (!classified) return c.json({ error: "NotFound" }, 404); + const { space, isCommunity } = classified; + + if (isCommunity) { + if (!community) return c.json({ error: "InvalidState" }, 500); + return relay(c, await community.create({ + spaceUri: body.spaceUri, + callerDid: sa.issuer, + accessLevel: body.accessLevel, + kind: body.kind, + expiresAt: body.expiresAt ?? null, + maxUses: body.maxUses ?? null, + note: body.note ?? null, + })); + } + + if (body.accessLevel) { + return c.json( + { error: "InvalidRequest", reason: "accessLevel-on-user-space", message: "user-owned spaces take kind, not accessLevel" }, + 400 + ); + } + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const kind = (body.kind ?? "join") as InviteKind; + if (kind !== "join" && kind !== "read" && kind !== "read-join") { + return c.json({ error: "InvalidRequest", message: "kind must be 'join', 'read', or 'read-join'" }, 400); + } + const { token, tokenHash } = await mintInviteToken(); + const invite = await authority.createInvite({ + spaceUri: body.spaceUri, + tokenHash, + kind, + expiresAt: body.expiresAt ?? null, + maxUses: body.maxUses ?? null, + createdBy: sa.issuer, + note: body.note ?? null, + }); + return c.json({ token, invite: toSpacesView(invite) }); + }); + + app.get(`/xrpc/${NS}.list`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const includeRevoked = c.req.query("includeRevoked") === "true"; + + const classified = await classifySpace(spaceUri); + if (!classified) return c.json({ error: "NotFound" }, 404); + const { space, isCommunity } = classified; + + if (isCommunity) { + return relay(c, await community!.list({ + spaceUri, + callerDid: sa.issuer, + includeRevoked, + })); + } + + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const rows = await authority.listInvites(spaceUri, { includeRevoked }); + return c.json({ invites: rows.map(toSpacesView) }); + }); + + app.post(`/xrpc/${NS}.revoke`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; tokenHash?: string } + | null; + if (!body?.tokenHash) { + return c.json({ error: "InvalidRequest", message: "tokenHash required" }, 400); + } + + if (body.spaceUri) { + const classified = await classifySpace(body.spaceUri); + if (!classified) return c.json({ error: "NotFound" }, 404); + if (classified.isCommunity) { + return relay(c, await community!.revoke({ + spaceUri: body.spaceUri, + tokenHash: body.tokenHash, + callerDid: sa.issuer, + })); + } + if (classified.space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const ok = await authority.revokeInvite(body.tokenHash); + return c.json({ ok }); + } + + if (community) { + const r = await community.tryRevokeByToken({ + tokenHash: body.tokenHash, + callerDid: sa.issuer, + }); + if (r) return relay(c, r); + } + const srow = await authority.getInvite(body.tokenHash); + if (!srow) return c.json({ error: "NotFound" }, 404); + const space = await authority.getSpace(srow.spaceUri); + if (space && space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const ok = await authority.revokeInvite(body.tokenHash); + return c.json({ ok }); + }); + + app.post(`/xrpc/${NS}.redeem`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { token?: string } | null; + if (!body?.token) { + return c.json({ error: "InvalidRequest", message: "token required" }, 400); + } + const tokenHash = await hashInviteToken(body.token); + const now = Date.now(); + + if (community) { + const r = await community.tryRedeem({ + tokenHash, + callerDid: sa.issuer, + now, + }); + if (r) return relay(c, r); + } + + const sinvite = await authority.redeemInvite(tokenHash, now); + if (!sinvite) { + return c.json({ error: "InvalidInvite", reason: "expired-revoked-or-exhausted" }, 400); + } + await authority.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); + return c.json({ spaceUri: sinvite.spaceUri, kind: sinvite.kind }); + }); +} + +function relay(c: Context, r: HandlerResponse) { + return c.json(r.body, r.status as Parameters<typeof c.json>[1]); +} + +function getAuth(c: Context): ServiceAuth { + const a = c.get("serviceAuth") as ServiceAuth | undefined; + if (!a) throw new Error("service auth not set"); + return a; +} diff --git a/packages/contrail-authority/src/routes.ts b/packages/contrail-authority/src/routes.ts new file mode 100644 index 0000000..fe80e37 --- /dev/null +++ b/packages/contrail-authority/src/routes.ts @@ -0,0 +1,412 @@ +/** Authority XRPC routes — space lifecycle, members, app policy, whoami, + * credential issuance. Does NOT touch records or blobs. + * + * Deployments wire this via `registerAuthorityRoutes(app, authority, ...)`. + * The umbrella `registerSpacesRoutes` in @atmo-dev/contrail composes this + * with the record-host routes; split deployments call this directly. */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { + AuthorityConfig, + ContrailConfig, + CredentialClaims, + RecordHost, + ServiceAuth, + SpaceAuthority, + SpaceRow, + WhoamiExtension, +} from "@atmo-dev/contrail-base"; +import { + buildSpaceUri, + checkInviteReadGrant, + decodeUnverifiedClaims, + DEFAULT_CREDENTIAL_TTL_MS, + extractInviteToken, + hashInviteToken, + issueCredential, + nextTid, + verifyCredential, +} from "@atmo-dev/contrail-base"; + +/** When `localRecordHost` is non-null, the authority's `createSpace` handler + * also enrolls the new space on that host — convenient for in-process + * deployments where the same operator runs both roles. Split deployments + * pass `null` and arrange enrollment explicitly via `recordHost.enroll`. */ +export function registerAuthorityRoutes( + app: Hono, + authority: SpaceAuthority, + authorityConfig: AuthorityConfig, + config: ContrailConfig, + auth: MiddlewareHandler, + whoamiExtension?: WhoamiExtension, + localRecordHost?: RecordHost | null +): void { + /** Space endpoints are emitted per-deployment under the configured namespace; + * the deployment owns and publishes its own lexicons. */ + const SPACE = `${config.namespace}.space`; + const SPACE_EXT = `${config.namespace}.spaceExt`; + + // ---- Read endpoints ---- + + app.get(`/xrpc/${SPACE}.listSpaces`, auth, async (c) => { + const sa = getAuth(c); + const scope = c.req.query("scope") ?? "member"; + const type = c.req.query("type") ?? undefined; + const owner = c.req.query("owner") ?? undefined; + const cursor = c.req.query("cursor") ?? undefined; + const limit = c.req.query("limit") ? Number(c.req.query("limit")) : undefined; + + const opts: Parameters<typeof authority.listSpaces>[0] = { type, cursor, limit }; + if (scope === "owner") opts.ownerDid = sa.issuer; + else { + opts.memberDid = sa.issuer; + if (owner) opts.ownerDid = owner; + } + + const result = await authority.listSpaces(opts); + return c.json({ + spaces: result.spaces.map((s) => publicSpaceView(s, s.ownerDid === sa.issuer)), + cursor: result.cursor, + }); + }); + + app.get(`/xrpc/${SPACE}.listMembers`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + const member = isOwner ? null : await authority.getMember(spaceUri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + const members = await authority.listMembers(spaceUri); + return c.json({ members }); + }); + + /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is + * present so anonymous bearer reads don't 401 before the route handler can + * validate the token. */ + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return auth(c, next); + }; + + app.get(`/xrpc/${SPACE}.getSpace`, readAuth, async (c) => { + const uri = c.req.query("uri"); + if (!uri) return c.json({ error: "InvalidRequest", message: "uri required" }, 400); + const space = await authority.getSpace(uri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const authz = await authorizeRead(c, authority, uri); + if (authz instanceof Response) return authz; + + if (authz.via === "token") { + return c.json({ space: publicSpaceView(space, false) }); + } + + if (authz.via === "credential") { + const isOwner = authz.claims.sub === space.ownerDid; + return c.json({ space: publicSpaceView(space, isOwner) }); + } + + const sa = authz.sa; + const isOwner = sa.issuer === space.ownerDid; + const member = isOwner ? null : await authority.getMember(uri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + return c.json({ space: publicSpaceView(space, isOwner) }); + }); + + // ---- Space management (owner-gated) ---- + + app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => ({}))) as { + type?: string; + key?: string; + appPolicy?: SpaceRow["appPolicy"]; + appPolicyRef?: string; + }; + + const type = body.type ?? authorityConfig.type; + const key = body.key ?? nextTid(); + const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); + + const existing = await authority.getSpace(uri); + if (existing) return c.json({ error: "AlreadyExists", uri }, 409); + + const space = await authority.createSpace({ + uri, + ownerDid: sa.issuer, + type, + key, + serviceDid: authorityConfig.serviceDid, + appPolicyRef: body.appPolicyRef ?? null, + appPolicy: body.appPolicy ?? authorityConfig.defaultAppPolicy ?? null, + }); + await authority.addMember(uri, sa.issuer, sa.issuer); + + if (localRecordHost) { + await localRecordHost.enroll({ + spaceUri: uri, + authorityDid: authorityConfig.serviceDid, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + } + + return c.json({ space: publicSpaceView(space, true) }); + }); + + app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + await authority.addMember(body.spaceUri, body.did, sa.issuer); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + if (body.did === space.ownerDid) { + return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); + } + await authority.removeMember(body.spaceUri, body.did); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid === sa.issuer) { + return c.json( + { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, + 400 + ); + } + await authority.removeMember(body.spaceUri, sa.issuer); + return c.json({ ok: true }); + }); + + // Unified whoami — extension can override with richer data (e.g. community + // accessLevel); without one, returns binary owner/member. + app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + + if (whoamiExtension) { + const ext = await whoamiExtension({ + spaceUri, + callerDid: sa.issuer, + isOwner, + ownerDid: space.ownerDid, + }); + if (ext) return c.json(ext); + } + + if (isOwner) return c.json({ isOwner: true, isMember: true }); + const member = await authority.getMember(spaceUri, sa.issuer); + return c.json({ isOwner: false, isMember: !!member }); + }); + + // ---- Credential endpoints ---- + + app.post(`/xrpc/${SPACE}.getCredential`, auth, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + const member = isOwner ? null : await authority.getMember(body.spaceUri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + if (space.appPolicy) { + const allowed = checkClientId(space.appPolicy, sa.clientId); + if (!allowed) return c.json({ error: "Forbidden", reason: "app-not-allowed" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: sa.issuer, + space: body.spaceUri, + scope: "rw", + ttlMs: ttl, + }, + authorityConfig.signing + ); + return c.json({ credential, expiresAt }); + }); + + app.post(`/xrpc/${SPACE}.refreshCredential`, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const body = (await c.req.json().catch(() => null)) as { credential?: string } | null; + if (!body?.credential) { + return c.json({ error: "InvalidRequest", message: "credential required" }, 400); + } + const signing = authorityConfig.signing; + const claims = await verifyAndAuthorizeRefresh(body.credential, authorityConfig); + if ("error" in claims) return c.json(claims, claims.status); + + const space = await authority.getSpace(claims.space); + if (!space) return c.json({ error: "NotFound" }, 404); + const isOwner = space.ownerDid === claims.sub; + const member = isOwner ? null : await authority.getMember(claims.space, claims.sub); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: claims.sub, + space: claims.space, + scope: claims.scope, + ttlMs: ttl, + }, + signing + ); + return c.json({ credential, expiresAt }); + }); +} + +/** Verify a credential presented at refreshCredential. */ +async function verifyAndAuthorizeRefresh( + credential: string, + authorityConfig: AuthorityConfig +): Promise<CredentialClaims | { error: string; reason?: string; message?: string; status: 400 | 401 }> { + const peek = decodeUnverifiedClaims(credential); + if (!peek) return { error: "InvalidRequest", reason: "malformed", status: 400 }; + if (peek.iss !== authorityConfig.serviceDid) { + return { error: "Forbidden", reason: "wrong-issuer", status: 401 }; + } + if (!authorityConfig.signing) { + return { error: "InvalidState", status: 401 }; + } + const signing = authorityConfig.signing; + const result = await verifyCredential(credential, { + expectedSpace: peek.space, + resolveKey: async (iss) => (iss === authorityConfig.serviceDid ? signing.publicKey : null), + }); + if (!result.ok) { + return { error: "InvalidCredential", reason: result.reason, status: 401 }; + } + return result.claims; +} + +function checkClientId( + appPolicy: NonNullable<SpaceRow["appPolicy"]>, + clientId: string | undefined +): boolean { + const listed = clientId ? appPolicy.apps.includes(clientId) : false; + if (appPolicy.mode === "allow") return !listed; + return listed; +} + +/** Authorize a read request on the authority side — three valid paths: + * credential (set by upstream middleware), invite token, or service-auth JWT. */ +async function authorizeRead( + c: Context, + authority: SpaceAuthority, + spaceUri: string +): Promise< + | { via: "credential"; claims: CredentialClaims } + | { via: "token" } + | { via: "jwt"; sa: ServiceAuth } + | Response +> { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== spaceUri) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + return { via: "credential", claims: cred }; + } + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); + if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); + return { via: "token" }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (sa) return { via: "jwt", sa }; + return c.json( + { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, + 401 + ); +} + +function getAuth(c: Context): ServiceAuth { + const auth = c.get("serviceAuth") as ServiceAuth | undefined; + if (!auth) throw new Error("service auth not set"); + return auth; +} + +function publicSpaceView(space: SpaceRow, forOwner: boolean) { + return { + uri: space.uri, + ownerDid: space.ownerDid, + type: space.type, + key: space.key, + serviceDid: space.serviceDid, + appPolicyRef: space.appPolicyRef, + createdAt: space.createdAt, + ...(forOwner ? { appPolicy: space.appPolicy } : {}), + }; +} diff --git a/packages/contrail-record-host/package.json b/packages/contrail-record-host/package.json new file mode 100644 index 0000000..2182f37 --- /dev/null +++ b/packages/contrail-record-host/package.json @@ -0,0 +1,43 @@ +{ + "name": "@atmo-dev/contrail-record-host", + "version": "0.6.0", + "description": "Default record-host implementation for contrail — stores records and blobs for permissioned spaces, enforces local enrollment as the host's consent layer.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-record-host" + }, + "keywords": [ + "atproto", + "contrail" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/cid": "^2.4.1", + "@atmo-dev/contrail-base": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "license": "MIT" +} diff --git a/packages/contrail-record-host/src/adapter.ts b/packages/contrail-record-host/src/adapter.ts new file mode 100644 index 0000000..4bfb321 --- /dev/null +++ b/packages/contrail-record-host/src/adapter.ts @@ -0,0 +1,530 @@ +/** Default {@link RecordHost} implementation backed by a Database. + * + * Owns the host-side tables — `spaces_records_<short>` (per-collection), + * `spaces_blobs`, `record_host_enrollments` — and exposes record CRUD, + * blob metadata management, enrollment, and `findOrphanBlobs` for GC. + * + * Independent of the authority adapter: takes a Database directly, doesn't + * inherit from anything. Bundles that want a single adapter satisfying both + * roles instantiate this alongside HostedAuthorityAdapter against the same + * DB. The duplication of the underlying tables is fine — `IF NOT EXISTS` + * guards make schema application idempotent. */ + +import type { + ContrailConfig, + Database, + RelationConfig, + ResolvedContrailConfig, + RecordHost, + BlobMetaRow, + CollectionCount, + EnrollmentRow, + ListBlobsOptions, + ListBlobsResult, + ListOptions, + ListResult, + StoredRecord, +} from "@atmo-dev/contrail-base"; +import { + shortNameForNsid, + spacesRecordsTableName, + countColumnName, + groupedCountColumnName, + getRelationField, + getNestedValue, + getDialect, + buildRecordUri, +} from "@atmo-dev/contrail-base"; + +function parseJson<T>(value: unknown): T | null { + if (value == null) return null; + if (typeof value === "string") { + try { + return JSON.parse(value) as T; + } catch { + return null; + } + } + return value as T; +} + +function toNum(v: unknown): number { + return typeof v === "string" ? Number(v) : (v as number); +} + +export function mapBlobMetaRow(row: any): BlobMetaRow { + return { + spaceUri: row.space_uri, + cid: row.cid, + mimeType: row.mime_type, + size: Number(row.size), + authorDid: row.author_did, + createdAt: toNum(row.created_at), + }; +} + +export function mapEnrollmentRow(row: any): EnrollmentRow { + return { + spaceUri: row.space_uri, + authorityDid: row.authority_did, + enrolledAt: toNum(row.enrolled_at), + enrolledBy: row.enrolled_by, + }; +} + +/** Row mapper for per-collection spaces_records_<short> tables. + * `collection` is injected by the caller (known from the table name). */ +export function mapRecordRow(row: any, collection: string): StoredRecord { + return { + spaceUri: row.space_uri, + collection, + authorDid: row.did, + rkey: row.rkey, + cid: row.cid ?? null, + record: parseJson<Record<string, unknown>>(row.record) ?? {}, + createdAt: toNum(row.time_us), + }; +} + +export class HostedRecordHostAdapter implements RecordHost { + constructor( + protected readonly db: Database, + protected readonly config?: ContrailConfig + ) {} + + /** Resolve the per-collection spaces table name, or throw if the collection + * isn't configured (and therefore has no table). */ + protected tableFor(collection: string): string { + if (!this.config) { + throw new Error( + `HostedRecordHostAdapter: config not provided; cannot resolve table for collection ${collection}` + ); + } + const short = shortNameForNsid(this.config, collection); + if (!short) { + throw new Error( + `HostedRecordHostAdapter: collection ${collection} is not configured in this deployment` + ); + } + return spacesRecordsTableName(short); + } + + // ---- Enrollment ---- + + async enroll(input: EnrollmentRow): Promise<void> { + await this.db + .prepare( + `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri) DO UPDATE SET + authority_did = excluded.authority_did, + enrolled_at = excluded.enrolled_at, + enrolled_by = excluded.enrolled_by` + ) + .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) + .run(); + } + + async getEnrollment(spaceUri: string): Promise<EnrollmentRow | null> { + const row = await this.db + .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .first<any>(); + return row ? mapEnrollmentRow(row) : null; + } + + async listEnrollments( + options: { authorityDid?: string; limit?: number } = {} + ): Promise<EnrollmentRow[]> { + const limit = Math.min(options.limit ?? 200, 1000); + if (options.authorityDid) { + const { results } = await this.db + .prepare( + `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` + ) + .bind(options.authorityDid, limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + const { results } = await this.db + .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) + .bind(limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + + async removeEnrollment(spaceUri: string): Promise<void> { + await this.db + .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .run(); + } + + // ---- Records ---- + + async putRecord(record: StoredRecord): Promise<void> { + const table = this.tableFor(record.collection); + const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); + + const childShort = this.config ? shortNameForNsid(this.config, record.collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(record.spaceUri, record.authorDid, record.rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `INSERT INTO ${table} (space_uri, uri, did, rkey, cid, record, time_us, indexed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, did, rkey) DO UPDATE SET + uri = excluded.uri, + cid = excluded.cid, + record = excluded.record, + time_us = excluded.time_us, + indexed_at = excluded.indexed_at` + ) + .bind( + record.spaceUri, + uri, + record.authorDid, + record.rkey, + record.cid, + JSON.stringify(record.record), + record.createdAt, + Date.now() + ) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace( + record.spaceUri, + childShort, + beforeRecord, + record.record, + record.authorDid + ); + } + } + + async getRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<StoredRecord | null> { + const table = this.tableFor(collection); + const row = await this.db + .prepare( + `SELECT * FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .first<any>(); + return row ? mapRecordRow(row, collection) : null; + } + + async listRecords( + spaceUri: string, + collection: string, + options: ListOptions = {} + ): Promise<ListResult> { + const table = this.tableFor(collection); + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("time_us < ?"); + params.push(Number(options.cursor)); + } + + const sql = `SELECT * FROM ${table} + WHERE ${clauses.join(" AND ")} + ORDER BY time_us DESC + LIMIT ?`; + params.push(limit + 1); + + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const records = results.map((r) => mapRecordRow(r, collection)); + let cursor: string | undefined; + if (records.length > limit) { + const next = records.pop()!; + cursor = String(next.createdAt); + } + return { records, cursor }; + } + + async deleteRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<void> { + const table = this.tableFor(collection); + + const childShort = this.config ? shortNameForNsid(this.config, collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(spaceUri, authorDid, rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `DELETE FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace(spaceUri, childShort, beforeRecord, null, authorDid); + } + } + + /** Recompute count columns on parent records in the same space, scoped to the + * targets derived from before/after versions of the written/deleted child record. */ + protected async recountParentsForSpace( + spaceUri: string, + childShort: string, + before: Record<string, unknown> | null, + after: Record<string, unknown> | null, + childDid: string + ): Promise<void> { + if (!this.config) return; + const config = this.config; + const resolved = (config as ResolvedContrailConfig)._resolved; + const childTable = spacesRecordsTableName(childShort); + + type Inbound = { parentShort: string; relationName: string; rel: RelationConfig }; + const inbound: Inbound[] = []; + for (const [parentShort, parentCfg] of Object.entries(config.collections)) { + if (parentCfg.allowInSpaces === false) continue; + for (const [relName, rel] of Object.entries(parentCfg.relations ?? {})) { + if (rel.count === false) continue; + if (rel.collection !== childShort) continue; + inbound.push({ parentShort, relationName: relName, rel }); + } + } + if (inbound.length === 0) return; + + // Deduplicate (parent, relation, target) across before/after. + const keyed = new Map<string, { parentShort: string; relationName: string; rel: RelationConfig; target: string }>(); + for (const { parentShort, relationName, rel } of inbound) { + const field = getRelationField(rel); + const collectTarget = (rec: Record<string, unknown> | null) => { + if (!rec) return; + if (rel.match === "did") { + keyed.set(`${parentShort}:${relationName}:${childDid}`, { + parentShort, relationName, rel, target: childDid, + }); + return; + } + const v = getNestedValue(rec, field); + if (typeof v === "string" && v.length > 0) { + keyed.set(`${parentShort}:${relationName}:${v}`, { + parentShort, relationName, rel, target: v, + }); + } + }; + collectTarget(before); + collectTarget(after); + } + if (keyed.size === 0) return; + + const dialect = getDialect(this.db); + const stmts: ReturnType<Database["prepare"]>[] = []; + + for (const { parentShort, relationName, rel, target } of keyed.values()) { + const parentTable = spacesRecordsTableName(parentShort); + const matchColumn = rel.match === "did" ? "did" : "uri"; + const field = getRelationField(rel); + const countExpr = rel.countDistinct + ? `COUNT(DISTINCT ${rel.countDistinct})` + : "COUNT(*)"; + + const setClauses: string[] = []; + const binds: (string | number)[] = []; + + const totalCol = countColumnName(rel.collection); + setClauses.push( + `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ?)` + ); + binds.push(spaceUri, target); + + if (rel.groupBy) { + const mapping = resolved?.relations[parentShort]?.[relationName]; + if (mapping?.groups) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + setClauses.push( + `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ? AND ${dialect.jsonExtract("record", rel.groupBy)} = ?)` + ); + binds.push(spaceUri, target, fullToken); + } + } + } + + binds.push(spaceUri, target); + stmts.push( + this.db + .prepare( + `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE space_uri = ? AND ${matchColumn} = ?` + ) + .bind(...binds) + ); + } + + if (stmts.length > 0) await this.db.batch(stmts); + } + + async listCollections( + spaceUri: string, + options: { byUser?: string } = {} + ): Promise<CollectionCount[]> { + if (!this.config) return []; + const results: CollectionCount[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + const table = spacesRecordsTableName(short); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + try { + const row = await this.db + .prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${clauses.join(" AND ")}`) + .bind(...params) + .first<{ count: number }>(); + const count = Number(row?.count ?? 0); + if (count > 0) results.push({ collection: colConfig.collection, count }); + } catch { + // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip + } + } + return results; + } + + // ---- Blobs ---- + + async putBlobMeta(row: BlobMetaRow): Promise<void> { + const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, cid) DO NOTHING`; + await this.db + .prepare(sql) + .bind(row.spaceUri, row.cid, row.mimeType, row.size, row.authorDid, row.createdAt) + .run(); + } + + async getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null> { + const r = await this.db + .prepare(`SELECT * FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .first<any>(); + return r ? mapBlobMetaRow(r) : null; + } + + async listBlobMeta( + spaceUri: string, + options: ListBlobsOptions = {} + ): Promise<ListBlobsResult> { + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("author_did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("created_at < ?"); + params.push(Number(options.cursor)); + } + const sql = `SELECT * FROM spaces_blobs + WHERE ${clauses.join(" AND ")} + ORDER BY created_at DESC + LIMIT ?`; + params.push(limit + 1); + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const blobs = results.map(mapBlobMetaRow); + let cursor: string | undefined; + if (blobs.length > limit) { + const next = blobs.pop()!; + cursor = String(next.createdAt); + } + return { blobs, cursor }; + } + + async deleteBlobMeta(spaceUri: string, cid: string): Promise<void> { + await this.db + .prepare(`DELETE FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .run(); + } + + async findOrphanBlobs( + spaceUri: string, + cutoff: number, + limit: number + ): Promise<BlobMetaRow[]> { + if (!this.config) return []; + // Gather candidate blobs older than cutoff, then filter out any whose CID + // appears in any record JSON in this space. We use a cheap substring probe + // (LIKE) per collection — false positives are OK because an orphan that + // survives GC just gets collected next cycle; false negatives (deleting + // a referenced blob) would be a bug, and substring search over the full + // CID is safe enough for that. + const { results } = await this.db + .prepare( + `SELECT * FROM spaces_blobs + WHERE space_uri = ? AND created_at < ? + ORDER BY created_at ASC + LIMIT ?` + ) + .bind(spaceUri, cutoff, limit) + .all<any>(); + const candidates = results.map(mapBlobMetaRow); + if (candidates.length === 0) return []; + + const tables: string[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + tables.push(spacesRecordsTableName(short)); + } + + const orphans: BlobMetaRow[] = []; + for (const blob of candidates) { + let referenced = false; + const pattern = `%${blob.cid}%`; + for (const table of tables) { + try { + const row = await this.db + .prepare( + `SELECT 1 FROM ${table} WHERE space_uri = ? AND record LIKE ? LIMIT 1` + ) + .bind(spaceUri, pattern) + .first<any>(); + if (row) { + referenced = true; + break; + } + } catch { + // table missing — ignore + } + } + if (!referenced) orphans.push(blob); + } + return orphans; + } +} diff --git a/packages/contrail-record-host/src/blob-gc.ts b/packages/contrail-record-host/src/blob-gc.ts new file mode 100644 index 0000000..19ca37f --- /dev/null +++ b/packages/contrail-record-host/src/blob-gc.ts @@ -0,0 +1,37 @@ +import type { BlobAdapter, RecordHost } from "@atmo-dev/contrail-base"; +import { blobKey } from "@atmo-dev/contrail-base"; + +export interface BlobGcOptions { + /** Orphan rows created before this timestamp are eligible for deletion. */ + olderThan: number; + /** Maximum number of blobs to delete in this pass. Defaults to 500. */ + batchSize?: number; +} + +export interface BlobGcResult { + deleted: number; + cids: string[]; +} + +/** Delete blob bytes + metadata for any blob older than `olderThan` that + * is not referenced by any record in the space. Safe to run periodically. */ +export async function gcOrphanBlobs( + storage: RecordHost, + blobs: BlobAdapter, + spaceUri: string, + options: BlobGcOptions +): Promise<BlobGcResult> { + const batchSize = options.batchSize ?? 500; + const orphans = await storage.findOrphanBlobs(spaceUri, options.olderThan, batchSize); + if (orphans.length === 0) return { deleted: 0, cids: [] }; + + const keys: string[] = []; + for (const row of orphans) { + keys.push(await blobKey(row.spaceUri, row.cid)); + } + await blobs.delete(keys); + for (const row of orphans) { + await storage.deleteBlobMeta(row.spaceUri, row.cid); + } + return { deleted: orphans.length, cids: orphans.map((o) => o.cid) }; +} diff --git a/packages/contrail-record-host/src/blob-refs.ts b/packages/contrail-record-host/src/blob-refs.ts new file mode 100644 index 0000000..62cf693 --- /dev/null +++ b/packages/contrail-record-host/src/blob-refs.ts @@ -0,0 +1,27 @@ +/** + * Walk a record JSON and collect every atproto blob ref. + * + * Blob refs look like: + * { "$type": "blob", "ref": { "$link": "<cid>" }, "mimeType": "...", "size": N } + * + * We return the CID strings. + */ +export function collectBlobCids(value: unknown, out: Set<string> = new Set()): Set<string> { + if (value == null) return out; + if (Array.isArray(value)) { + for (const v of value) collectBlobCids(v, out); + return out; + } + if (typeof value !== "object") return out; + + const obj = value as Record<string, unknown>; + if (obj["$type"] === "blob") { + const ref = obj["ref"] as { $link?: unknown } | undefined; + if (ref && typeof ref["$link"] === "string") out.add(ref["$link"]); + // Don't descend — a blob ref's own shape has no nested blobs. + return out; + } + + for (const v of Object.values(obj)) collectBlobCids(v, out); + return out; +} diff --git a/packages/contrail-record-host/src/index.ts b/packages/contrail-record-host/src/index.ts new file mode 100644 index 0000000..ff4eacd --- /dev/null +++ b/packages/contrail-record-host/src/index.ts @@ -0,0 +1,27 @@ +/** @atmo-dev/contrail-record-host — default record-host implementation. + * + * Owns the host-side adapter (records, blobs, enrollment) + DDL + blob-GC. + * Independent of the authority — takes a Database directly, doesn't inherit + * from anything. Bundles can instantiate this alongside HostedAuthorityAdapter + * against the same DB to get full StorageAdapter behavior. */ + +export { + HostedRecordHostAdapter, + mapBlobMetaRow, + mapEnrollmentRow, + mapRecordRow, +} from "./adapter"; + +export { + buildRecordHostBaseSchema, + applyRecordHostSchema, +} from "./schema"; + +export { + gcOrphanBlobs, +} from "./blob-gc"; +export type { BlobGcOptions, BlobGcResult } from "./blob-gc"; + +export { collectBlobCids } from "./blob-refs"; + +export { registerRecordHostRoutes } from "./routes"; diff --git a/packages/contrail-record-host/src/routes.ts b/packages/contrail-record-host/src/routes.ts new file mode 100644 index 0000000..63fc951 --- /dev/null +++ b/packages/contrail-record-host/src/routes.ts @@ -0,0 +1,510 @@ +/** Record-host XRPC routes — record + blob CRUD plus enrollment. + * + * Auth precedence on every route: + * 1. `X-Space-Credential` header (if a verifier is wired and the credential + * is valid) — caller DID = credential `sub`, no clientId. + * 2. Read-route invite token (`?inviteToken=` or `Bearer atmo-invite:...`). + * 3. Service-auth JWT — caller DID = JWT issuer. + * + * When a credential is presented, the record host trusts it: no member + * check, no app-policy check (those happen at issuance time on the + * authority side). Service-auth requests still consult the authority. */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { + ContrailConfig, + CredentialClaims, + CredentialScope, + CredentialVerifier, + RecordHost, + RecordHostConfig, + ServiceAuth, + SpaceAuthority, +} from "@atmo-dev/contrail-base"; +import { + blobKey, + checkAccess, + checkInviteReadGrant, + DEFAULT_BLOB_MAX_SIZE, + extractInviteToken, + extractSpaceCredential, + hashInviteToken, + nextTid, + parseSpaceUri, +} from "@atmo-dev/contrail-base"; +import { collectBlobCids } from "./blob-refs"; +import { create as createCid, toString as cidToString } from "@atcute/cid"; + +export function registerRecordHostRoutes( + app: Hono, + recordHost: RecordHost, + authority: SpaceAuthority, + recordHostConfig: RecordHostConfig, + config: ContrailConfig, + auth: MiddlewareHandler, + /** Optional credential verifier. When present, the record host accepts + * `X-Space-Credential` as an alternative to a service-auth JWT. */ + credentialVerifier?: CredentialVerifier +): void { + const SPACE = `${config.namespace}.space`; + + /** Auth wrapper: tries credential first, then delegates to JWT auth. */ + const authWithCredential: MiddlewareHandler = async (c, next) => { + const credToken = extractSpaceCredential(c.req.raw); + if (credToken) { + if (!credentialVerifier) { + return c.json( + { error: "AuthRequired", reason: "credential-verifier-not-configured" }, + 401 + ); + } + const result = await credentialVerifier.verify(credToken); + if (!result.ok) { + return c.json({ error: "AuthRequired", reason: result.reason }, 401); + } + c.set("spaceCredential", result.claims); + await next(); + return; + } + return auth(c, next); + }; + + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return authWithCredential(c, next); + }; + + /** Hard gate on every record-host operation: the space must be enrolled. */ + async function requireEnrollment( + c: Context, + spaceUri: string + ): Promise<{ authorityDid: string } | Response> { + const enrollment = await recordHost.getEnrollment(spaceUri); + if (!enrollment) { + return c.json( + { error: "NotFound", reason: "not-enrolled", message: "space is not enrolled on this record host" }, + 404 + ); + } + return enrollment; + } + + // ---- Enrollment endpoint ---- + const RECORD_HOST = `${config.namespace}.recordHost`; + + app.post(`/xrpc/${RECORD_HOST}.enroll`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; authority?: string } + | null; + if (!body?.spaceUri || !body.authority) { + return c.json({ error: "InvalidRequest", message: "spaceUri and authority required" }, 400); + } + const parts = parseSpaceUri(body.spaceUri); + if (!parts) { + return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); + } + const callerIsOwner = sa.issuer === parts.ownerDid; + const callerIsAuthority = sa.issuer === body.authority; + if (!callerIsOwner && !callerIsAuthority) { + return c.json( + { error: "Forbidden", reason: "not-owner-or-authority" }, + 403 + ); + } + await recordHost.enroll({ + spaceUri: body.spaceUri, + authorityDid: body.authority, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + return c.json({ ok: true }); + }); + + app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { + const spaceUri = c.req.query("spaceUri"); + const collection = c.req.query("collection"); + if (!spaceUri || !collection) { + return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const authz = await authorizeRead(c, authority, spaceUri); + if (authz instanceof Response) return authz; + + if (authz.via === "jwt") { + const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, sa.issuer); + const result = checkAccess({ + op: "read", + space, + callerDid: sa.issuer, + member, + clientId: sa.clientId, + }); + if (!result.allow) { + return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + } + + const list = await recordHost.listRecords(spaceUri, collection, { + byUser: c.req.query("byUser") ?? undefined, + cursor: c.req.query("cursor") ?? undefined, + limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, + }); + return c.json(list); + }); + + app.get(`/xrpc/${SPACE}.getRecord`, readAuth, async (c) => { + const spaceUri = c.req.query("spaceUri"); + const collection = c.req.query("collection"); + const author = c.req.query("author"); + const rkey = c.req.query("rkey"); + if (!spaceUri || !collection || !author || !rkey) { + return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const authz = await authorizeRead(c, authority, spaceUri); + if (authz instanceof Response) return authz; + + if (authz.via === "jwt") { + const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, sa.issuer); + const result = checkAccess({ + op: "read", + space, + callerDid: sa.issuer, + member, + clientId: sa.clientId, + targetAuthorDid: author, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + + const record = await recordHost.getRecord(spaceUri, collection, author, rkey); + if (!record) return c.json({ error: "NotFound" }, 404); + return c.json({ record }); + }); + + app.post(`/xrpc/${SPACE}.putRecord`, authWithCredential, async (c) => { + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; collection?: string; rkey?: string; record?: Record<string, unknown> } + | null; + if (!body?.spaceUri || !body.collection || !body.record) { + return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); + } + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + + if (recordHostConfig.blobs) { + const cids = collectBlobCids(body.record); + for (const cid of cids) { + const meta = await recordHost.getBlobMeta(body.spaceUri, cid); + if (!meta) { + return c.json( + { + error: "InvalidRequest", + reason: "unknown-blob-ref", + message: `Record references blob ${cid} that has not been uploaded to this space.`, + }, + 400 + ); + } + } + } + + const rkey = body.rkey ?? nextTid(); + const now = Date.now(); + await recordHost.putRecord({ + spaceUri: body.spaceUri, + collection: body.collection, + authorDid: caller.callerDid, + rkey, + cid: null, + record: body.record, + createdAt: now, + }); + return c.json({ rkey, authorDid: caller.callerDid, createdAt: now }); + }); + + app.post(`/xrpc/${SPACE}.deleteRecord`, authWithCredential, async (c) => { + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; collection?: string; rkey?: string } + | null; + if (!body?.spaceUri || !body.collection || !body.rkey) { + return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); + } + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "delete", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + targetAuthorDid: caller.callerDid, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + + await recordHost.deleteRecord(body.spaceUri, body.collection, caller.callerDid, body.rkey); + return c.json({ ok: true }); + }); + + // Blobs (only registered when a blob adapter is configured) + if (recordHostConfig.blobs) { + const blobsCfg = recordHostConfig.blobs; + const blobAdapter = blobsCfg.adapter; + const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; + const accept = blobsCfg.accept; + + app.post(`/xrpc/${SPACE}.uploadBlob`, authWithCredential, async (c) => { + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } + } + + const mimeType = c.req.header("content-type") ?? "application/octet-stream"; + if (accept && !accept.includes(mimeType)) { + return c.json( + { error: "InvalidMimeType", message: `MIME type ${mimeType} is not accepted.` }, + 400 + ); + } + + const declaredLen = c.req.header("content-length"); + if (declaredLen && Number(declaredLen) > maxSize) { + return c.json( + { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, + 413 + ); + } + + const buf = await c.req.arrayBuffer(); + const bytes = new Uint8Array(buf); + if (bytes.byteLength > maxSize) { + return c.json( + { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, + 413 + ); + } + + const cid = await createCid(0x55, bytes); + const cidString = cidToString(cid); + const key = await blobKey(spaceUri, cidString); + + await blobAdapter.put(key, bytes, { mimeType, size: bytes.byteLength }); + await recordHost.putBlobMeta({ + spaceUri, + cid: cidString, + mimeType, + size: bytes.byteLength, + authorDid: caller.callerDid, + createdAt: Date.now(), + }); + + return c.json({ + blob: { + $type: "blob", + ref: { $link: cidString }, + mimeType, + size: bytes.byteLength, + }, + }); + }); + + app.get(`/xrpc/${SPACE}.getBlob`, readAuth, async (c) => { + const spaceUri = c.req.query("spaceUri"); + const cid = c.req.query("cid"); + if (!spaceUri || !cid) { + return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const authz = await authorizeRead(c, authority, spaceUri); + if (authz instanceof Response) return authz; + + if (authz.via === "jwt") { + const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, sa.issuer); + const aclResult = checkAccess({ + op: "read", + space, + callerDid: sa.issuer, + member, + clientId: sa.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } + } + + const meta = await recordHost.getBlobMeta(spaceUri, cid); + if (!meta) return c.json({ error: "NotFound" }, 404); + const key = await blobKey(spaceUri, cid); + const bytes = await blobAdapter.get(key); + if (!bytes) return c.json({ error: "NotFound" }, 404); + + return new Response(bytes as BodyInit, { + headers: { + "content-type": meta.mimeType, + "content-length": String(meta.size), + }, + }); + }); + + app.get(`/xrpc/${SPACE}.listBlobs`, authWithCredential, async (c) => { + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, spaceUri, "read"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "read", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } + } + + const result = await recordHost.listBlobMeta(spaceUri, { + byUser: c.req.query("byUser") ?? undefined, + cursor: c.req.query("cursor") ?? undefined, + limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, + }); + return c.json(result); + }); + } +} + +/** Authorize a read request — three valid paths: a verified space credential + * (set by the credential middleware), a read-grant invite token, or a + * service-auth JWT. */ +async function authorizeRead( + c: Context, + authority: SpaceAuthority, + spaceUri: string +): Promise< + | { via: "credential"; claims: CredentialClaims } + | { via: "token" } + | { via: "jwt"; sa: ServiceAuth } + | Response +> { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== spaceUri) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + return { via: "credential", claims: cred }; + } + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); + if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); + return { via: "token" }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (sa) return { via: "jwt", sa }; + return c.json( + { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, + 401 + ); +} + +function resolveCaller( + c: Context, + requestSpace: string, + requiredScope: CredentialScope +): { callerDid: string; clientId: string | undefined; viaCredential: boolean } | Response { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== requestSpace) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + if (requiredScope === "rw" && cred.scope !== "rw") { + return c.json({ error: "Forbidden", reason: "credential-wrong-scope" }, 403); + } + return { callerDid: cred.sub, clientId: undefined, viaCredential: true }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (!sa) return c.json({ error: "AuthRequired", reason: "no-auth" }, 401); + return { callerDid: sa.issuer, clientId: sa.clientId, viaCredential: false }; +} + +function getAuth(c: Context): ServiceAuth { + const auth = c.get("serviceAuth") as ServiceAuth | undefined; + if (!auth) throw new Error("service auth not set"); + return auth; +} diff --git a/packages/contrail-record-host/src/schema.ts b/packages/contrail-record-host/src/schema.ts new file mode 100644 index 0000000..881d16b --- /dev/null +++ b/packages/contrail-record-host/src/schema.ts @@ -0,0 +1,51 @@ +/** Record-host DDL: `spaces_blobs`, `record_host_enrollments`, plus the + * per-collection `spaces_records_<short>` tables. The latter are config- + * driven so they're built via the shared collection-table helper from + * contrail-base, not declared here directly. */ + +import type { Database, ContrailConfig, SqlDialect } from "@atmo-dev/contrail-base"; +import { getDialect } from "@atmo-dev/contrail-base"; + +/** The fixed host tables — independent of the user's collection config. */ +export function buildRecordHostBaseSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS spaces_blobs ( + space_uri TEXT NOT NULL, + cid TEXT NOT NULL, + mime_type TEXT NOT NULL, + size INTEGER NOT NULL, + author_did TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + PRIMARY KEY (space_uri, cid) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_author ON spaces_blobs(space_uri, author_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_created ON spaces_blobs(space_uri, created_at)`, + + // Local cache: spaces this host has agreed to store records for, and + // which authority signs credentials for each. Filled by the + // recordHost.enroll endpoint, or auto-populated by the colocated + // authority's createSpace. + `CREATE TABLE IF NOT EXISTS record_host_enrollments ( + space_uri TEXT PRIMARY KEY, + authority_did TEXT NOT NULL, + enrolled_at ${dialect.bigintType} NOT NULL, + enrolled_by TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, + ]; +} + +/** SchemaModule-shaped function suitable for `initSchema({ extraSchemas: [...] })`. + * Applies the host tables that don't depend on the collection config. The + * per-collection `spaces_records_<short>` tables are still applied through + * the appview's initSchema path (which knows about collections). */ +export async function applyRecordHostSchema(db: Database): Promise<void> { + const dialect = getDialect(db); + const stmts = buildRecordHostBaseSchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); +} + +// Re-export for callers that want a config-driven schema (per-collection +// tables plus blobs/enrollments). Kept as a convenience for split +// deployments where the host doesn't share a DB with the appview. +export type { ContrailConfig }; diff --git a/packages/contrail-record-host/tsconfig.build.json b/packages/contrail-record-host/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-record-host/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-record-host/tsconfig.json b/packages/contrail-record-host/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-record-host/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-record-host/tsup.config.ts b/packages/contrail-record-host/tsup.config.ts new file mode 100644 index 0000000..7d6b7eb --- /dev/null +++ b/packages/contrail-record-host/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["@atmo-dev/contrail-base"], +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 02a833f..4a3e79b 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,8 +73,10 @@ "@atcute/jetstream": "^1.0.2", "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-appview": "workspace:*", "@atmo-dev/contrail-authority": "workspace:*", "@atmo-dev/contrail-base": "workspace:*", + "@atmo-dev/contrail-record-host": "workspace:*", "cac": "^7.0.0", "hono": "^4.12.8", "jiti": "^2.4.0" diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index 713e23e..551c495 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -1,523 +1 @@ -import type {} from "@atcute/atproto"; -import { type Did } from "@atcute/lexicons"; -import { isDid, isNsid } from "@atcute/lexicons/syntax"; - -import type { Client } from "@atcute/client"; -import type { ContrailConfig, Database, IngestEvent } from "./types"; -import { getDiscoverableNsids, getDependentNsids, DEFAULT_RELAYS } from "./types"; -import { applyEvents, getLastCursor, saveCursor } from "./db"; -import { getClient, getPDS } from "./client"; - -const PAGE_SIZE = 100; -const BATCH_SIZE = 100; -const MAX_RETRIES = 5; - -const REQUEST_TIMEOUT_MS = 10_000; - -async function withRetry<T>( - fn: () => Promise<T>, - label: string, - maxRetries = 3, - timeoutMs = REQUEST_TIMEOUT_MS -): Promise<T> { - let lastError: unknown; - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - return await Promise.race([ - fn(), - new Promise<never>((_, reject) => - setTimeout(() => reject(new Error(`Timeout: ${label}`)), timeoutMs) - ), - ]); - } catch (err) { - lastError = err; - if (attempt < maxRetries) { - const delay = Math.min(1000 * 2 ** attempt, 10000); - await new Promise((r) => setTimeout(r, delay)); - } - } - } - throw lastError; -} - -async function markFailed( - db: Database, - did: string, - collection: string, - error: string -): Promise<void> { - await db - .prepare( - "UPDATE backfills SET retries = retries + 1, last_error = ? WHERE did = ? AND collection = ?" - ) - .bind(error, did, collection) - .run(); -} - -export interface BackfillOptions { - /** Pre-resolved client — avoids redundant PDS lookups when batching by DID */ - client?: Client; - /** Skip replay detection in applyEvents (safe during initial backfill) */ - skipReplayDetection?: boolean; - /** Max retries per request (default: 3). Set to 0 for single-attempt mode. */ - maxRetries?: number; - /** Per-request timeout in ms (default: 10000). */ - requestTimeout?: number; -} - -export async function backfillUser( - db: Database, - did: string, - collection: string, - deadline: number, - config?: ContrailConfig, - options?: BackfillOptions -): Promise<number> { - if (Date.now() >= deadline) return 0; - - const status = await db - .prepare( - "SELECT completed, pds_cursor, retries FROM backfills WHERE did = ? AND collection = ?" - ) - .bind(did, collection) - .first<{ completed: number; pds_cursor: string | null; retries: number }>(); - - if (status?.completed) return 0; - - if (!status) { - await db - .prepare( - "INSERT INTO backfills (did, collection, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" - ) - .bind(did, collection) - .run(); - } - - let currentCursor: string | undefined = status?.pds_cursor ?? undefined; - const retries = options?.maxRetries ?? 3; - const timeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; - - if (!isDid(did)) { - await markFailed(db, did, collection, `Invalid DID: ${did}`); - return 0; - } - - if (!isNsid(collection)) { - await markFailed(db, did, collection, `Invalid NSID: ${collection}`); - return 0; - } - - let client = options?.client; - if (!client) { - try { - client = await withRetry( - () => getClient(did as Did, db), - `getClient(${did})`, - Math.min(retries, 1), - timeout - ); - } catch (err) { - await markFailed(db, did, collection, String(err)); - return 0; - } - } - - let totalInserted = 0; - let done = false; - - try { - while (Date.now() < deadline) { - const response = await withRetry( - () => - client!.get("com.atproto.repo.listRecords", { - params: { - repo: did as Did, - collection, - limit: PAGE_SIZE, - cursor: currentCursor, - }, - }), - `listRecords(${did}/${collection})`, - retries, - timeout - ); - if (!response.ok) { - await markFailed( - db, - did, - collection, - `listRecords status ${response.status}` - ); - return totalInserted; - } - - if (response.data.records.length === 0) { - done = true; - break; - } - - const now = Date.now(); - const events: IngestEvent[] = response.data.records.map((r) => ({ - uri: r.uri, - did, - collection, - rkey: r.uri.split("/").pop()!, - operation: "create" as const, - cid: r.cid, - record: JSON.stringify(r.value), - time_us: now * 1000, - indexed_at: now * 1000, - })); - - await applyEvents(db, events, config, { - skipReplayDetection: options?.skipReplayDetection, - skipFeedFanout: true, - }); - totalInserted += events.length; - - currentCursor = response.data.cursor ?? undefined; - - await db - .prepare( - "UPDATE backfills SET pds_cursor = ? WHERE did = ? AND collection = ?" - ) - .bind(currentCursor ?? null, did, collection) - .run(); - - if (!currentCursor) { - done = true; - break; - } - } - } catch (err) { - await markFailed(db, did, collection, String(err)); - return totalInserted; - } - - if (done) { - await db - .prepare( - "UPDATE backfills SET completed = 1 WHERE did = ? AND collection = ?" - ) - .bind(did, collection) - .run(); - } - - return totalInserted; -} - -// --- Bulk backfill (groups by DID, resolves client once) --- - -export interface BackfillProgress { - records: number; - usersComplete: number; - usersTotal: number; - usersFailed: number; -} - -export interface BackfillAllOptions { - concurrency?: number; - onProgress?: (progress: BackfillProgress) => void; -} - -export async function backfillPending( - db: Database, - config: ContrailConfig, - options?: BackfillAllOptions -): Promise<number> { - const concurrency = options?.concurrency ?? 100; - let totalBackfilled = 0; - - // Anchor the jetstream cursor to now if it hasn't been set yet, so records - // emitted during backfill are replayed once jetstream starts. - if ((await getLastCursor(db)) === null) { - await saveCursor(db, Date.now() * 1000); - } - - // Reset retries so users that hit the cap in a prior run get another chance. - await db - .prepare("UPDATE backfills SET retries = 0 WHERE completed = 0") - .run(); - - while (true) { - const pending = await db - .prepare( - "SELECT did, collection FROM backfills WHERE completed = 0 AND retries < ? ORDER BY did" - ) - .bind(MAX_RETRIES) - .all<{ did: string; collection: string }>(); - - const rows = pending.results ?? []; - if (rows.length === 0) break; - - // Group by DID so we resolve PDS once per user - const byDid = new Map<string, string[]>(); - for (const row of rows) { - const cols = byDid.get(row.did) ?? []; - cols.push(row.collection); - byDid.set(row.did, cols); - } - - const dids = [...byDid.keys()]; - - // Resolve PDS endpoints in background (populates in-memory cache) - const resolvePromise = (async () => { - for (let i = 0; i < dids.length; i += 200) { - await Promise.allSettled( - dids.slice(i, i + 200).map((did) => - getPDS(did as Did, db).catch(() => {}) - ) - ); - } - })(); - - let roundBackfilled = 0; - let usersComplete = 0; - let usersFailed = 0; - const failedDids: string[] = []; - - const FAST_TIMEOUT = 3_000; - - const emitProgress = () => - options?.onProgress?.({ - records: totalBackfilled + roundBackfilled, - usersComplete, - usersTotal: dids.length, - usersFailed, - }); - - // Fast pass: single attempt per user with short timeout - for (let i = 0; i < dids.length; i += concurrency) { - const batch = dids.slice(i, i + concurrency); - - const results = await Promise.allSettled( - batch.map(async (did) => { - let client: Client | undefined; - try { - client = await withRetry( - () => getClient(did as Did, db), - `getClient(${did})`, - 0, - FAST_TIMEOUT - ); - } catch { - failedDids.push(did); - return 0; - } - - const cols = byDid.get(did)!; - const counts = await Promise.all( - cols.map((col) => - backfillUser(db, did, col, Infinity, config, { - client, - skipReplayDetection: true, - maxRetries: 0, - requestTimeout: FAST_TIMEOUT, - }).catch(() => { - failedDids.push(did); - return 0; - }) - ) - ); - - usersComplete++; - return counts.reduce((a, b) => a + b, 0); - }) - ); - - for (const r of results) { - if (r.status === "fulfilled") roundBackfilled += r.value; - } - - emitProgress(); - } - - // Retry pass: failed DIDs get retries with backoff, still in concurrent batches - if (failedDids.length > 0) { - const uniqueFailed = [...new Set(failedDids)]; - usersComplete -= uniqueFailed.length; // don't count them yet - - for (let i = 0; i < uniqueFailed.length; i += concurrency) { - const batch = uniqueFailed.slice(i, i + concurrency); - - const results = await Promise.allSettled( - batch.map(async (did) => { - let client: Client | undefined; - try { - client = await withRetry( - () => getClient(did as Did, db), - `getClient(${did})`, - 2 - ); - } catch (err) { - for (const col of byDid.get(did)!) { - await markFailed(db, did, col, String(err)); - } - usersFailed++; - usersComplete++; - return 0; - } - - const cols = byDid.get(did)!; - const counts = await Promise.all( - cols.map((col) => - backfillUser(db, did, col, Infinity, config, { - client, - skipReplayDetection: true, - maxRetries: 2, - }) - ) - ); - usersComplete++; - return counts.reduce((a, b) => a + b, 0); - }) - ); - - for (const r of results) { - if (r.status === "fulfilled") roundBackfilled += r.value; - } - - emitProgress(); - } - } - - await resolvePromise; - totalBackfilled += roundBackfilled; - - // If nothing was backfilled this round, we're stuck - if (roundBackfilled === 0) break; - } - - return totalBackfilled; -} - -// --- Discovery --- - -interface DiscoveryPage { - repos: { did: string }[]; - cursor?: string; -} - -async function fetchPage( - relay: string, - collection: string, - cursor?: string -): Promise<DiscoveryPage | null> { - const url = new URL( - `/xrpc/com.atproto.sync.listReposByCollection`, - relay - ); - url.searchParams.set("collection", collection); - url.searchParams.set("limit", "1000"); - if (cursor) { - url.searchParams.set("cursor", cursor); - } - - try { - return await withRetry( - async () => { - const response = await fetch(url.toString()); - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); - } - return (await response.json()) as DiscoveryPage; - }, - `fetchPage(${relay}, ${collection})` - ); - } catch (err) { - // Discovery page fetch failed after retries — skip this relay - return null; - } -} - -async function insertDiscoveredDIDs( - db: Database, - dids: string[], - collection: string -): Promise<void> { - if (dids.length === 0) return; - - // Use multi-row INSERT to reduce the number of statements - const CHUNK_SIZE = 50; - for (let i = 0; i < dids.length; i += CHUNK_SIZE) { - const chunk = dids.slice(i, i + CHUNK_SIZE); - const placeholders = chunk.map(() => "(?, ?, 0)").join(", "); - const bindings: string[] = []; - for (const did of chunk) { - bindings.push(did, collection); - } - await db - .prepare( - `INSERT INTO backfills (did, collection, completed) VALUES ${placeholders} ON CONFLICT DO NOTHING` - ) - .bind(...bindings) - .run(); - } -} - -async function saveDiscoveryState( - db: Database, - collection: string, - relay: string, - cursor: string | null, - completed: boolean -): Promise<void> { - await db - .prepare( - "INSERT INTO discovery (collection, relay, cursor, completed) VALUES (?, ?, ?, ?) ON CONFLICT(collection, relay) DO UPDATE SET cursor = excluded.cursor, completed = excluded.completed" - ) - .bind(collection, relay, cursor, completed ? 1 : 0) - .run(); -} - -export async function discoverDIDs( - db: Database, - config: ContrailConfig, - deadline: number -): Promise<string[]> { - const collections = getDiscoverableNsids(config); - const relays = config.relays ?? DEFAULT_RELAYS; - if (relays.length === 0 || collections.length === 0) return []; - - const discovered: string[] = []; - - for (const collection of collections) { - if (Date.now() >= deadline) break; - - let data: DiscoveryPage | null = null; - let relay: string | null = null; - - for (const r of relays) { - const row = await db - .prepare( - "SELECT cursor, completed FROM discovery WHERE collection = ? AND relay = ?" - ) - .bind(collection, r) - .first<{ cursor: string | null; completed: number }>(); - - if (row?.completed) continue; - - data = await fetchPage(r, collection, row?.cursor ?? undefined); - if (data) { - relay = r; - break; - } else { - await saveDiscoveryState(db, collection, r, null, true); - } - } - if (!data || !relay) continue; - - const dids = data.repos?.map((r) => r.did) ?? []; - await insertDiscoveredDIDs(db, dids, collection); - discovered.push(...dids); - - for (const depCollection of getDependentNsids(config)) { - await insertDiscoveredDIDs(db, dids, depCollection); - } - - const completed = !data.cursor; - await saveDiscoveryState(db, collection, relay, data.cursor ?? null, completed); - } - - return discovered; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/client.ts b/packages/contrail/src/core/client.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/client.ts +++ b/packages/contrail/src/core/client.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/community-integration.ts b/packages/contrail/src/core/community-integration.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/community-integration.ts +++ b/packages/contrail/src/core/community-integration.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/db/index.ts b/packages/contrail/src/core/db/index.ts index 2469f31..551c495 100644 --- a/packages/contrail/src/core/db/index.ts +++ b/packages/contrail/src/core/db/index.ts @@ -1,4 +1 @@ -export { initSchema } from "./schema"; -export { getLastCursor, saveCursor, applyEvents, lookupExistingRecords, queryRecords, queryAcrossSources, pruneFeedItems } from "./records"; -export type { QueryOptions, SortOption, ExistingRecordInfo } from "./records"; -export type { RecordSource } from "../types"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index a45ab9b..551c495 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -1,896 +1 @@ -import type { - ContrailConfig, - ResolvedContrailConfig, - RelationConfig, - Database, - Statement, - IngestEvent, - RecordRow, - RecordSource, -} from "../types"; -import { - getNestedValue, - getRelationField, - countColumnName, - groupedCountColumnName, - getFeedFollowShortNames, - recordsTableName, - spacesRecordsTableName, - shortNameForNsid, - nsidForShortName, -} from "../types"; -import { getSearchableFields, ftsTableName, buildFtsContent } from "../search"; -import { ftsQueryClause, getDialect } from "../dialect"; - -// --- Counts --- - -interface InboundRelation { - /** Short name of the parent collection. */ - parentCollection: string; - relationName: string; - rel: RelationConfig; -} - -/** Find relations that target the given short-named child collection. */ -function getInboundRelations( - config: ContrailConfig, - childShortName: string -): InboundRelation[] { - const results: InboundRelation[] = []; - for (const [colName, colConfig] of Object.entries(config.collections)) { - for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { - if (rel.collection === childShortName) { - results.push({ parentCollection: colName, relationName: relName, rel }); - } - } - } - return results; -} - -/** - * Collect recount targets from a single event into a shared map. - * The map is keyed by `parentCollection:relationName:targetValue` to deduplicate - * across the entire batch — so 50 RSVPs to the same event produce one recount, not 50. - */ -function collectCountTargets( - event: IngestEvent, - config: ContrailConfig, - existingRecordJson: string | null, - targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> -): void { - const childShort = shortNameForNsid(config, event.collection); - if (!childShort) return; - const inbound = getInboundRelations(config, childShort); - if (inbound.length === 0) return; - - const record = event.record ? JSON.parse(event.record) : null; - const existingRecord = existingRecordJson ? JSON.parse(existingRecordJson) : null; - - for (const { parentCollection, relationName, rel } of inbound) { - if (rel.count === false) continue; - - const field = getRelationField(rel); - - const values: string[] = []; - if (record) { - const t = getNestedValue(record, field); - if (t) values.push(t); - } - if (existingRecord) { - const t = getNestedValue(existingRecord, field); - if (t && !values.includes(t)) values.push(t); - } - - for (const targetValue of values) { - const key = `${parentCollection}:${relationName}:${targetValue}`; - if (!targets.has(key)) { - targets.set(key, { parentCollection, relationName, rel, targetValue }); - } - } - } -} - -/** - * Build deduplicated count UPDATE statements from collected targets. - * One UPDATE per unique parent+relation+target, regardless of how many - * events in the batch affected that target. - */ -function buildBatchCountStatements( - db: Database, - config: ContrailConfig, - targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> -): Statement[] { - const statements: Statement[] = []; - - for (const { parentCollection, relationName, rel, targetValue } of targets.values()) { - const field = getRelationField(rel); - const matchColumn = rel.match === "did" ? "did" : "uri"; - const childTable = recordsTableName(rel.collection); - const parentTable = recordsTableName(parentCollection); - - const setClauses: string[] = []; - const setBindings: (string | number)[] = []; - - const countExpr = rel.countDistinct - ? `COUNT(DISTINCT ${rel.countDistinct})` - : "COUNT(*)"; - - // Total count - const totalCol = countColumnName(rel.collection); - setClauses.push( - `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ?)` - ); - setBindings.push(targetValue); - - // Grouped counts — column names are `count_<child-short>_<group-key>`; match - // against the group's full token value in the record. - if (rel.groupBy) { - const mapping = (config as ResolvedContrailConfig)._resolved?.relations[parentCollection]?.[relationName]; - if (mapping?.groups) { - for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - setClauses.push( - `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ? AND ${getDialect(db).jsonExtract('record', rel.groupBy)} = ?)` - ); - setBindings.push(targetValue, fullToken); - } - } - } - - if (setClauses.length > 0) { - statements.push( - db - .prepare( - `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE ${matchColumn} = ?` - ) - .bind(...setBindings, targetValue) - ); - } - } - - return statements; -} - -// --- FTS --- - -function buildFtsStatements( - db: Database, - event: IngestEvent, - config: ContrailConfig, - existingMap: Map<string, ExistingRecordInfo> -): Statement[] { - // PostgreSQL: tsvector generated column is auto-maintained, no manual FTS sync - if (getDialect(db).ftsStrategy === "generated-column") return []; - - const short = shortNameForNsid(config, event.collection); - if (!short) return []; - const colConfig = config.collections[short]; - if (!colConfig) return []; - - const fields = getSearchableFields(short, colConfig); - if (!fields || fields.length === 0) return []; - - const table = ftsTableName(short); - const stmts: Statement[] = []; - - if (event.operation === "delete") { - stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); - } else { - const record = event.record ? JSON.parse(event.record) : null; - if (!record) return []; - - const content = buildFtsContent(record, fields); - if (!content) return []; - - // Only delete existing FTS row if this is an update (record already existed) - if (existingMap.has(event.uri)) { - stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); - } - stmts.push( - db.prepare(`INSERT INTO ${table} (uri, content) VALUES (?, ?)`).bind(event.uri, content) - ); - } - - return stmts; -} - -// --- Feeds --- - -function buildFeedStatements( - db: Database, - event: IngestEvent, - config: ContrailConfig, - existingRecords: Map<string, string | null> -): Statement[] { - if (!config.feeds) return []; - - const stmts: Statement[] = []; - - const eventShort = shortNameForNsid(config, event.collection); - if (!eventShort) return []; - - for (const [, feedConfig] of Object.entries(config.feeds)) { - const followTable = recordsTableName(feedConfig.follow); - - // Target collection: fan out to followers - if (feedConfig.targets.includes(eventShort)) { - if (event.operation === "create" || event.operation === "update") { - stmts.push( - db - .prepare( - getDialect(db).insertOrIgnore( - `INSERT INTO feed_items (actor, uri, collection, time_us) - SELECT r.did, ?, ?, ? - FROM ${followTable} r - WHERE ${getDialect(db).jsonExtract('r.record', 'subject')} = ?` - ) - ) - .bind(event.uri, event.collection, event.time_us, event.did) - ); - } else if (event.operation === "delete") { - stmts.push( - db.prepare("DELETE FROM feed_items WHERE uri = ?").bind(event.uri) - ); - } - } - - // Follow collection: handle follow/unfollow - if (eventShort === feedConfig.follow) { - if (event.operation === "create") { - const record = event.record ? JSON.parse(event.record) : null; - const subject = record?.subject; - if (subject) { - for (const targetShort of feedConfig.targets) { - const targetTable = recordsTableName(targetShort); - const targetNsid = nsidForShortName(config, targetShort) ?? targetShort; - stmts.push( - db - .prepare( - getDialect(db).insertOrIgnore( - `INSERT INTO feed_items (actor, uri, collection, time_us) - SELECT ?, r.uri, ?, r.time_us - FROM ${targetTable} r - WHERE r.did = ? - ORDER BY r.time_us DESC - LIMIT 100` - ) - ) - .bind(event.did, targetNsid, subject) - ); - } - } - } else if (event.operation === "delete") { - const existingRecord = existingRecords.get(event.uri); - if (existingRecord) { - const parsed = JSON.parse(existingRecord); - const subject = parsed?.subject; - if (subject) { - for (const targetShort of feedConfig.targets) { - const targetTable = recordsTableName(targetShort); - stmts.push( - db - .prepare( - `DELETE FROM feed_items WHERE actor = ? AND uri IN ( - SELECT uri FROM ${targetTable} WHERE did = ? - )` - ) - .bind(event.did, subject) - ); - } - } - } - } - } - } - - return stmts; -} - -// --- Feed pruning --- - -export async function pruneFeedItems( - db: Database, - maxItems: number -): Promise<number> { - const result = await db - .prepare( - `DELETE FROM feed_items WHERE (actor, uri) NOT IN ( - SELECT actor, uri FROM ( - SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn - FROM feed_items - ) sub WHERE rn <= ? - )` - ) - .bind(maxItems) - .run(); - return (result as any)?.changes ?? 0; -} - -// --- Cursor --- - -export async function getLastCursor(db: Database): Promise<number | null> { - const row = await db - .prepare("SELECT time_us FROM cursor WHERE id = 1") - .first<{ time_us: number }>(); - return row ? row.time_us : null; -} - -export async function saveCursor( - db: Database, - timeUs: number -): Promise<void> { - await db - .prepare( - "INSERT INTO cursor (id, time_us) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET time_us = excluded.time_us" - ) - .bind(timeUs) - .run(); -} - -// --- Existing record lookup --- - -export interface ExistingRecordInfo { - cid: string | null; - record: string | null; - /** When the row was last written to our DB (microseconds). Populated - * whenever `lookupExistingRecords` runs, regardless of `includeRecord`. */ - indexed_at: number | null; -} - -/** - * Look up existing records for a set of events, grouped by collection. - * Returns a map of uri → { cid, record }. - * When includeRecord is false, record will always be null (saves reading large blobs). - */ -export async function lookupExistingRecords( - db: Database, - events: { uri: string; collection: string }[], - includeRecord: boolean = true, - config?: ContrailConfig -): Promise<Map<string, ExistingRecordInfo>> { - const result = new Map<string, ExistingRecordInfo>(); - if (events.length === 0) return result; - - // Group by short name (config lookup); skip events for collections not in our config. - const byShort = new Map<string, string[]>(); - for (const e of events) { - const short = config ? shortNameForNsid(config, e.collection) : e.collection; - if (!short) continue; - const uris = byShort.get(short) ?? []; - uris.push(e.uri); - byShort.set(short, uris); - } - - const selectCols = includeRecord ? "uri, cid, record, indexed_at" : "uri, cid, indexed_at"; - for (const [short, uris] of byShort) { - const table = recordsTableName(short); - for (let i = 0; i < uris.length; i += 50) { - const chunk = uris.slice(i, i + 50); - const placeholders = chunk.map(() => "?").join(","); - const rows = await db - .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri IN (${placeholders})`) - .bind(...chunk) - .all<{ - uri: string; - cid: string | null; - record?: string | null; - indexed_at: number | null; - }>(); - for (const row of rows.results ?? []) { - result.set(row.uri, { - cid: row.cid, - record: includeRecord ? (row.record ?? null) : null, - indexed_at: row.indexed_at ?? null, - }); - } - } - } - - return result; -} - -// --- Events --- - -export async function applyEvents( - db: Database, - events: IngestEvent[], - config?: ContrailConfig, - options?: { - skipReplayDetection?: boolean; - skipFeedFanout?: boolean; - /** Pre-fetched existing records — skips the internal lookup when provided */ - existing?: Map<string, ExistingRecordInfo>; - /** When provided, publish `collection:<nsid>` and `actor:<did>` realtime - * events for each applied event. Space-scoped publishing happens elsewhere - * (see `realtime/publishing-adapter.ts`); public topics carry public - * records only, which is exactly the scope of this function. */ - pubsub?: import("../realtime/types").PubSub; - } -): Promise<void> { - if (events.length === 0) return; - - const followCollections = config ? getFeedFollowShortNames(config) : []; - const hasCountingRelations = config ? Object.values(config.collections).some(c => - Object.values(c.relations ?? {}).some(r => r.count !== false) - ) : false; - const needRecordContent = followCollections.length > 0 || hasCountingRelations; - - // Use pre-fetched data or look up existing records - let existingMap: Map<string, ExistingRecordInfo>; - if (options?.existing) { - existingMap = options.existing; - } else if (config && !options?.skipReplayDetection) { - existingMap = await lookupExistingRecords(db, events, needRecordContent, config); - } else { - existingMap = new Map(); - } - - const batch: Statement[] = []; - - // Build a record-content map for feed statements (needs string values) - const existingRecordStrings = new Map<string, string | null>(); - for (const [uri, info] of existingMap) { - existingRecordStrings.set(uri, info.record); - } - - // Collect all count recount targets across the batch, deduplicated - const countTargets = new Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }>(); - - for (const e of events) { - // Event's collection is an NSID. Look up the short name from config. - // If no config or not found, treat collection string as-is (for tests that pre-populate tables). - const short = config - ? shortNameForNsid(config, e.collection) ?? (config.collections[e.collection] ? e.collection : null) - : e.collection; - if (!short) { - (config?.logger ?? console).warn( - `[ingest] drop (unknown collection in applyEvents): ${e.operation} ${e.uri} collection=${e.collection}` - ); - continue; - } - const table = recordsTableName(short); - - if (e.operation === "delete") { - batch.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(e.uri)); - } else { - batch.push( - db.prepare( - `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, time_us = excluded.time_us, indexed_at = excluded.indexed_at` - ).bind( - e.uri, - e.did, - e.rkey, - e.cid, - e.record, - e.time_us, - e.indexed_at - ) - ); - } - - if (config) { - // Collect count targets (deduplicated across the whole batch) - const existingRecordJson = existingMap.get(e.uri)?.record ?? null; - collectCountTargets(e, config, existingRecordJson, countTargets); - - // Feed fanout still needs replay detection - const existingInfo = existingMap.get(e.uri); - const isReplay = - e.operation === "delete" - ? existingInfo === undefined - : existingInfo?.cid === e.cid; - - if (!isReplay && !options?.skipFeedFanout) { - batch.push(...buildFeedStatements(db, e, config, existingRecordStrings)); - } - batch.push(...buildFtsStatements(db, e, config, existingMap)); - } - } - - // Build deduplicated count statements — one UPDATE per unique target - if (config) { - batch.push(...buildBatchCountStatements(db, config, countTargets)); - } - - await db.batch(batch); - - // Publish realtime events for public records (collection: and actor:). - // Space records publish via the wrapping adapter; this path is public-only. - if (options?.pubsub) { - const pubsub = options.pubsub; - const ts = Date.now(); - for (const e of events) { - if (e.operation === "delete") { - const payload = { - uri: e.uri, - did: e.did, - collection: e.collection, - rkey: e.rkey, - }; - await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.deleted", payload, ts }); - await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.deleted", payload, ts }); - } else { - const record = e.record ? safeParseJson(e.record) : {}; - const payload = { - uri: e.uri, - did: e.did, - collection: e.collection, - rkey: e.rkey, - cid: e.cid, - record, - time_us: e.time_us, - }; - await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.created", payload, ts }); - await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.created", payload, ts }); - } - } - } -} - -function safeParseJson(s: string): Record<string, unknown> { - try { - const v = JSON.parse(s); - return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {}; - } catch { - return {}; - } -} - -// --- Count columns --- - -/** Count column descriptor. `type` is the identifier returned in API responses and - * accepted in countFilters — we keep the full record token for grouped counts so - * callers pass e.g. "community.lexicon.calendar.rsvp#going" and filter/hydrate by it. */ -function getCountColumns( - config: ContrailConfig, - shortName: string -): { type: string; column: string }[] { - const colConfig = config.collections[shortName]; - if (!colConfig?.relations) return []; - const columns: { type: string; column: string }[] = []; - const relMap = (config as ResolvedContrailConfig)._resolved?.relations[shortName] ?? {}; - - for (const [relName, rel] of Object.entries(colConfig.relations)) { - if (rel.count === false) continue; - // Total: identifier is the child's short name; column is `count_<child-short>`. - columns.push({ type: rel.collection, column: countColumnName(rel.collection) }); - const mapping = relMap[relName]; - if (mapping) { - for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { - // Grouped: identifier is the full record token (stable across deployments); - // column is `count_<child-short>_<group-key>`. - columns.push({ - type: fullToken, - column: groupedCountColumnName(rel.collection, groupKey), - }); - } - } - } - return columns; -} - -/** For a given "count type" (short name or full group token), return the DB column. */ -function countColumnForType( - config: ContrailConfig, - shortName: string, - type: string -): string | null { - for (const col of getCountColumns(config, shortName)) { - if (col.type === type) return col.column; - } - return null; -} - -// --- Query --- - -export interface SortOption { - recordField?: string; - countType?: string; - direction: "asc" | "desc"; -} - -/** Opaque keyset cursor. `t` is the tiebreaker (time_us of the last row), - * `v` is the sort-key value (string for record fields, number for counts), - * `k` identifies the sort so we can reject mismatched cursors. */ -interface CursorPayload { - t: number; - v?: string | number; - k: "time" | string; // "time" | `field:<name>` | `count:<type>` -} - -function sortKind(sort?: SortOption): "time" | string { - if (sort?.recordField) return `field:${sort.recordField}`; - if (sort?.countType) return `count:${sort.countType}`; - return "time"; -} - -function encodeCursor(payload: CursorPayload): string { - return Buffer.from(JSON.stringify(payload), "utf8").toString("base64url"); -} - -function decodeCursor(cursor: string): CursorPayload | null { - try { - const json = Buffer.from(cursor, "base64url").toString("utf8"); - const p = JSON.parse(json); - if (typeof p?.t !== "number" || typeof p?.k !== "string") return null; - return p as CursorPayload; - } catch { - return null; - } -} - -export interface QueryOptions { - collection: string; - did?: string; - limit?: number; - cursor?: string; - filters?: Record<string, string>; - rangeFilters?: Record<string, { min?: string; max?: string }>; - countFilters?: Record<string, number>; - sort?: SortOption; - search?: string; - source?: RecordSource; - /** When set, query the per-space table (`spaces_records_<short>`) instead of the - * public `records_<short>` table, scoped to rows where `space_uri = ?`. */ - spaceUri?: string; -} - -export async function queryRecords( - db: Database, - config: ContrailConfig, - options: QueryOptions -): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { - const { - collection: collectionInput, - did, - limit: rawLimit, - cursor, - filters = {}, - rangeFilters = {}, - countFilters = {}, - sort, - search, - source, - spaceUri, - } = options; - - // Accept either the short name (canonical) or the full NSID for convenience. - const collection = - config.collections[collectionInput] - ? collectionInput - : shortNameForNsid(config, collectionInput) ?? collectionInput; - - const table = spaceUri ? spacesRecordsTableName(collection) : recordsTableName(collection); - const limit = Math.min(Math.max(1, rawLimit ?? 50), 200); - const conditions: string[] = []; - const bindings: (string | number)[] = []; - - if (spaceUri) { - conditions.push("r.space_uri = ?"); - bindings.push(spaceUri); - } - - if (source?.conditions) conditions.push(...source.conditions); - if (source?.params) bindings.push(...source.params); - - const countCols = getCountColumns(config, collection); - - if (did) { - conditions.push("r.did = ?"); - bindings.push(did); - } - - // Opaque keyset cursor encoding { t, v?, k }. Silently ignored if it doesn't - // match the current sort — callers shouldn't mix sort params with stale cursors. - const expectedKind = sortKind(sort); - if (cursor) { - const payload = decodeCursor(cursor); - if (payload && payload.k === expectedKind) { - if (sort?.recordField) { - const sortExpr = getDialect(db).jsonExtract('r.record', sort.recordField); - const cmp = sort.direction === "desc" ? "<" : ">"; - conditions.push(`(${sortExpr} ${cmp} ? OR (${sortExpr} = ? AND r.time_us < ?))`); - const v = payload.v ?? ""; - bindings.push(v as string | number, v as string | number, payload.t); - } else if (sort?.countType) { - const sortCol = countColumnForType(config, collection, sort.countType); - if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); - const cmp = sort.direction === "desc" ? "<" : ">"; - conditions.push(`(r.${sortCol} ${cmp} ? OR (r.${sortCol} = ? AND r.time_us < ?))`); - const v = Number(payload.v ?? 0); - bindings.push(v, v, payload.t); - } else { - conditions.push("r.time_us < ?"); - bindings.push(payload.t); - } - } - } - - for (const [field, value] of Object.entries(filters)) { - conditions.push(`${getDialect(db).jsonExtract('r.record', field)} = ?`); - bindings.push(value); - } - - for (const [field, range] of Object.entries(rangeFilters)) { - if (range.min != null) { - conditions.push(`${getDialect(db).jsonExtract('r.record', field)} >= ?`); - bindings.push(range.min); - } - if (range.max != null) { - conditions.push(`${getDialect(db).jsonExtract('r.record', field)} <= ?`); - bindings.push(range.max); - } - } - - for (const [type, minCount] of Object.entries(countFilters)) { - const col = countColumnForType(config, collection, type); - if (!col) continue; // unknown count type — skip filter - conditions.push(`r.${col} >= ?`); - bindings.push(minCount); - } - - // FTS search. Not supported in space mode yet (would need composite keying - // because the same at-URI can appear in multiple spaces). - let ftsJoin = ""; - let ftsClause: ReturnType<typeof ftsQueryClause> | null = null; - if (search && !spaceUri) { - const colConfig2 = config.collections[collection]; - const fields = colConfig2 ? getSearchableFields(collection, colConfig2) : null; - if (fields && fields.length > 0) { - ftsClause = ftsQueryClause(getDialect(db), recordsTableName(collection)); - ftsJoin = ftsClause.join; - conditions.push(ftsClause.condition); - // SECURITY: `search` is user input bound as a parameter, not interpolated. - bindings.push(search); - } - } - - const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; - - const countSelect = countCols.length > 0 - ? ", " + countCols.map(({ column }) => `r.${column}`).join(", ") - : ""; - const select = `r.uri, r.did, r.rkey, r.cid, r.record, r.time_us, r.indexed_at${countSelect}`; - - const join = [source?.joins, ftsJoin].filter(Boolean).join(" "); - - let orderBy: string; - if (sort?.recordField) { - const dir = sort.direction === "desc" ? "DESC" : "ASC"; - orderBy = `${getDialect(db).jsonExtract('r.record', sort.recordField)} ${dir}, r.time_us DESC`; - } else if (sort?.countType) { - const dir = sort.direction === "desc" ? "DESC" : "ASC"; - const sortCol = countColumnForType(config, collection, sort.countType); - if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); - orderBy = `r.${sortCol} ${dir}, r.time_us DESC`; - } else if (ftsClause) { - orderBy = `${ftsClause.orderExpr}, r.time_us DESC`; - // PG ts_rank needs the search term bound again for ORDER BY - if (getDialect(db).ftsStrategy === "generated-column" && search) { - bindings.push(search); - } - } else { - orderBy = "r.time_us DESC"; - } - - bindings.push(limit); - - const query = `SELECT ${select} FROM ${table} r ${join} ${where} ORDER BY ${orderBy} LIMIT ?`; - - const result = await db - .prepare(query) - .bind(...bindings) - .all<any>(); - - const nsid = nsidForShortName(config, collection) ?? collection; - const records = (result.results ?? []).map((row: any) => { - const rec: RecordRow & { counts?: Record<string, number> } = { - uri: row.uri, - did: row.did, - collection: nsid, - rkey: row.rkey, - cid: row.cid, - record: row.record, - time_us: row.time_us, - indexed_at: row.indexed_at, - ...(spaceUri ? { space: spaceUri } : {}), - }; - if (countCols.length > 0) { - const counts: Record<string, number> = {}; - for (const { type, column } of countCols) { - const val = row[column]; - if (val != null && val !== 0) counts[type] = val; - } - if (Object.keys(counts).length > 0) rec.counts = counts; - } - return rec; - }); - - const nextCursor = - records.length === limit - ? buildCursor(records[records.length - 1], sort, expectedKind) - : undefined; - - return { records, cursor: nextCursor }; -} - -/** Build an opaque keyset cursor from the last row of a page. */ -function buildCursor( - row: RecordRow & { counts?: Record<string, number> }, - sort: SortOption | undefined, - kind: string -): string { - const t = Number(row.time_us); - if (sort?.recordField) { - const parsed = row.record ? JSON.parse(row.record) : null; - const v = parsed ? getNestedValue(parsed, sort.recordField) : undefined; - return encodeCursor({ t, v: v == null ? "" : String(v), k: kind }); - } - if (sort?.countType) { - const v = row.counts?.[sort.countType] ?? 0; - return encodeCursor({ t, v, k: kind }); - } - return encodeCursor({ t, k: kind }); -} - -/** Compare two rows according to the active sort order. Returns negative if - * `a` should come before `b`, positive otherwise. Matches the SQL ORDER BY. */ -function compareRows( - a: RecordRow & { counts?: Record<string, number> }, - b: RecordRow & { counts?: Record<string, number> }, - sort: SortOption | undefined -): number { - const timeCmp = Number(b.time_us) - Number(a.time_us); // time_us DESC - if (sort?.recordField) { - const ar = a.record ? JSON.parse(a.record) : null; - const br = b.record ? JSON.parse(b.record) : null; - const av = ar ? getNestedValue(ar, sort.recordField) : undefined; - const bv = br ? getNestedValue(br, sort.recordField) : undefined; - const dir = sort.direction === "desc" ? -1 : 1; - const cmp = (av === bv ? 0 : (av! < bv! ? -1 : 1)) * dir; - return cmp !== 0 ? cmp : timeCmp; - } - if (sort?.countType) { - const av = a.counts?.[sort.countType] ?? 0; - const bv = b.counts?.[sort.countType] ?? 0; - const dir = sort.direction === "desc" ? -1 : 1; - const cmp = (av === bv ? 0 : (av < bv ? -1 : 1)) * dir; - return cmp !== 0 ? cmp : timeCmp; - } - return timeCmp; -} - -/** Run a listRecords query across the public table and a set of per-space tables - * in parallel, then merge according to the active sort order. The cursor is a - * shared keyset cursor — every sub-query applies the same `WHERE` keyset, so - * pagination is consistent across sources. */ -export async function queryAcrossSources( - db: Database, - config: ContrailConfig, - options: QueryOptions, - spaceUris: string[] -): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { - if (spaceUris.length === 0) { - return queryRecords(db, config, options); - } - const limit = Math.min(Math.max(1, options.limit ?? 50), 200); - const perSourceLimit = limit; // each source fetches up to `limit`; we trim after merge - - const tasks: Promise<{ records: (RecordRow & { counts?: Record<string, number> })[] }>[] = [ - queryRecords(db, config, { ...options, limit: perSourceLimit }), - ]; - for (const spaceUri of spaceUris) { - tasks.push(queryRecords(db, config, { ...options, spaceUri, limit: perSourceLimit })); - } - const results = await Promise.all(tasks); - const merged = results.flatMap((r) => r.records); - merged.sort((a, b) => compareRows(a, b, options.sort)); - const trimmed = merged.slice(0, limit); - const kind = sortKind(options.sort); - const cursor = - trimmed.length === limit ? buildCursor(trimmed[trimmed.length - 1], options.sort, kind) : undefined; - return { records: trimmed, cursor }; -} - -// --- Users --- - +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 562a6ee..551c495 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -1,355 +1 @@ -import type { ContrailConfig, Database, ResolvedContrailConfig, ResolvedMaps } from "../types"; -import type { SqlDialect } from "../dialect"; -import { buildFtsSchema, getDialect } from "../dialect"; -import { - getRelationField, - countColumnName, - groupedCountColumnName, - recordsTableName, - spacesRecordsTableName, - resolveConfig, -} from "../types"; -import { getSearchableFields } from "../search"; -import { buildSpacesBaseSchema } from "../spaces/schema"; -import { buildLabelsSchema } from "../labels/schema"; - -function getResolved(config: ContrailConfig): ResolvedMaps { - return (config as ResolvedContrailConfig)._resolved ?? resolveConfig(config)._resolved; -} - -function buildBaseSchema(dialect: SqlDialect): string { - return ` -CREATE TABLE IF NOT EXISTS backfills ( - did TEXT NOT NULL, - collection TEXT NOT NULL, - completed INTEGER NOT NULL DEFAULT 0, - pds_cursor TEXT, - retries INTEGER NOT NULL DEFAULT 0, - last_error TEXT, - PRIMARY KEY (did, collection) -); -CREATE TABLE IF NOT EXISTS discovery ( - collection TEXT NOT NULL, - relay TEXT NOT NULL, - cursor TEXT, - completed INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (collection, relay) -); -CREATE TABLE IF NOT EXISTS cursor ( - id INTEGER PRIMARY KEY CHECK (id = 1), - time_us ${dialect.bigintType} NOT NULL -); -CREATE TABLE IF NOT EXISTS identities ( - did TEXT PRIMARY KEY, - handle TEXT, - pds TEXT, - resolved_at ${dialect.bigintType} NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_identities_handle ON identities(handle); -`; -} - -function sanitizeName(name: string): string { - return name.replace(/[^a-zA-Z0-9]/g, "_"); -} - -interface BuilderOpts { - /** Emit tables for the spaces variant (spaces_records_<short> with space_uri column). */ - forSpaces?: boolean; -} - -function tableFor(shortName: string, opts: BuilderOpts): string { - return opts.forSpaces ? spacesRecordsTableName(shortName) : recordsTableName(shortName); -} - -function namePrefix(opts: BuilderOpts): string { - return opts.forSpaces ? "sp_" : ""; -} - -export function buildCollectionTables( - config: ContrailConfig, - dialect: SqlDialect, - opts: BuilderOpts = {} -): string[] { - const stmts: string[] = []; - for (const [shortName, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const table = tableFor(shortName, opts); - const np = namePrefix(opts); - if (opts.forSpaces) { - stmts.push( - `CREATE TABLE IF NOT EXISTS ${table} ( - space_uri TEXT NOT NULL, - uri TEXT NOT NULL, - did TEXT NOT NULL, - rkey TEXT NOT NULL, - cid TEXT, - record ${dialect.recordColumnType}, - time_us ${dialect.bigintType} NOT NULL, - indexed_at ${dialect.bigintType} NOT NULL, - PRIMARY KEY (space_uri, did, rkey) - )` - ); - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_time ON ${table}(space_uri, time_us DESC)` - ); - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_did ON ${table}(space_uri, did)` - ); - } else { - stmts.push( - `CREATE TABLE IF NOT EXISTS ${table} ( - uri TEXT PRIMARY KEY, - did TEXT NOT NULL, - rkey TEXT NOT NULL, - cid TEXT, - record ${dialect.recordColumnType}, - time_us ${dialect.bigintType} NOT NULL, - indexed_at ${dialect.bigintType} NOT NULL - )` - ); - stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_did ON ${table}(did)`); - stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_time ON ${table}(time_us DESC)`); - } - } - return stmts; -} - -export function buildDynamicIndexes( - config: ContrailConfig, - dialect: SqlDialect, - opts: BuilderOpts = {} -): string[] { - const resolved = getResolved(config); - const indexes: string[] = []; - const np = namePrefix(opts); - for (const [collection, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const table = tableFor(collection, opts); - const queryable = resolved.queryable[collection] ?? colConfig.queryable ?? {}; - for (const field of Object.keys(queryable)) { - const idxName = `idx_${np}${sanitizeName(collection)}_${sanitizeName(field)}`; - indexes.push( - `CREATE INDEX IF NOT EXISTS ${idxName} ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', field))})` - ); - } - - for (const [, rel] of Object.entries(colConfig.relations ?? {})) { - const childShort = rel.collection; - const childConfig = config.collections[childShort]; - if (opts.forSpaces && childConfig?.allowInSpaces === false) continue; - const on = getRelationField(rel); - const childTable = tableFor(childShort, opts); - const idxName = `idx_${np}${sanitizeName(childShort)}_${sanitizeName(on)}`; - indexes.push( - `CREATE INDEX IF NOT EXISTS ${idxName} ON ${childTable}(${dialect.indexExpression(dialect.jsonExtract('record', on))})` - ); - } - } - return indexes; -} - -export function buildCountColumns(config: ContrailConfig, opts: BuilderOpts = {}): string[] { - const resolved = getResolved(config); - const stmts: string[] = []; - const addedColumns = new Map<string, Set<string>>(); - const np = namePrefix(opts); - - for (const [collection, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const table = tableFor(collection, opts); - const relMap = resolved.relations[collection] ?? {}; - - if (!addedColumns.has(table)) addedColumns.set(table, new Set()); - const tableColumns = addedColumns.get(table)!; - - for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { - if (rel.count === false) continue; - if (opts.forSpaces && config.collections[rel.collection]?.allowInSpaces === false) continue; - const totalCol = countColumnName(rel.collection); - if (!tableColumns.has(totalCol)) { - tableColumns.add(totalCol); - stmts.push( - `ALTER TABLE ${table} ADD COLUMN ${totalCol} INTEGER NOT NULL DEFAULT 0` - ); - } - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${totalCol} ON ${table}(${totalCol} DESC, time_us DESC)` - ); - - const mapping = relMap[relName]; - if (mapping) { - for (const groupKey of Object.keys(mapping.groups)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - if (!tableColumns.has(groupCol)) { - tableColumns.add(groupCol); - stmts.push( - `ALTER TABLE ${table} ADD COLUMN ${groupCol} INTEGER NOT NULL DEFAULT 0` - ); - } - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${groupCol} ON ${table}(${groupCol} DESC, time_us DESC)` - ); - } - } - } - } - return stmts; -} - -function buildFeedTables(config: ContrailConfig, dialect: SqlDialect): string[] { - if (!config.feeds || Object.keys(config.feeds).length === 0) return []; - const stmts = [ - `CREATE TABLE IF NOT EXISTS feed_items ( - actor TEXT NOT NULL, - uri TEXT NOT NULL, - collection TEXT NOT NULL, - time_us ${dialect.bigintType} NOT NULL, - PRIMARY KEY (actor, uri) - )`, - `CREATE INDEX IF NOT EXISTS idx_feed_actor_coll_time ON feed_items(actor, collection, time_us DESC)`, - `CREATE INDEX IF NOT EXISTS idx_feed_actor_time ON feed_items(actor, time_us DESC)`, - `CREATE TABLE IF NOT EXISTS feed_backfills ( - actor TEXT NOT NULL, - feed TEXT NOT NULL, - completed INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (actor, feed) - )`, - ]; - - const followCollections = new Set(Object.values(config.feeds).map((f) => f.follow)); - for (const col of followCollections) { - const table = recordsTableName(col); - const safe = sanitizeName(col); - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${safe}_subject ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', 'subject'))})` - ); - } - - return stmts; -} - -export function buildFtsTables( - config: ContrailConfig, - dialect: SqlDialect, - opts: BuilderOpts = {} -): string[] { - const stmts: string[] = []; - for (const [collection, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const fields = getSearchableFields(collection, colConfig); - if (!fields || fields.length === 0) continue; - const table = tableFor(collection, opts); - stmts.push(...buildFtsSchema(dialect, table, fields)); - } - return stmts; -} - -const MIGRATIONS = [ - "ALTER TABLE backfills ADD COLUMN retries INTEGER NOT NULL DEFAULT 0", - "ALTER TABLE backfills ADD COLUMN last_error TEXT", - "ALTER TABLE spaces_invites ADD COLUMN kind TEXT NOT NULL DEFAULT 'join'", -]; - -async function runMigrations(db: Database): Promise<void> { - for (const sql of MIGRATIONS) { - try { - await db.prepare(sql).run(); - } catch { - // Column already exists — ignore - } - } -} - -/** Pluggable schema applier — passed in by extension packages (community, - * third-party plugins) to install their own tables alongside contrail's. */ -export type SchemaModule = (db: Database) => Promise<void>; - -export interface InitSchemaOptions { - /** Separate DB for the spaces tables. Defaults to the main `db`. */ - spacesDb?: Database; - /** Extra schema modules to apply after contrail's own DDL. Used by the - * community package to install its tables — contrail core no longer - * imports community schema directly. */ - extraSchemas?: SchemaModule[]; -} - -async function applySpacesSchema( - target: Database, - config: ContrailConfig, - dialect: SqlDialect -): Promise<void> { - const base = buildSpacesBaseSchema(dialect); - const perCollection = buildCollectionTables(config, dialect, { forSpaces: true }); - const indexes = buildDynamicIndexes(config, dialect, { forSpaces: true }); - await target.batch([...base, ...perCollection, ...indexes].map((s) => target.prepare(s))); - - const ftsStmts = buildFtsTables(config, dialect, { forSpaces: true }); - for (const stmt of ftsStmts) { - try { await target.prepare(stmt).run(); } catch { /* FTS5 unavailable */ } - } - for (const stmt of buildCountColumns(config, { forSpaces: true })) { - try { await target.prepare(stmt).run(); } catch { /* already exists */ } - } -} - -export async function initSchema( - db: Database, - config: ContrailConfig, - options: InitSchemaOptions = {} -): Promise<void> { - const dialect = getDialect(db); - const baseStatements = buildBaseSchema(dialect).split(";") - .map((s) => s.trim()) - .filter((s) => s.length > 0); - const collectionStatements = buildCollectionTables(config, dialect); - const indexStatements = buildDynamicIndexes(config, dialect); - const ftsStatements = buildFtsTables(config, dialect); - const feedStatements = buildFeedTables(config, dialect); - - const spacesDb = options.spacesDb; - const spacesSharesMainDb = !spacesDb || spacesDb === db; - - const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...feedStatements]; - - await db.batch(all.map((s) => db.prepare(s))); - - if (config.spaces?.authority || config.spaces?.recordHost) { - await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); - } - - // Extension schemas (e.g. community) — applied to the spacesDb when one's - // configured separately, since they typically reference space_uri. The - // caller is responsible for routing the schema to the right db; we just - // hand it the spaces-or-main DB as a sensible default. - const extensionTarget = spacesSharesMainDb ? db : spacesDb!; - for (const apply of options.extraSchemas ?? []) { - await apply(extensionTarget); - } - - if (config.labels) { - // Labels tables live on the main DB — they're keyed by at-URI / DID and - // are read alongside public records during hydration. - const labelsStmts = buildLabelsSchema(dialect); - await db.batch(labelsStmts.map((s) => db.prepare(s))); - } - - // FTS5 may not be available (e.g. node:sqlite) — skip gracefully - for (const stmt of ftsStatements) { - try { - await db.prepare(stmt).run(); - } catch { - // FTS5 not supported in this environment - } - } - await runMigrations(db); - - // Add count columns (ALTER TABLE — may already exist) - for (const stmt of buildCountColumns(config)) { - try { - await db.prepare(stmt).run(); - } catch { - // Column/index already exists — ignore - } - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/dialect.ts b/packages/contrail/src/core/dialect.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/dialect.ts +++ b/packages/contrail/src/core/dialect.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/identity.ts b/packages/contrail/src/core/identity.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/identity.ts +++ b/packages/contrail/src/core/identity.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/community-handler.ts b/packages/contrail/src/core/invite/community-handler.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/invite/community-handler.ts +++ b/packages/contrail/src/core/invite/community-handler.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/index.ts b/packages/contrail/src/core/invite/index.ts index 8dae0df..551c495 100644 --- a/packages/contrail/src/core/invite/index.ts +++ b/packages/contrail/src/core/invite/index.ts @@ -1,3 +1 @@ -export { generateInviteToken, hashInviteToken, mintInviteToken } from "./token"; -export { registerInviteRoutes } from "./router"; -export type { InviteRoutesOptions } from "./router"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/router.ts b/packages/contrail/src/core/invite/router.ts index 941af9b..551c495 100644 --- a/packages/contrail/src/core/invite/router.ts +++ b/packages/contrail/src/core/invite/router.ts @@ -1,250 +1 @@ -/** Unified invite surface: a single `<ns>.invite.*` family serving both - * user-owned and community-owned spaces. Dispatches on space ownership. - * - * - User-owned space → `kind` in create, `addMember` on redeem, owner-only. - * - Community-owned → routed to a {@link CommunityInviteHandler} provided - * by the community module (or null when community is - * not configured). - * - * Storage stays separate (`spaces_invites` vs `community_invites` tables) — - * schemas differ enough that unifying them would be net-negative. The token - * primitive and HTTP dance are shared. invite/router has zero imports from - * community/ — coupling is via the {@link CommunityInviteHandler} interface. */ - -import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig } from "../types"; -import type { ServiceAuth } from "../spaces/auth"; -import type { SpaceAuthority } from "../spaces/types"; -import type { InviteKind, InviteRow } from "../spaces/types"; -import { hashInviteToken, mintInviteToken } from "./token"; -import type { CommunityInviteHandler, HandlerResponse } from "./community-handler"; - -export interface InviteRoutesOptions { - authMiddleware: MiddlewareHandler; -} - -interface PublicInviteView { - tokenHash: string; - spaceUri: string; - kind?: InviteKind; - createdBy: string; - createdAt: number; - expiresAt: number | null; - maxUses: number | null; - usedCount: number; - revokedAt: number | null; - note: string | null; -} - -function toSpacesView(row: InviteRow): PublicInviteView { - return { - tokenHash: row.tokenHash, - spaceUri: row.spaceUri, - kind: row.kind, - createdBy: row.createdBy, - createdAt: row.createdAt, - expiresAt: row.expiresAt, - maxUses: row.maxUses, - usedCount: row.usedCount, - revokedAt: row.revokedAt, - note: row.note, - }; -} - -export function registerInviteRoutes( - app: Hono, - config: ContrailConfig, - authority: SpaceAuthority, - community: CommunityInviteHandler | null, - options: InviteRoutesOptions -): void { - if (!config.spaces?.authority) return; - - const NS = `${config.namespace}.invite`; - const auth = options.authMiddleware; - - /** Resolve whether a space is community-owned. Returns null if the space - * doesn't exist. */ - const classifySpace = async (spaceUri: string) => { - const space = await authority.getSpace(spaceUri); - if (!space) return null; - const isCommunity = community ? await community.isCommunityOwned(spaceUri) : false; - return { space, isCommunity }; - }; - - app.post(`/xrpc/${NS}.create`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { - spaceUri?: string; - kind?: string; - accessLevel?: string; - expiresAt?: number; - maxUses?: number; - note?: string; - } - | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - if (body.kind && body.accessLevel) { - return c.json( - { error: "InvalidRequest", reason: "kind-or-accessLevel", message: "pass kind OR accessLevel, not both" }, - 400 - ); - } - - const classified = await classifySpace(body.spaceUri); - if (!classified) return c.json({ error: "NotFound" }, 404); - const { space, isCommunity } = classified; - - if (isCommunity) { - if (!community) return c.json({ error: "InvalidState" }, 500); - return relay(c, await community.create({ - spaceUri: body.spaceUri, - callerDid: sa.issuer, - accessLevel: body.accessLevel, - kind: body.kind, - expiresAt: body.expiresAt ?? null, - maxUses: body.maxUses ?? null, - note: body.note ?? null, - })); - } - - // User-owned space. - if (body.accessLevel) { - return c.json( - { error: "InvalidRequest", reason: "accessLevel-on-user-space", message: "user-owned spaces take kind, not accessLevel" }, - 400 - ); - } - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const kind = (body.kind ?? "join") as InviteKind; - if (kind !== "join" && kind !== "read" && kind !== "read-join") { - return c.json({ error: "InvalidRequest", message: "kind must be 'join', 'read', or 'read-join'" }, 400); - } - const { token, tokenHash } = await mintInviteToken(); - const invite = await authority.createInvite({ - spaceUri: body.spaceUri, - tokenHash, - kind, - expiresAt: body.expiresAt ?? null, - maxUses: body.maxUses ?? null, - createdBy: sa.issuer, - note: body.note ?? null, - }); - return c.json({ token, invite: toSpacesView(invite) }); - }); - - app.get(`/xrpc/${NS}.list`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const includeRevoked = c.req.query("includeRevoked") === "true"; - - const classified = await classifySpace(spaceUri); - if (!classified) return c.json({ error: "NotFound" }, 404); - const { space, isCommunity } = classified; - - if (isCommunity) { - return relay(c, await community!.list({ - spaceUri, - callerDid: sa.issuer, - includeRevoked, - })); - } - - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const rows = await authority.listInvites(spaceUri, { includeRevoked }); - return c.json({ invites: rows.map(toSpacesView) }); - }); - - app.post(`/xrpc/${NS}.revoke`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; tokenHash?: string } - | null; - if (!body?.tokenHash) { - return c.json({ error: "InvalidRequest", message: "tokenHash required" }, 400); - } - - if (body.spaceUri) { - const classified = await classifySpace(body.spaceUri); - if (!classified) return c.json({ error: "NotFound" }, 404); - if (classified.isCommunity) { - return relay(c, await community!.revoke({ - spaceUri: body.spaceUri, - tokenHash: body.tokenHash, - callerDid: sa.issuer, - })); - } - if (classified.space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const ok = await authority.revokeInvite(body.tokenHash); - return c.json({ ok }); - } - - // No spaceUri — try the community handler first (it returns null if the - // token isn't a community invite), then fall back to the user-owned path. - if (community) { - const r = await community.tryRevokeByToken({ - tokenHash: body.tokenHash, - callerDid: sa.issuer, - }); - if (r) return relay(c, r); - } - const srow = await authority.getInvite(body.tokenHash); - if (!srow) return c.json({ error: "NotFound" }, 404); - const space = await authority.getSpace(srow.spaceUri); - if (space && space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const ok = await authority.revokeInvite(body.tokenHash); - return c.json({ ok }); - }); - - app.post(`/xrpc/${NS}.redeem`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { token?: string } | null; - if (!body?.token) { - return c.json({ error: "InvalidRequest", message: "token required" }, 400); - } - const tokenHash = await hashInviteToken(body.token); - const now = Date.now(); - - // Try community first (atomic — null if not a community invite). - if (community) { - const r = await community.tryRedeem({ - tokenHash, - callerDid: sa.issuer, - now, - }); - if (r) return relay(c, r); - } - - // Fall back to the user-owned spaces path. The redeem filter at the SQL - // level already restricts to `kind IN ('join','read-join')`. - const sinvite = await authority.redeemInvite(tokenHash, now); - if (!sinvite) { - return c.json({ error: "InvalidInvite", reason: "expired-revoked-or-exhausted" }, 400); - } - await authority.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); - return c.json({ spaceUri: sinvite.spaceUri, kind: sinvite.kind }); - }); -} - -/** Forward a community-handler response to the wire. */ -function relay(c: Context, r: HandlerResponse) { - return c.json(r.body, r.status as Parameters<typeof c.json>[1]); -} - -function getAuth(c: Context): ServiceAuth { - const a = c.get("serviceAuth") as ServiceAuth | undefined; - if (!a) throw new Error("service auth not set"); - return a; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/token.ts b/packages/contrail/src/core/invite/token.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/invite/token.ts +++ b/packages/contrail/src/core/invite/token.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index d42a5e8..551c495 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -1,280 +1 @@ -import { JetstreamSubscription } from "@atcute/jetstream"; -import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; -import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS } from "./types"; -import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; -import { refreshStaleIdentities } from "./identity"; - -const BATCH_SIZE = 50; -const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour - -/** Mutable state that persists across ingest cycles within the same process. */ -export interface IngestState { - cachedKnownDids?: Set<string>; - schemaInitialized: boolean; - lastFeedPruneMs: number; -} - -export function createIngestState(): IngestState { - return { schemaInitialized: false, lastFeedPruneMs: 0 }; -} - -function getLogger(config: ContrailConfig): Logger { - return config.logger ?? console; -} - -export async function ingestEvents( - config: ContrailConfig, - cursor: number | null, - safetyTimeoutMs: number = 25_000, - knownDids?: Set<string> -): Promise<{ events: IngestEvent[]; lastCursor: number | null }> { - const log = getLogger(config); - const startTimeUs = Date.now() * 1000; - const deadline = Date.now() + safetyTimeoutMs; - const collected: IngestEvent[] = []; - - const collections = getCollectionNsids(config); - const dependentCollections = new Set(getDependentNsids(config)); - const urls = config.jetstreams ?? []; - - let totalCommits = 0; - let filteredUnknownDid = 0; - const filteredDidSamples = new Set<string>(); - let lastYieldedTimeUs: number | null = null; - let firstYieldedTimeUs: number | null = null; - let connectCount = 0; - const seenUris = new Map<string, number>(); // uri -> time_us of first occurrence - const duplicateUris: string[] = []; - - const subscription = new JetstreamSubscription({ - url: urls, - wantedCollections: collections, - ...(cursor !== null ? { cursor } : {}), - onConnectionOpen() { - connectCount++; - log.log( - `[ingest] connected to Jetstream #${connectCount} (url=${urls.join("|")}, cursor=${cursor ?? "none"}, wanted=${collections.join(",")})` - ); - }, - onConnectionClose(event) { - log.log( - `[ingest] disconnected from Jetstream: ${event.code} ${event.reason}` - ); - }, - onConnectionError(event) { - log.error("[ingest] Jetstream error:", event.error); - }, - }); - - for await (const event of subscription) { - if (firstYieldedTimeUs === null) firstYieldedTimeUs = event.time_us; - lastYieldedTimeUs = event.time_us; - if (event.kind === "commit") { - const { commit } = event; - totalCommits++; - - const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; - - if (dependentCollections.has(commit.collection) && knownDids) { - if (!knownDids.has(event.did)) { - filteredUnknownDid++; - if (filteredDidSamples.size < 10) filteredDidSamples.add(event.did); - continue; - } - } - - const prev = seenUris.get(uri); - if (prev !== undefined) { - duplicateUris.push(uri); - log.warn( - `[ingest] DUPLICATE in cycle: ${uri} first time_us=${prev}, again=${event.time_us}, delta=${event.time_us - prev}us` - ); - } else { - seenUris.set(uri, event.time_us); - } - - const now = Date.now(); - - collected.push({ - uri, - did: event.did, - time_us: event.time_us, - collection: commit.collection, - operation: commit.operation as "create" | "update" | "delete", - rkey: commit.rkey, - cid: commit.operation === "delete" ? null : commit.cid, - record: - commit.operation === "delete" - ? null - : JSON.stringify(commit.record), - indexed_at: now * 1000, - }); - - log.log( - `[ingest] keep: ${commit.operation} ${uri} time_us=${event.time_us}` - ); - - if (knownDids && !dependentCollections.has(commit.collection)) { - knownDids.add(event.did); - } - } - - if (event.time_us >= startTimeUs) { - log.log( - `[ingest] caught up to present, stopping (last time_us=${event.time_us}, startTimeUs=${startTimeUs})` - ); - break; - } - - if (Date.now() >= deadline) { - log.log( - `[ingest] safety timeout reached, stopping (deadline=${deadline}, collected=${collected.length})` - ); - break; - } - } - - if (filteredUnknownDid > 0) { - const sample = [...filteredDidSamples].join(", "); - log.log( - `[ingest] ${filteredUnknownDid} events filtered (unknown did). sample dids: ${sample}` - ); - } - const lastCursor = subscription.cursor || null; - - const cursorGap = - lastCursor !== null && lastYieldedTimeUs !== null - ? lastCursor - lastYieldedTimeUs - : null; - - // Detect the library's internal cursor rollback (picks a different URL → rolls - // back 10s → first event comes in BEFORE the cursor we asked it to start from). - const rolledBackUs = - cursor !== null && firstYieldedTimeUs !== null && firstYieldedTimeUs < cursor - ? cursor - firstYieldedTimeUs - : 0; - - log.log( - `[ingest] jetstream loop done. commits_seen=${totalCommits}, filtered=${filteredUnknownDid}, kept=${collected.length}, dupes=${duplicateUris.length}, connects=${connectCount}, first_yielded=${firstYieldedTimeUs ?? "none"}, last_yielded=${lastYieldedTimeUs ?? "none"}, subscription_cursor=${lastCursor ?? "none"}, cursor_gap=${cursorGap ?? "n/a"}us, rolled_back=${rolledBackUs}us` - ); - - if (cursorGap !== null && cursorGap > 1000) { - log.warn( - `[ingest] CURSOR GAP: subscription cursor is ${cursorGap}us (${Math.floor( - cursorGap / 1000 - )}ms) ahead of last yielded event — buffered events may be dropped` - ); - } - - if (connectCount > 1) { - log.warn( - `[ingest] RECONNECTED ${connectCount} times during cycle — each reconnect picks a URL at random and rolls cursor back 10s` - ); - } - - return { events: collected, lastCursor }; -} - -// Run a full ingest cycle: init schema, load cursor, ingest, apply, save cursor -export async function runIngestCycle( - db: Database, - config: ContrailConfig, - timeoutMs: number = 25_000, - state?: IngestState, - pubsub?: import("./realtime/types").PubSub -): Promise<void> { - const log = getLogger(config); - const s = state ?? createIngestState(); - - if (!s.schemaInitialized) { - await initSchema(db, config); - s.schemaInitialized = true; - } - - const cursor = await getLastCursor(db); - const collections = getCollectionNsids(config); - const nowUs = Date.now() * 1000; - const lagMs = cursor !== null ? Math.floor((nowUs - cursor) / 1000) : null; - - log.log( - `[ingest] starting cycle. cursor=${cursor ?? "none"}${ - lagMs !== null ? ` (lag=${lagMs}ms)` : "" - }, timeout=${timeoutMs}ms, collections=${collections.join(", ")}` - ); - - // Load known DIDs for filtering dependent collections - const dependentCollections = getDependentNsids(config); - let knownDids: Set<string> | undefined; - - if (dependentCollections.length > 0) { - if (s.cachedKnownDids) { - knownDids = s.cachedKnownDids; - log.log(`Using cached known DIDs (${knownDids.size} users)`); - } else { - const result = await db - .prepare("SELECT did FROM identities") - .all<{ did: string }>(); - knownDids = new Set((result.results ?? []).map((r) => r.did)); - s.cachedKnownDids = knownDids; - log.log(`Loaded ${knownDids.size} known DIDs from database`); - } - } - - const { events, lastCursor } = await ingestEvents( - config, - cursor, - timeoutMs, - knownDids - ); - - if (events.length > 0) { - const breakdown: Record<string, number> = {}; - for (const e of events) { - const key = `${e.collection}:${e.operation}`; - breakdown[key] = (breakdown[key] ?? 0) + 1; - } - log.log( - `[ingest] received ${events.length} events. breakdown=${JSON.stringify(breakdown)}` - ); - } else { - log.log(`[ingest] received 0 events from Jetstream`); - } - - for (let i = 0; i < events.length; i += BATCH_SIZE) { - const batch = events.slice(i, i + BATCH_SIZE); - await applyEvents(db, batch, config, { pubsub }); - } - - // Refresh stale/missing identities for DIDs in this batch - const uniqueDids = [...new Set(events.map((e) => e.did))]; - if (uniqueDids.length > 0) { - try { - await refreshStaleIdentities(db, uniqueDids); - } catch (err) { - log.warn(`Identity refresh failed: ${err}`); - } - } - - if (lastCursor !== null) { - await saveCursor(db, lastCursor); - log.log( - `[ingest] saved cursor=${lastCursor} (advanced ${ - cursor !== null ? lastCursor - cursor : "n/a" - }us)` - ); - } else { - log.log(`[ingest] no cursor returned from subscription; not saving`); - } - - // Prune feed items hourly - if (config.feeds && Date.now() - s.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { - const maxItems = Math.max( - ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) - ); - const pruned = await pruneFeedItems(db, maxItems); - if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); - s.lastFeedPruneMs = Date.now(); - } - - log.log(`[ingest] cycle complete. stored=${events.length}`); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/apply.ts b/packages/contrail/src/core/labels/apply.ts index cbf5558..551c495 100644 --- a/packages/contrail/src/core/labels/apply.ts +++ b/packages/contrail/src/core/labels/apply.ts @@ -1,64 +1 @@ -import type { Database, Statement } from "../types"; - -/** Wire shape of a single `com.atproto.label.defs#label` entry. Field names - * match the spec exactly. We accept the spec's ISO-8601 strings and - * convert to unix seconds at the storage boundary. */ -export interface IncomingLabel { - src: string; - uri: string; - val: string; - cid?: string; - neg?: boolean; - exp?: string; - cts: string; - sig?: Uint8Array; -} - -/** Upsert a batch of labels. Idempotent on `(src, uri, val, cts)`. Bad rows - * (missing required fields, unparseable timestamps) are dropped silently; - * we don't want one malformed label to abort an entire labeler frame. */ -export async function applyLabels( - db: Database, - labels: IncomingLabel[], -): Promise<number> { - if (labels.length === 0) return 0; - const stmts: Statement[] = []; - let kept = 0; - for (const l of labels) { - if (!l.src || !l.uri || !l.val || !l.cts) continue; - const cts = isoToUnixSec(l.cts); - if (cts == null) continue; - const exp = l.exp ? isoToUnixSec(l.exp) : null; - stmts.push( - db - .prepare( - `INSERT INTO labels (src, uri, val, cid, neg, exp, cts, sig) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(src, uri, val, cts) DO UPDATE SET - cid = excluded.cid, - neg = excluded.neg, - exp = excluded.exp, - sig = excluded.sig`, - ) - .bind( - l.src, - l.uri, - l.val, - l.cid ?? null, - l.neg ? 1 : 0, - exp, - cts, - l.sig ?? null, - ), - ); - kept++; - } - if (stmts.length > 0) await db.batch(stmts); - return kept; -} - -function isoToUnixSec(iso: string): number | null { - const ms = Date.parse(iso); - if (!Number.isFinite(ms)) return null; - return Math.floor(ms / 1000); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/hydrate.ts b/packages/contrail/src/core/labels/hydrate.ts index 02b947a44ef764f5c31f64fefd468727f76fd7d1..551c49528106738c1198722f3089b7f856050fe3 100644 GIT binary patch literal 44 zcmYeTD9A4=QP5IIE6UGRP;y8t$<5bINiEY)&d)0;O3cjBO)MxV%S<g-vgQH+V0;gI literal 3547 zcmd1IEyyn_Q7EY_NL8p-a7iplOiC<HRj5@+E6UGRP}0-W2T2txS#v>ERx2bWmLzAS zrg-KBmZla}A}cM*FD*$e($7fEDM&2>>EY7X(ozV|EK1ca&PXgsRY=P(Qb<(Ds7xtJ zEJ;mK$Vp5}%~5#NFhwC3Y-X`SLUMktUSdf>QGQ9j9#~p0B{i*B8O%)JQqWRRNXtyk zNzqNqFVY3G6pHf|lJoP5OLJ3;iWQO*^Arjai;ER9OB9MriV`!^GfEUnGK%s`(=!xG z@)e*)D+IfE7wai#>2sx4fc>ADSCU$kmYAHX-~n}<57=$hTnY-sMafnm!!q;It+^Bw zN{cd)xn+qt$lT=26nkVoC;<HPaw-)Hic*VH^GX!TGg9*uN{UKT!PYA%<fW$DTY&{Y zhJ%!Q2K(z;m>C%ADCFm6mVm+utS2vDA+@3)v#3%bvsj@xwFIg^wW0vSLYN5($wjG& zC7Jno3MHAjsl_FUxdl*N$tA@ISJrZIA^abdSPu4jP=2`rIlhM(t5BnmSDKRp@>E`G zx|Kp+X>L+#5r~^wQ2^s2DFV9)rl^(+oO0Y!OOi7nsiYX1c8XI=6!Oy)ic6ESQj<%H zbrjMvb4pT+KyjO#n3I!Qq+YC$n4Fwi017v7s!1&>)&VD+<oukR#Dd~fg#?Y_qGTOV zMCvG%CFW=*D3p{I<fIlWBvmTpB$lKWmnbA8mlP-HC?w{kD5MnS7r+#hXXF>Bf(?VJ z%Pdw%R7l86O}7O_X@WvgYDrOIGAPLP6oOJqN{jM93JOw-bfGR*$W1KJ<5JK98xjol z1vuFzr79$r=!OP)7AxczDI_Hpr7F01x)g)6M@dGiLUCelszOO(QckKuMt)98u|iUQ zNd`DNQz2<MvA8lXSs|@74{Sdymw_X*Sc6MJAtlKQp22lM=^AP~EXJZ^L87q0LlrGb zP0lY$an4L}sthg4w6{|5O)Rh}E-A{)OV@$s4QO22>2PUUDFhVd=VlhC+6001!BwJU zuIN}hJ8%-qOjFQ+22ZhGPHJ9yNrr;0t*wHALQM@kGEpQnK>=M_l&4T#Yt6-_prEg> z;F6kBT967)7^uPK;_0H0n^>ukSWu9fSfr4dr;w16nNpkpj^M;%1yJ}x0v}>nVrfZ6 zevvg;931WtNw68o`MIf((lIBqxI{r47SMWJ3gCjMM4>b<vk)G&whGaDdU|=O<qE;6 zC2((Q#)5JJsN8}mcZ4NbROPV904awXUyzfSoC-<;paRbpY#U6wUT$K6hK8nst(}6B zy^^M0R(@ulhLVnwCc?J7{PN(`WCdFV-^7v(y|kSC{2~pP#FA9Ky!>(vO$B`gLjwZ? zO<1TpW#*-1=A{=alw~HO7yOAusa9am6%?hWWmf1Y=;`StXO!k;$Lc8rl!A)gRE31p zih=}4m_jqER#IwOeo?9xIBXN*<30W2;}aB+`~)h{LGh6Uv05R&C?&NBRzX0F0!ar4 z`k?3nD?`LVQGR)`f~`Vgd17V>n!|07ia0w^#sFmvP|{aWNC<ZIadi#>75RwrK}R7u zGerkfQ0st7U>$|zl41q7Ab($I$;71q0pT95L9QUJ3Z8xn8Y<ODek!ijgsD<caP)Ig z&`7N)Q1A>^@C)_vQSc7}nX6!@V2`G%v?vp779^B36CjQRr6*7lgGIWI0whks*;fIa z9$={tqyZk60h!6!;82H_T}Aoj3I(Y}n8l%<LP%z6u|j?xq!?5vN-fUMDN9vIs)PoA zVs>U;I@l<X86_EsB?^f}MVV#bDjifBXXb%r3&9P9qSWHjoDxVumI6{$q??jjkeZj0 znpXlbFsZaep$JsqW#%aqfXmP1RBMIeg4AS%l>F4<JoOTVl+?7$yi|pf%v9Z^qExU0 zK#>h<oG5swRw~#+OCdx;hxpQ|5+V!AfS@wVJGD|r0VM%|^9HD4q5;W{ATQ;oDS#4? zCb)P~fM)A#ur&!P)kXQ`dc{S_wG0quX;CJOS(cbnn*eeGOiw{kW`2=^EwowzJ4P=( zwL~L3wGwQvf<k7Rf`%eQSxpVdRy|Piva?kHi-9<rD7uRwx^+NG!A8`A3P6a{^Gi#t z6bKZ#3bx=vQvqC`qIwxA#PrG%b4pW-H8f#C0CFbC-+8I&nxOC~$;>NFwFVWu&?+xA z1>_EJQS9vLqEL{TXQhw`$sr2h6q1*in+mE7LD8C-mYJ*osZ$g{suK$eav(`Ar&6IL zADmb8^&$GoQj3Z+^Yio+Jk!953R2C3TY0G|3W-o-bdxhvKy@p)L<f}*Mfv6G#V~_F zcBEt$C#DyrrWWf!s`ShfcpU+5C%}^$$T1*SfNg`?4l}4cRlO(`TKQz=rRym;CMV~Y z=9TE?q?V=TK)hS5paH6Hpf#GNLULkJQKdp&KFFsqa|$x^K(!Dev6rVR<b#@t$r-81 z+0aHJsM3ej(F)0+%D+6bBtsz)WDF?jLy`_CPC#B)Q-cNstd53794sY5LfbhrMZp$L zB{)HX%qcC(1g9uaaR!cWaODlsqG$`MXCQenMFH6^1&|a-b8==1Qo;cH6Pz+(_JhON zN&%$>07{EsmC!~u$Q)493@Qe0X@kT{i!wna8OUAW<~B&IEHMWr25vf*6o=${7Uye# z<dRE@H6fC<kjx17O@6r^*pXnDfS90=fC!?h0+r2*w&3O~DDNeu=IDWhY!#60NUbP< z29`#CX-PE5+R~!TSOt4~TZQOYO}&EB;tUP2RbXY{uqXnz2o>^6ORTwSxwzm>HJG`j wd6^Z#smZX`u_mky1}=u6nn48_sJ_;KDpt^f)Yy6@`JTc4!QdvDh9<~-0O=5WfdBvi diff --git a/packages/contrail/src/core/labels/resolve.ts b/packages/contrail/src/core/labels/resolve.ts index 694bfec..551c495 100644 --- a/packages/contrail/src/core/labels/resolve.ts +++ b/packages/contrail/src/core/labels/resolve.ts @@ -1,134 +1 @@ -import { - CompositeDidDocumentResolver, - PlcDidDocumentResolver, - WebDidDocumentResolver, -} from "@atcute/identity-resolver"; -import type { Did } from "@atcute/lexicons"; -import type { Database } from "../types"; - -/** Reject endpoint URLs that point to private/internal addresses or non-HTTPS. - * Mirrors the validator in core/client.ts — labeler endpoints should be - * publicly reachable for the same reasons PDS endpoints should. */ -function validateEndpointUrl(url: string): boolean { - try { - const parsed = new URL(url); - if (parsed.protocol !== "https:") return false; - const host = parsed.hostname; - if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; - if (host.startsWith("10.")) return false; - if (host.startsWith("192.168.")) return false; - if (host.startsWith("169.254.")) return false; - if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; - return true; - } catch { - return false; - } -} - -const didResolver = new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, -}); - -/** Look up the labeler service endpoint from a DID. - * Reads the DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ -export async function resolveLabelerEndpoint(did: string): Promise<string | null> { - if (!did.startsWith("did:plc:") && !did.startsWith("did:web:")) return null; - try { - const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); - const endpoint = doc.service - ?.find((s) => s.id === "#atproto_labeler") - ?.serviceEndpoint?.toString(); - if (!endpoint) return null; - if (!validateEndpointUrl(endpoint)) return null; - return endpoint; - } catch { - return null; - } -} - -/** State row for a labeler — the per-DID equivalent of the singleton - * jetstream `cursor` table, with cached endpoint to avoid repeated DID-doc - * fetches. */ -export interface LabelerState { - did: string; - cursor: number; - endpoint: string | null; - resolved_at: number | null; -} - -const ENDPOINT_TTL_MS = 6 * 60 * 60 * 1000; // 6h, matches the recommended client cache for label-defs - -/** Get cached `(endpoint, cursor)` for a labeler. Resolves endpoint on - * cache miss or staleness; persists endpoint + resolved_at back to the DB - * so subsequent ingest cycles avoid the network round-trip. */ -export async function getLabelerState( - db: Database, - did: string, - endpointOverride: string | undefined, -): Promise<LabelerState | null> { - const row = await db - .prepare( - "SELECT did, cursor, endpoint, resolved_at FROM labeler_cursors WHERE did = ?", - ) - .bind(did) - .first<LabelerState>(); - - let endpoint = endpointOverride ?? row?.endpoint ?? null; - const stale = - !row?.resolved_at || Date.now() - row.resolved_at > ENDPOINT_TTL_MS; - - if (!endpoint || (!endpointOverride && stale)) { - endpoint = await resolveLabelerEndpoint(did); - if (!endpoint) return null; - const now = Date.now(); - await db - .prepare( - `INSERT INTO labeler_cursors (did, cursor, endpoint, resolved_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(did) DO UPDATE SET endpoint = excluded.endpoint, resolved_at = excluded.resolved_at`, - ) - .bind(did, row?.cursor ?? 0, endpoint, now) - .run(); - return { - did, - cursor: row?.cursor ?? 0, - endpoint, - resolved_at: now, - }; - } - - return row ?? { did, cursor: 0, endpoint, resolved_at: null }; -} - -/** Persist the highest seen seq number for a labeler. Idempotent; - * the next ingest cycle resumes from `cursor + 1` via the `?cursor=` param. */ -export async function saveLabelerCursor( - db: Database, - did: string, - cursor: number, -): Promise<void> { - await db - .prepare( - `INSERT INTO labeler_cursors (did, cursor) - VALUES (?, ?) - ON CONFLICT(did) DO UPDATE SET cursor = excluded.cursor`, - ) - .bind(did, cursor) - .run(); -} - -/** Reset cursor to 0 — used in response to `#info { name: "OutdatedCursor" }` - * frames, which signal that the labeler's seq history was rewound. */ -export async function resetLabelerCursor(db: Database, did: string): Promise<void> { - await db - .prepare( - `INSERT INTO labeler_cursors (did, cursor) - VALUES (?, 0) - ON CONFLICT(did) DO UPDATE SET cursor = 0`, - ) - .bind(did) - .run(); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/schema.ts b/packages/contrail/src/core/labels/schema.ts index 486ce27..551c495 100644 --- a/packages/contrail/src/core/labels/schema.ts +++ b/packages/contrail/src/core/labels/schema.ts @@ -1,30 +1 @@ -import type { SqlDialect } from "../dialect"; - -/** DDL for the labels module. Single `labels` table covers record-level - * (uri starts with `at://`) and account-level (uri is a bare DID) entries — - * the spec collapses both into the same row shape. `labeler_cursors` - * mirrors the role of the singleton `cursor` table for jetstream, but - * per-labeler. */ -export function buildLabelsSchema(dialect: SqlDialect): string[] { - return [ - `CREATE TABLE IF NOT EXISTS labels ( - src TEXT NOT NULL, - uri TEXT NOT NULL, - val TEXT NOT NULL, - cid TEXT, - neg INTEGER NOT NULL DEFAULT 0, - exp ${dialect.bigintType}, - cts ${dialect.bigintType} NOT NULL, - sig BLOB, - PRIMARY KEY (src, uri, val, cts) - )`, - `CREATE INDEX IF NOT EXISTS idx_labels_uri ON labels(uri)`, - `CREATE INDEX IF NOT EXISTS idx_labels_src_cts ON labels(src, cts DESC)`, - `CREATE TABLE IF NOT EXISTS labeler_cursors ( - did TEXT PRIMARY KEY, - cursor ${dialect.bigintType} NOT NULL DEFAULT 0, - endpoint TEXT, - resolved_at ${dialect.bigintType} - )`, - ]; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/select.ts b/packages/contrail/src/core/labels/select.ts index da89d9c..551c495 100644 --- a/packages/contrail/src/core/labels/select.ts +++ b/packages/contrail/src/core/labels/select.ts @@ -1,64 +1 @@ -import type { LabelsConfig } from "./types"; -import { DEFAULT_LABELS_MAX_PER_REQUEST } from "./types"; - -/** Pick which labelers to honor for this request. - * - * Order of precedence: - * 1. `atproto-accept-labelers` header (atproto spec) - * 2. `?labelers=` query param (fallback for SSE/WS where headers are awkward) - * 3. `config.defaults` (operator policy) - * 4. every entry in `config.sources` - * - * Each candidate DID is checked against `config.sources`. Unknowns are - * dropped — we only have rows for labelers we've subscribed to. - * - * Header values can carry `;param` modifiers (e.g. `did:plc:...;redact`); - * v1 strips and ignores those — only the bare DID is honored. */ -export interface SelectedLabelers { - /** DIDs to use for hydration this request. */ - accepted: string[]; -} - -export function selectAcceptedLabelers( - headerValue: string | null | undefined, - paramValue: string | null | undefined, - cfg: LabelsConfig, -): SelectedLabelers { - const cap = cfg.maxPerRequest ?? DEFAULT_LABELS_MAX_PER_REQUEST; - const known = new Set(cfg.sources.map((s) => s.did)); - - const fromCaller = parseLabelerList(headerValue) ?? parseLabelerList(paramValue); - - let candidates: string[]; - if (fromCaller && fromCaller.length > 0) { - candidates = fromCaller; - } else { - candidates = (cfg.defaults ?? cfg.sources.map((s) => s.did)).slice(); - } - - const accepted: string[] = []; - const seen = new Set<string>(); - for (const did of candidates) { - if (seen.has(did)) continue; - seen.add(did); - if (known.has(did)) accepted.push(did); - if (accepted.length >= cap) break; - } - - return { accepted }; -} - -/** Parse a comma-separated DID list. Returns null when the input is empty - * or undefined so callers can distinguish "absent" from "empty list" (the - * latter — `atproto-accept-labelers: ` — is technically valid and means - * "no labelers"; we treat it the same as absent for ergonomics). */ -function parseLabelerList(value: string | null | undefined): string[] | null { - if (!value) return null; - const out: string[] = []; - for (const raw of value.split(",")) { - // Drop `;param` modifiers from the spec (e.g. `;redact`). v1 ignores them. - const head = raw.split(";")[0]!.trim(); - if (head.startsWith("did:")) out.push(head); - } - return out.length > 0 ? out : null; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/subscribe.ts b/packages/contrail/src/core/labels/subscribe.ts index c80ed2b..551c495 100644 --- a/packages/contrail/src/core/labels/subscribe.ts +++ b/packages/contrail/src/core/labels/subscribe.ts @@ -1,315 +1 @@ -import { decodeFirst } from "@atcute/cbor"; -import type { ContrailConfig, Database, Logger } from "../types"; -import type { LabelerSource } from "./types"; -import { applyLabels, type IncomingLabel } from "./apply"; -import { - getLabelerState, - resetLabelerCursor, - saveLabelerCursor, -} from "./resolve"; - -const DEFAULT_CYCLE_TIMEOUT_MS = 25_000; -const DEFAULT_BATCH_SIZE = 100; -const DEFAULT_FLUSH_INTERVAL_MS = 5_000; - -function getLogger(config: ContrailConfig): Logger { - return config.logger ?? console; -} - -/** One catch-up cycle for every configured labeler. Designed to fit inside a - * Cloudflare Workers cron tick — we drain frames until the labeler has no - * more buffered events for us, or `timeoutMs` is reached, then save cursor - * and disconnect. Mirrors the shape of `runIngestCycle` for jetstream. */ -export async function runLabelIngestCycle( - db: Database, - config: ContrailConfig, - timeoutMs = DEFAULT_CYCLE_TIMEOUT_MS, -): Promise<void> { - if (!config.labels) return; - const log = getLogger(config); - const deadline = Date.now() + timeoutMs; - - for (const source of config.labels.sources) { - if (Date.now() >= deadline) { - log.log(`[labels] cycle deadline hit before processing ${source.did}`); - break; - } - const remaining = Math.max(2_000, deadline - Date.now()); - try { - await pumpOneLabeler(db, source, log, remaining, /* persistent */ false); - } catch (err) { - log.warn(`[labels] cycle for ${source.did} failed: ${err}`); - } - } -} - -export interface PersistentLabelsOptions { - signal?: AbortSignal; - batchSize?: number; - flushIntervalMs?: number; - logger?: Logger; -} - -/** Long-lived equivalent — keeps one socket per labeler open forever, with - * exponential backoff reconnect. Mirrors `runPersistent` for jetstream. */ -export async function runPersistentLabels( - db: Database, - config: ContrailConfig, - options: PersistentLabelsOptions = {}, -): Promise<void> { - if (!config.labels) return; - const log = options.logger ?? config.logger ?? console; - const signal = options.signal; - - const tasks = config.labels.sources.map((source) => - runOneLabelerForever(db, source, log, signal, options), - ); - await Promise.all(tasks); -} - -async function runOneLabelerForever( - db: Database, - source: LabelerSource, - log: Logger, - signal: AbortSignal | undefined, - options: PersistentLabelsOptions, -): Promise<void> { - let attempts = 0; - while (!signal?.aborted) { - try { - await pumpOneLabeler(db, source, log, /* timeoutMs */ Infinity, true, { - signal, - batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE, - flushIntervalMs: options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, - }); - attempts = 0; - } catch (err) { - if (signal?.aborted) break; - log.error(`[labels] ${source.did} stream error: ${err}`); - const delay = Math.min(1_000 * 2 ** attempts, 30_000); - attempts++; - log.log(`[labels] ${source.did} reconnecting in ${delay}ms (attempt ${attempts})`); - await new Promise((r) => setTimeout(r, delay)); - } - } -} - -interface PumpOptions { - signal?: AbortSignal; - batchSize?: number; - flushIntervalMs?: number; -} - -/** Open a `subscribeLabels` WebSocket, drain frames into a buffer, flush - * the buffer to `labels` in batches, and persist the seq cursor. Returns - * when: - * - the labeler closes the socket cleanly (caught up + no more events) - * - `timeoutMs` is reached (cron mode) - * - `signal` is aborted (persistent mode) - * - an error tears the socket down (caller may retry) */ -async function pumpOneLabeler( - db: Database, - source: LabelerSource, - log: Logger, - timeoutMs: number, - persistent: boolean, - pumpOpts: PumpOptions = {}, -): Promise<void> { - const state = await getLabelerState(db, source.did, source.endpoint); - if (!state) { - log.warn(`[labels] could not resolve labeler endpoint for ${source.did}; skipping`); - return; - } - - // First-time policy: cursor 0 = "from the beginning" if backfill is on - // (default), null = "from now" otherwise. After the first cycle we always - // resume from the saved cursor — `backfill` only flips the start point. - const isFirstRun = state.cursor === 0 && state.resolved_at === null; - const backfill = source.backfill !== false; - const startCursor = isFirstRun && !backfill ? null : state.cursor; - - const url = buildWsUrl(state.endpoint!, startCursor); - log.log(`[labels] connecting to ${source.did} (cursor=${startCursor ?? "now"})`); - - const ws = new WebSocket(url); - ws.binaryType = "arraybuffer"; - - const buffer: IncomingLabel[] = []; - let highestSeq = state.cursor; - let flushing = false; - let resolveDone!: () => void; - let rejectDone!: (err: unknown) => void; - const done = new Promise<void>((res, rej) => { - resolveDone = res; - rejectDone = rej; - }); - - const flush = async () => { - if (buffer.length === 0 || flushing) return; - flushing = true; - const batch = buffer.splice(0); - try { - const kept = await applyLabels(db, batch); - if (highestSeq > state.cursor) { - await saveLabelerCursor(db, source.did, highestSeq); - state.cursor = highestSeq; - } - log.log( - `[labels] ${source.did} flushed ${kept}/${batch.length} labels, cursor=${highestSeq}`, - ); - } catch (err) { - log.error(`[labels] ${source.did} flush failed: ${err}`); - } finally { - flushing = false; - } - }; - - const batchSize = pumpOpts.batchSize ?? DEFAULT_BATCH_SIZE; - const flushInterval = pumpOpts.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS; - const flushTimer = setInterval(() => { - flush().catch(() => {}); - }, flushInterval); - - const cleanup = () => { - clearInterval(flushTimer); - try { - ws.close(); - } catch { - /* already closed */ - } - }; - - // External abort (persistent mode) — close socket gracefully. - const abortHandler = () => { - cleanup(); - flush().finally(() => resolveDone()); - }; - pumpOpts.signal?.addEventListener("abort", abortHandler, { once: true }); - - // Cron-mode time budget — close socket gracefully when reached. - let deadlineTimer: ReturnType<typeof setTimeout> | undefined; - if (Number.isFinite(timeoutMs)) { - deadlineTimer = setTimeout(() => { - log.log(`[labels] ${source.did} cycle deadline reached, closing`); - cleanup(); - flush().finally(() => resolveDone()); - }, timeoutMs); - } - - ws.addEventListener("error", (ev) => { - cleanup(); - if (deadlineTimer) clearTimeout(deadlineTimer); - pumpOpts.signal?.removeEventListener("abort", abortHandler); - rejectDone(new Error(`WebSocket error: ${(ev as ErrorEvent)?.message ?? "unknown"}`)); - }); - - ws.addEventListener("close", () => { - if (deadlineTimer) clearTimeout(deadlineTimer); - pumpOpts.signal?.removeEventListener("abort", abortHandler); - flush().finally(() => { - clearInterval(flushTimer); - resolveDone(); - }); - }); - - ws.addEventListener("message", async (ev) => { - let bytes: Uint8Array; - if (ev.data instanceof ArrayBuffer) { - bytes = new Uint8Array(ev.data); - } else if (ev.data instanceof Uint8Array) { - bytes = ev.data; - } else { - // Binary-only protocol — text frames shouldn't arrive. - return; - } - const frame = decodeFrame(bytes); - if (!frame) return; - - if (frame.t === "#labels") { - const seq = Number(frame.payload?.seq ?? 0); - const rawLabels = Array.isArray(frame.payload?.labels) ? frame.payload.labels : []; - for (const raw of rawLabels) { - const lab = normalizeLabel(raw, source.did); - if (lab) buffer.push(lab); - } - if (Number.isFinite(seq) && seq > highestSeq) highestSeq = seq; - if (buffer.length >= batchSize) { - flush().catch(() => {}); - } - } else if (frame.t === "#info") { - const name = String(frame.payload?.name ?? ""); - log.log(`[labels] ${source.did} info: ${name}`); - if (name === "OutdatedCursor") { - // Labeler rewound its log — discard our cursor and let the next - // run start from the beginning. We don't reconnect here; the - // caller (or the persistent loop) will pick up the reset on retry. - await resetLabelerCursor(db, source.did); - cleanup(); - } - } else if (frame.op === -1) { - log.warn(`[labels] ${source.did} error frame: ${JSON.stringify(frame.payload)}`); - cleanup(); - } - }); - - // Workers WebSocket doesn't always emit `open`; just await `done` directly. - await done; -} - -function buildWsUrl(httpEndpoint: string, cursor: number | null): string { - const u = new URL("/xrpc/com.atproto.label.subscribeLabels", httpEndpoint); - // wss:// for HTTPS endpoints — the protocol on the labeler service is - // expected to be HTTPS already (validated at resolution time). - u.protocol = u.protocol === "https:" ? "wss:" : "ws:"; - if (cursor !== null) u.searchParams.set("cursor", String(cursor)); - return u.toString(); -} - -interface DecodedFrame { - op: number; - t: string | undefined; - payload: Record<string, unknown>; -} - -/** Decode an atproto subscription frame: two consecutive CBOR objects. - * Header `{ op, t? }`, payload — shape depends on `t`. Returns null on - * decode failure or non-object frames. */ -function decodeFrame(bytes: Uint8Array): DecodedFrame | null { - try { - const [header, rest] = decodeFirst(bytes); - if (!header || typeof header !== "object") return null; - const op = typeof (header as { op?: number }).op === "number" ? (header as { op: number }).op : 1; - const t = typeof (header as { t?: string }).t === "string" ? (header as { t: string }).t : undefined; - const [payload] = decodeFirst(rest); - if (!payload || typeof payload !== "object") return null; - return { op, t, payload: payload as Record<string, unknown> }; - } catch { - return null; - } -} - -/** Coerce a wire `Label` object into our `IncomingLabel` shape. Returns - * null when required fields are missing — we'd rather skip a row than - * insert one with placeholder values. */ -function normalizeLabel(raw: unknown, expectedSrc: string): IncomingLabel | null { - if (!raw || typeof raw !== "object") return null; - const r = raw as Record<string, unknown>; - const src = typeof r.src === "string" ? r.src : null; - const uri = typeof r.uri === "string" ? r.uri : null; - const val = typeof r.val === "string" ? r.val : null; - const cts = typeof r.cts === "string" ? r.cts : null; - if (!src || !uri || !val || !cts) return null; - // A labeler shouldn't emit labels under a different `src` than its own - // DID — drop them rather than poison our table with cross-issuer rows. - if (src !== expectedSrc) return null; - return { - src, - uri, - val, - cts, - cid: typeof r.cid === "string" ? r.cid : undefined, - neg: r.neg === true, - exp: typeof r.exp === "string" ? r.exp : undefined, - sig: r.sig instanceof Uint8Array ? r.sig : undefined, - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/types.ts b/packages/contrail/src/core/labels/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/labels/types.ts +++ b/packages/contrail/src/core/labels/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index c1e1c18..551c495 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -1,243 +1 @@ -import type { JetstreamSubscription } from "@atcute/jetstream"; -import type { ContrailConfig, IngestEvent, Database, Logger, ResolvedContrailConfig } from "./types"; -import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS, resolveConfig } from "./types"; -import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; -import { refreshStaleIdentities } from "./identity"; -import { createIngestState } from "./jetstream"; -import type { IngestState } from "./jetstream"; - -const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; - -export interface PersistentIngestOptions { - batchSize?: number; - flushIntervalMs?: number; - signal?: AbortSignal; - /** Override subscription creation for testing */ - createSubscription?: (cursor: number | null) => JetstreamSubscription; - logger?: Logger; - /** Publish `collection:<nsid>` / `actor:<did>` events for each applied - * public record. Usually supplied by the Contrail instance. */ - pubsub?: import("./realtime/types").PubSub; -} - -function getLogger(config: ContrailConfig, options?: PersistentIngestOptions): Logger { - return options?.logger ?? config.logger ?? console; -} - -export async function runPersistent( - db: Database, - config: ContrailConfig, - options?: PersistentIngestOptions, -): Promise<void> { - // Internals (applyEvents, count updates, query planning) read `_resolved` - // and silently skip features when it's missing. The Contrail class resolves - // in its constructor; callers using this raw export must also get a resolved - // config, so do it defensively here. resolveConfig is idempotent. - if (!(config as ResolvedContrailConfig)._resolved) { - config = resolveConfig(config); - } - const log = getLogger(config, options); - const batchSize = options?.batchSize ?? 50; - const flushIntervalMs = options?.flushIntervalMs ?? 5_000; - const signal = options?.signal; - const state = createIngestState(); - - // Init schema once - if (!state.schemaInitialized) { - await initSchema(db, config); - state.schemaInitialized = true; - } - - // Load known DIDs for dependent collection filtering - const dependentCollections: Set<string> = new Set(getDependentNsids(config)); - let knownDids: Set<string> | undefined; - if (dependentCollections.size > 0) { - const result = await db - .prepare("SELECT did FROM identities") - .all<{ did: string }>(); - knownDids = new Set((result.results ?? []).map((r) => r.did)); - state.cachedKnownDids = knownDids; - log.log(`Loaded ${knownDids.size} known DIDs from database`); - } - - const collections = getCollectionNsids(config); - let reconnectAttempts = 0; - - while (!signal?.aborted) { - const cursor = await getLastCursor(db); - log.log(`Starting persistent ingestion. Cursor: ${cursor ?? "none"}, Collections: ${collections.join(", ")}`); - - try { - await streamAndFlush(db, config, cursor, { - batchSize, - flushIntervalMs, - signal, - collections, - dependentCollections, - knownDids, - state, - log, - createSubscription: options?.createSubscription, - pubsub: options?.pubsub, - }); - reconnectAttempts = 0; - } catch (err) { - if (signal?.aborted) break; - log.error(`Jetstream connection error: ${err}`); - const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30_000); - reconnectAttempts++; - log.log(`Reconnecting in ${delay}ms (attempt ${reconnectAttempts})...`); - await new Promise((r) => setTimeout(r, delay)); - } - } - - log.log("Persistent ingestion stopped"); -} - -interface StreamOptions { - batchSize: number; - flushIntervalMs: number; - signal?: AbortSignal; - collections: string[]; - dependentCollections: Set<string>; - knownDids?: Set<string>; - state: IngestState; - log: Logger; - createSubscription?: (cursor: number | null) => any; - pubsub?: import("./realtime/types").PubSub; -} - -async function streamAndFlush( - db: Database, - config: ContrailConfig, - cursor: number | null, - opts: StreamOptions, -): Promise<void> { - const { batchSize, flushIntervalMs, signal, collections, dependentCollections, knownDids, state, log } = opts; - - const subscription = opts.createSubscription - ? opts.createSubscription(cursor) - : new (await import("@atcute/jetstream")).JetstreamSubscription({ - url: config.jetstreams ?? [], - wantedCollections: collections, - ...(cursor !== null ? { cursor } : {}), - onConnectionOpen() { log.log("Connected to Jetstream"); }, - onConnectionClose(event: any) { log.log(`Disconnected: ${event.code} ${event.reason}`); }, - onConnectionError(event: any) { log.error("Jetstream error:", event.error); }, - }); - - const buffer: IngestEvent[] = []; - // Guards against overlap between the periodic timer flush and a main-loop - // batchSize-driven flush. The main loop only ever awaits flush() sequentially, - // but the setInterval callback is a second entry point on another tick. - let flushing = false; - - const flush = async () => { - if (buffer.length === 0 || flushing) return; - flushing = true; - const batch = buffer.splice(0); - - try { - await applyEvents(db, batch, config, { pubsub: opts.pubsub }); - - const lastTimeUs = Math.max(...batch.map((e) => e.time_us)); - await saveCursor(db, lastTimeUs); - - const uniqueDids = [...new Set(batch.map((e) => e.did))]; - if (uniqueDids.length > 0) { - try { - await refreshStaleIdentities(db, uniqueDids); - } catch (err) { - log.warn(`Identity refresh failed: ${err}`); - } - } - - if (config.feeds && Date.now() - state.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { - const maxItems = Math.max( - ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) - ); - const pruned = await pruneFeedItems(db, maxItems); - if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); - state.lastFeedPruneMs = Date.now(); - } - - log.log(`Flushed ${batch.length} events. Cursor: ${lastTimeUs}`); - } finally { - flushing = false; - } - }; - - // Periodic flush decoupled from the main loop. Runs even when Jetstream is - // idle, which is the whole point — without it, buffered events strand until - // the next event or abort. Errors log and retry next interval rather than - // propagate, so transient DB hiccups don't force a reconnect. - const flushTimer = setInterval(() => { - flush().catch((err) => log.error(`Timer flush failed: ${err}`)); - }, flushIntervalMs); - - const onAbort = () => { - clearInterval(flushTimer); - }; - signal?.addEventListener("abort", onAbort, { once: true }); - - const iterator = subscription[Symbol.asyncIterator](); - - try { - while (!signal?.aborted) { - // Per-iteration abort race so the handler can be removed synchronously - // after the race settles — otherwise addEventListener calls accumulate on - // the signal across the streamAndFlush lifetime. - let abortHandler!: () => void; - const abortPromise = new Promise<IteratorResult<any>>((resolve) => { - abortHandler = () => resolve({ value: undefined, done: true }); - signal?.addEventListener("abort", abortHandler, { once: true }); - }); - - let result: IteratorResult<any>; - try { - result = await Promise.race([iterator.next(), abortPromise]); - } finally { - signal?.removeEventListener("abort", abortHandler); - } - - if (result.done) break; - const event = result.value; - - if (event.kind === "commit") { - const { commit } = event; - - if (dependentCollections.has(commit.collection) && knownDids) { - if (!knownDids.has(event.did)) continue; - } - - const now = Date.now(); - const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; - - buffer.push({ - uri, - did: event.did, - time_us: event.time_us, - collection: commit.collection, - operation: commit.operation as "create" | "update" | "delete", - rkey: commit.rkey, - cid: commit.operation === "delete" ? null : commit.cid, - record: commit.operation === "delete" ? null : JSON.stringify(commit.record), - indexed_at: now * 1000, - }); - - if (knownDids && !dependentCollections.has(commit.collection)) { - knownDids.add(event.did); - } - } - - if (buffer.length >= batchSize) { - await flush(); - } - } - } finally { - clearInterval(flushTimer); - signal?.removeEventListener("abort", onAbort); - await iterator.return?.({ value: undefined, done: true }); - await flush(); - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/durable-object.ts b/packages/contrail/src/core/realtime/durable-object.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/durable-object.ts +++ b/packages/contrail/src/core/realtime/durable-object.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/in-memory.ts b/packages/contrail/src/core/realtime/in-memory.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/in-memory.ts +++ b/packages/contrail/src/core/realtime/in-memory.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/index.ts b/packages/contrail/src/core/realtime/index.ts index 05e39ec..551c495 100644 --- a/packages/contrail/src/core/realtime/index.ts +++ b/packages/contrail/src/core/realtime/index.ts @@ -1,37 +1 @@ -export { registerRealtimeRoutes } from "./router"; -export type { RealtimeRoutesOptions } from "./router"; -export { InMemoryPubSub } from "./in-memory"; -export { DurableObjectPubSub, RealtimePubSubDO } from "./durable-object"; -export type { - DurableObjectId, - DurableObjectNamespace, - DurableObjectStub, - DurableObjectState, -} from "./durable-object"; -export { TicketSigner } from "./ticket"; -export type { TicketPayload } from "./ticket"; -export { wrapWithPublishing } from "./publishing-adapter"; -export { sseResponse } from "./sse"; -export { pumpWebSocket } from "./websocket"; -export type { WebSocketLike } from "./websocket"; -export { mergeAsyncIterables } from "./merge"; -export { resolveTopicForCaller } from "./resolve"; -export type { TopicResolution, TopicResolutionContext, TopicResolutionError } from "./resolve"; -export type { - PubSub, - RealtimeConfig, - RealtimeEvent, - RealtimeEventKind, -} from "./types"; -export { - actorTopic, - collectionTopic, - communityTopic, - parseCommunityTopic, - parseSpaceTopic, - spaceTopic, - isCommunityTopic, - DEFAULT_KEEPALIVE_MS, - DEFAULT_QUEUE_BOUND, - DEFAULT_TICKET_TTL_MS, -} from "./types"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/merge.ts b/packages/contrail/src/core/realtime/merge.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/merge.ts +++ b/packages/contrail/src/core/realtime/merge.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/publishing-adapter.ts b/packages/contrail/src/core/realtime/publishing-adapter.ts index c5dc61f..551c495 100644 --- a/packages/contrail/src/core/realtime/publishing-adapter.ts +++ b/packages/contrail/src/core/realtime/publishing-adapter.ts @@ -1,157 +1 @@ -/** Decorator that wraps a spaces StorageAdapter and publishes realtime events - * after successful writes. Spaces and community modules stay unaware of - * realtime; the decorator is the only integration seam. */ - -import type { StorageAdapter, SpaceMemberRow } from "../spaces/types"; -import type { PubSub, RealtimeEvent } from "./types"; -import { communityTopic, spaceTopic } from "./types"; - -export interface PublishingAdapterOptions { - /** Optional lookup: given a space's ownerDid, return true if that DID is a - * community in the local `communities` table. When provided, writes also - * publish to `community:<ownerDid>` so subscribers who expanded that alias - * at ticket-mint time receive the event. - * - * The lookup is expected to be cheap (cached in the caller) — the decorator - * calls it on every write. */ - isCommunityDid?: (did: string) => Promise<boolean> | boolean; -} - -export function wrapWithPublishing( - inner: StorageAdapter, - pubsub: PubSub, - opts: PublishingAdapterOptions = {} -): StorageAdapter { - const publishSpaceAndCommunity = async ( - spaceUri: string, - ownerDid: string | null, - build: (topic: string) => RealtimeEvent - ): Promise<void> => { - await pubsub.publish(build(spaceTopic(spaceUri))); - if (ownerDid && opts.isCommunityDid && (await opts.isCommunityDid(ownerDid))) { - await pubsub.publish(build(communityTopic(ownerDid))); - } - }; - - const ownerOf = async (spaceUri: string): Promise<string | null> => { - const s = await inner.getSpace(spaceUri); - return s?.ownerDid ?? null; - }; - - const wrapped: StorageAdapter = { - ...inner, - createSpace: inner.createSpace.bind(inner), - getSpace: inner.getSpace.bind(inner), - listSpaces: inner.listSpaces.bind(inner), - deleteSpace: inner.deleteSpace.bind(inner), - updateSpaceAppPolicy: inner.updateSpaceAppPolicy.bind(inner), - getMember: inner.getMember.bind(inner), - listMembers: inner.listMembers.bind(inner), - createInvite: inner.createInvite.bind(inner), - listInvites: inner.listInvites.bind(inner), - revokeInvite: inner.revokeInvite.bind(inner), - getInvite: inner.getInvite.bind(inner), - redeemInvite: inner.redeemInvite.bind(inner), - enroll: inner.enroll.bind(inner), - getEnrollment: inner.getEnrollment.bind(inner), - listEnrollments: inner.listEnrollments.bind(inner), - removeEnrollment: inner.removeEnrollment.bind(inner), - getRecord: inner.getRecord.bind(inner), - listRecords: inner.listRecords.bind(inner), - listCollections: inner.listCollections.bind(inner), - putBlobMeta: inner.putBlobMeta.bind(inner), - getBlobMeta: inner.getBlobMeta.bind(inner), - listBlobMeta: inner.listBlobMeta.bind(inner), - deleteBlobMeta: inner.deleteBlobMeta.bind(inner), - findOrphanBlobs: inner.findOrphanBlobs.bind(inner), - - async addMember(spaceUri, did, addedBy) { - await inner.addMember(spaceUri, did, addedBy); - const owner = await ownerOf(spaceUri); - const now = Date.now(); - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.added", - payload: { space: spaceUri, did }, - ts: now, - })); - }, - - async removeMember(spaceUri, did) { - await inner.removeMember(spaceUri, did); - const owner = await ownerOf(spaceUri); - const now = Date.now(); - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.removed", - payload: { space: spaceUri, did }, - ts: now, - })); - }, - - async applyMembershipDiff(spaceUri, adds, removes, addedBy) { - await inner.applyMembershipDiff(spaceUri, adds, removes, addedBy); - if (adds.length === 0 && removes.length === 0) return; - const owner = await ownerOf(spaceUri); - const now = Date.now(); - for (const did of adds) { - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.added", - payload: { space: spaceUri, did }, - ts: now, - })); - } - for (const did of removes) { - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.removed", - payload: { space: spaceUri, did }, - ts: now, - })); - } - }, - - async putRecord(record) { - await inner.putRecord(record); - const owner = await ownerOf(record.spaceUri); - const now = Date.now(); - // Space records use ms timestamps; listRecords surface uses microseconds - // (time_us). Convert here so subscribers can render a row identically. - const time_us = record.createdAt * 1000; - const uri = `at://${record.authorDid}/${record.collection}/${record.rkey}`; - await publishSpaceAndCommunity(record.spaceUri, owner, (topic) => ({ - topic, - kind: "record.created", - payload: { - uri, - did: record.authorDid, - collection: record.collection, - rkey: record.rkey, - cid: record.cid, - record: record.record, - time_us, - space: record.spaceUri, - }, - ts: now, - })); - }, - - async deleteRecord(spaceUri, collection, authorDid, rkey) { - await inner.deleteRecord(spaceUri, collection, authorDid, rkey); - const owner = await ownerOf(spaceUri); - const now = Date.now(); - const uri = `at://${authorDid}/${collection}/${rkey}`; - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "record.deleted", - payload: { uri, did: authorDid, collection, rkey, space: spaceUri }, - ts: now, - })); - }, - }; - return wrapped; -} - -// Keep this import hint for types that downstream code might pull from here. -export type { SpaceMemberRow }; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/query-filter.ts b/packages/contrail/src/core/realtime/query-filter.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/query-filter.ts +++ b/packages/contrail/src/core/realtime/query-filter.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/resolve.ts b/packages/contrail/src/core/realtime/resolve.ts index c16abe5..551c495 100644 --- a/packages/contrail/src/core/realtime/resolve.ts +++ b/packages/contrail/src/core/realtime/resolve.ts @@ -1,92 +1 @@ -/** Resolve a raw topic request (as given by a caller) to the concrete set of - * delivery topics they are authorized to subscribe to. - * - * Rules (v1): - * - `space:<uri>` → allowed iff caller is owner or a member of the space. - * - `community:<did>` → expanded to `space:<uri>` for every space in the - * community reachable by the caller (direct grants - * or via delegation). `resolveReachableSpaces` - * already has exactly this semantics. - * - `actor:<did>` → self-only in v1. - * - `collection:<nsid>` → rejected unless the deployment opts in - * (not yet implemented). */ - -import type { StorageAdapter } from "../spaces/types"; -import type { CommunityProbe } from "../community-integration"; -import { spaceTopic, parseCommunityTopic, parseSpaceTopic } from "./types"; - -export interface TopicResolutionContext { - /** May be null when the deployment has no spaces module — in that case - * `space:` and `community:` topics are NotSupported. Public topics - * (`collection:`, `actor:`) still resolve. */ - spaces: StorageAdapter | null; - /** May be null if the community module is not enabled. */ - community: CommunityProbe | null; -} - -export interface TopicResolution { - ok: true; - topics: string[]; -} - -export interface TopicResolutionError { - ok: false; - error: "Forbidden" | "InvalidRequest" | "NotFound" | "NotSupported"; - reason: string; -} - -export async function resolveTopicForCaller( - rawTopic: string, - callerDid: string, - ctx: TopicResolutionContext -): Promise<TopicResolution | TopicResolutionError> { - // space:<uri> - const spaceUri = parseSpaceTopic(rawTopic); - if (spaceUri) { - if (!ctx.spaces) { - return { ok: false, error: "NotSupported", reason: "spaces-module-disabled" }; - } - const space = await ctx.spaces.getSpace(spaceUri); - if (!space) return { ok: false, error: "NotFound", reason: "space-not-found" }; - if (space.ownerDid === callerDid) return { ok: true, topics: [rawTopic] }; - const member = await ctx.spaces.getMember(spaceUri, callerDid); - if (!member) return { ok: false, error: "Forbidden", reason: "not-member" }; - return { ok: true, topics: [rawTopic] }; - } - - // community:<did> - const communityDid = parseCommunityTopic(rawTopic); - if (communityDid) { - if (!ctx.community || !ctx.spaces) { - return { ok: false, error: "NotSupported", reason: "community-module-disabled" }; - } - const row = await ctx.community.getCommunity(communityDid); - if (!row) return { ok: false, error: "NotFound", reason: "community-not-found" }; - const reachable = await ctx.community.resolveReachableSpaces(callerDid); - // Filter to spaces owned by THIS community — reachable may include spaces - // from other communities via cross-community delegation. - const ownedList = await ctx.spaces.listSpaces({ ownerDid: communityDid, limit: 1000 }); - const owned: Set<string> = new Set(ownedList.spaces.map((s) => s.uri)); - const topics: string[] = []; - for (const uri of reachable) { - if (owned.has(uri)) topics.push(spaceTopic(uri)); - } - if (topics.length === 0) { - return { ok: false, error: "Forbidden", reason: "no-reachable-spaces-in-community" }; - } - return { ok: true, topics }; - } - - // actor:<did> — public stream of records authored by this DID. - // Any caller can subscribe (parallels listRecords with an `actor` filter). - if (rawTopic.startsWith("actor:")) { - return { ok: true, topics: [rawTopic] }; - } - - // collection:<nsid> — public firehose for this collection. - if (rawTopic.startsWith("collection:")) { - return { ok: true, topics: [rawTopic] }; - } - - return { ok: false, error: "InvalidRequest", reason: "unknown-topic" }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/router.ts b/packages/contrail/src/core/realtime/router.ts index d10e45e..551c495 100644 --- a/packages/contrail/src/core/realtime/router.ts +++ b/packages/contrail/src/core/realtime/router.ts @@ -1,255 +1 @@ -/** Realtime XRPC routes: ticket mint + subscribe (SSE | WS). */ - -import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig } from "../types"; -import type { ServiceAuth } from "../spaces/auth"; -import type { StorageAdapter } from "../spaces/types"; -import type { CommunityProbe } from "../community-integration"; -import { InMemoryPubSub } from "./in-memory"; -import { TicketSigner } from "./ticket"; -import { sseResponse } from "./sse"; -import { pumpWebSocket, type WebSocketLike } from "./websocket"; -import { mergeAsyncIterables } from "./merge"; -import { resolveTopicForCaller } from "./resolve"; -import type { PubSub, RealtimeEvent } from "./types"; -import { DEFAULT_TICKET_TTL_MS, DEFAULT_KEEPALIVE_MS } from "./types"; - -export interface RealtimeRoutesOptions { - /** Auth middleware for `<ns>.realtime.ticket` and for JWT-based bot - * subscriptions to private topics. Null when no JWT verifier is available - * (deployments without a spaces config) — in that case, private-topic - * subscribe paths return NotSupported and public topics still work without - * auth. */ - authMiddleware: MiddlewareHandler | null; - pubsub?: PubSub; -} - -/** Public topics: subscribable without any auth. Mirrors listRecords - * semantics — no JWT means "public records only". */ -function isPublicTopic(topic: string): boolean { - return topic.startsWith("collection:") || topic.startsWith("actor:"); -} - -/** WebSocketPair exists on Cloudflare Workers; on Node/Bun it's absent. - * When absent, a platform-provided WebSocket accept hook is used instead. */ -interface WebSocketPairCtor { - new (): { 0: WebSocketLike & { accept?: () => void }; 1: WebSocketLike & { accept?: () => void } }; -} - -export function registerRealtimeRoutes( - app: Hono, - config: ContrailConfig, - spaces: StorageAdapter | null, - community: CommunityProbe | null, - options: RealtimeRoutesOptions -): void { - const cfg = config.realtime; - if (!cfg) return; - - const pubsub: PubSub = options.pubsub ?? cfg.pubsub ?? new InMemoryPubSub({ queueBound: cfg.queueBound }); - const signer = new TicketSigner(cfg.ticketSecret); - const ticketTtl = cfg.ticketTtlMs ?? DEFAULT_TICKET_TTL_MS; - const keepaliveMs = cfg.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; - - const NS = `${config.namespace}.realtime`; - - // POST /<ns>.realtime.ticket — { topic } → { ticket, topics, expiresAt } - // Ticket-minting exists so browsers (which can't set Authorization on - // EventSource) can subscribe to *private* topics. Public topics - // (collection:, actor:) don't need tickets — subscribe with `?topic=` directly. - if (options.authMiddleware) { - const authMw = options.authMiddleware; - app.post(`/xrpc/${NS}.ticket`, authMw, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { topic?: string } | null; - if (!body?.topic) { - return c.json({ error: "InvalidRequest", message: "topic required" }, 400); - } - const resolved = await resolveTopicForCaller(body.topic, sa.issuer, { spaces, community }); - if (!resolved.ok) { - const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; - return c.json({ error: resolved.error, reason: resolved.reason }, status); - } - const ticket = await signer.sign({ - topics: resolved.topics, - did: sa.issuer, - ttlMs: ticketTtl, - }); - return c.json({ - ticket, - topics: resolved.topics, - expiresAt: Date.now() + ticketTtl, - }); - }); - } - - // GET /<ns>.realtime.subscribe — SSE or WS. - // - // Three access paths, all land on the same stream: - // - `?topic=collection:<nsid>` or `?topic=actor:<did>` — *public*, no auth. - // Mirrors listRecords semantics. - // - `?ticket=<jwt>` — presented by browsers, minted via `.ticket` after - // a JWT-authenticated call. Only used for private topics. - // - `Authorization: Bearer <jwt>` + `?topic=space:<uri>` — server-side - // bots can skip the ticket dance and go straight to subscribe. - app.get(`/xrpc/${NS}.subscribe`, async (c) => { - const url = new URL(c.req.url); - const ticketParam = url.searchParams.get("ticket"); - const collectionFilter = url.searchParams.get("collection"); - const topicParam = url.searchParams.get("topic"); - - let callerDid: string | null = null; - let topics: string[]; - - if (ticketParam) { - const payload = await signer.verify(ticketParam); - if (!payload) { - return c.json({ error: "AuthRequired", reason: "invalid-or-expired-ticket" }, 401); - } - callerDid = payload.did; - topics = payload.topics; - // Optional: narrow to topics the query explicitly requests. - if (topicParam) { - if (!payload.topics.includes(topicParam)) { - return c.json({ error: "Forbidden", reason: "topic-not-in-ticket" }, 403); - } - topics = [topicParam]; - } - } else if (topicParam && isPublicTopic(topicParam)) { - // Public subscribe — no auth required. Jetstream ingestion publishes - // record events to collection:/actor: topics directly. - topics = [topicParam]; - } else { - // JWT path for private-topic bots. If no auth middleware is available - // (deployment has no spaces config), private topics aren't offered. - if (!options.authMiddleware) { - return c.json( - { - error: "InvalidRequest", - reason: "private-topic-without-auth", - message: - "Subscribing to space:/community: topics requires a JWT verifier; only public topics (collection:, actor:) are available on this deployment.", - }, - 400 - ); - } - let authed = false; - await options.authMiddleware(c, async () => { - authed = true; - }); - if (!authed) return c.res; // middleware already responded with 401 - const sa = getAuth(c); - callerDid = sa.issuer; - if (!topicParam) { - return c.json({ error: "InvalidRequest", message: "topic required" }, 400); - } - const resolved = await resolveTopicForCaller(topicParam, callerDid, { spaces, community }); - if (!resolved.ok) { - const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; - return c.json({ error: resolved.error, reason: resolved.reason }, status); - } - topics = resolved.topics; - } - - if (topics.length === 0) { - return c.json({ error: "InvalidRequest", reason: "no-topics" }, 400); - } - - // Build the merged iterable, with an inline filter that closes the stream - // on a matching `member.removed` event (self-kick on revocation). - const ac = new AbortController(); - const signals: AbortSignal[] = [ac.signal]; - const reqSignal = c.req.raw.signal; - if (reqSignal) signals.push(reqSignal); - const combined = anySignal(signals); - - const sources = topics.map((t) => pubsub.subscribe(t, combined)); - const merged = withSelfKickAndFilter( - mergeAsyncIterables(sources, combined), - callerDid, - collectionFilter, - ac - ); - - // Content negotiation: Upgrade: websocket → WS, else SSE. - if (c.req.header("Upgrade")?.toLowerCase() === "websocket") { - const Pair = (globalThis as unknown as { WebSocketPair?: WebSocketPairCtor }) - .WebSocketPair; - if (!Pair) { - return c.json( - { error: "NotSupported", reason: "websockets-require-worker-or-ws-adapter" }, - 426 - ); - } - const pair = new Pair(); - const clientWs = pair[0]; - const serverWs = pair[1]; - serverWs.accept?.(); - // Pump in the background; don't await. - void pumpWebSocket(serverWs, merged, combined, { keepaliveMs }); - return new Response(null, { - status: 101, - // Hono/undici-compat: some runtimes honor `webSocket` on the init. - // @ts-expect-error - Workers-specific init field - webSocket: clientWs, - }); - } - - return sseResponse(merged, combined, { keepaliveMs }); - }); -} - -// ============================================================================ -// Helpers -// ============================================================================ - -function getAuth(c: Context): ServiceAuth { - const a = c.get("serviceAuth") as ServiceAuth | undefined; - if (!a) throw new Error("service auth not set"); - return a; -} - -/** Merge multiple AbortSignals into one. Aborts when any source aborts. */ -function anySignal(signals: AbortSignal[]): AbortSignal { - const ac = new AbortController(); - for (const s of signals) { - if (s.aborted) { - ac.abort(); - return ac.signal; - } - s.addEventListener("abort", () => ac.abort(), { once: true }); - } - return ac.signal; -} - -/** Wrap an iterable: drop events that don't pass the collection filter (if - * any), and close the outer controller as soon as we see a `member.removed` - * for the caller's own DID. `callerDid` may be null on public subscriptions - * (anonymous) — in that case self-kick is not applicable. */ -function withSelfKickAndFilter( - source: AsyncIterable<RealtimeEvent>, - callerDid: string | null, - collectionFilter: string | null, - ac: AbortController -): AsyncIterable<RealtimeEvent> { - return { - async *[Symbol.asyncIterator]() { - for await (const event of source) { - if (event.kind === "member.removed" && event.payload.did === callerDid) { - // Deliver the kick event so the client sees why, then close. - yield event; - ac.abort(); - return; - } - if ( - collectionFilter && - (event.kind === "record.created" || event.kind === "record.deleted") && - event.payload.collection !== collectionFilter - ) { - continue; - } - yield event; - } - }, - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/sse.ts b/packages/contrail/src/core/realtime/sse.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/sse.ts +++ b/packages/contrail/src/core/realtime/sse.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/ticket.ts b/packages/contrail/src/core/realtime/ticket.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/ticket.ts +++ b/packages/contrail/src/core/realtime/ticket.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/types.ts b/packages/contrail/src/core/realtime/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/types.ts +++ b/packages/contrail/src/core/realtime/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/websocket.ts b/packages/contrail/src/core/realtime/websocket.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/websocket.ts +++ b/packages/contrail/src/core/realtime/websocket.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/refresh.ts b/packages/contrail/src/core/refresh.ts index 7b2fd21..551c495 100644 --- a/packages/contrail/src/core/refresh.ts +++ b/packages/contrail/src/core/refresh.ts @@ -1,267 +1 @@ -import type {} from "@atcute/atproto"; -/** - * Fresh refresh: re-walk every known DID's PDS for every configured collection - * and reconcile against what's in our DB. Unlike `backfillPending`, this - * ignores the `backfills` state machine — it's a "check what we might have - * missed" pass, not a resumable bulk load. - * - * Two categories of delta are counted: - * - missing — the PDS has a record we don't - * - staleUpdates — we have the same URI but a different CID, *and* our - * copy's `indexed_at` is older than `ignoreWindowMs` - * - * The ignore window exists because Jetstream can run ~seconds behind the - * PDS; without the window, "in-sync but racy" writes would show up as - * misses every run. Records inside the window are still applied (they - * might be legit updates), just not counted toward stats. - * - * Typical uses: - * - dev: "I ran backfillAll on Monday, haven't touched it for a week, - * how much did jetstream miss?" - * - prod: "we had jetstream outage yesterday, what did we drop?" - */ -import { type Did, type Nsid } from "@atcute/lexicons"; -import { isDid, isNsid } from "@atcute/lexicons/syntax"; - -import type { Client } from "@atcute/client"; -import type { ContrailConfig, Database, IngestEvent } from "./types.js"; -import { applyEvents, lookupExistingRecords } from "./db/records.js"; -import { getClient } from "./client.js"; - -const PAGE_SIZE = 100; -const REQUEST_TIMEOUT_MS = 10_000; - -async function withTimeout<T>(fn: () => Promise<T>, ms: number): Promise<T> { - return Promise.race([ - fn(), - new Promise<never>((_, rej) => - setTimeout(() => rej(new Error(`timeout after ${ms}ms`)), ms) - ), - ]); -} - -export interface CollectionStats { - /** Record exists on PDS but was absent from our DB. */ - missing: number; - /** Record exists in our DB with a different CID than the PDS, and our - * copy was written before the ignore window. */ - staleUpdates: number; - /** Record is present and matches (same CID, or within ignore window). */ - inSync: number; -} - -export interface RefreshProgress { - usersComplete: number; - usersTotal: number; - usersFailed: number; - recordsScanned: number; -} - -export interface RefreshResult { - /** Per-NSID stats. */ - byCollection: Record<string, CollectionStats>; - /** Sum across every NSID. */ - total: CollectionStats; - usersScanned: number; - usersFailed: number; - /** Effective ignore window used for classification, in ms. */ - ignoreWindowMs: number; - /** Wall-clock runtime, in ms. */ - elapsedMs: number; -} - -export interface RefreshOptions { - /** How many DIDs to fan out against in parallel. Default: 50. */ - concurrency?: number; - /** Records whose local `indexed_at` is within this window of `now` are - * still upserted but excluded from `staleUpdates` counts — guards - * against jetstream being briefly behind the PDS. Default: 60_000 ms. */ - ignoreWindowMs?: number; - /** Override which NSIDs to walk. Default: every `config.collections[*].collection`. */ - nsids?: string[]; - /** Optional progress callback (fires per completed DID). */ - onProgress?: (p: RefreshProgress) => void; - /** Max attempts per listRecords request. Default: 3. */ - maxRetries?: number; - /** Per-request timeout in ms. Default: 10000. */ - requestTimeout?: number; -} - -function emptyStats(): CollectionStats { - return { missing: 0, staleUpdates: 0, inSync: 0 }; -} - -export async function refresh( - db: Database, - config: ContrailConfig, - options?: RefreshOptions -): Promise<RefreshResult> { - const concurrency = options?.concurrency ?? 50; - const ignoreWindowMs = options?.ignoreWindowMs ?? 60_000; - const requestTimeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; - const maxRetries = options?.maxRetries ?? 3; - const startedAt = Date.now(); - - // Default to every configured collection NSID. Profiles are already - // included because `resolveConfig` adds them to `config.collections`. - const nsids = - options?.nsids ?? - Object.values(config.collections).map((c) => c.collection); - - const byCollection: Record<string, CollectionStats> = {}; - for (const nsid of nsids) byCollection[nsid] = emptyStats(); - const total: CollectionStats = emptyStats(); - - // Known DIDs = every author we've ever written for. `backfills` is a - // superset (it also includes failed/pending users that we never got - // records from), which is actually what we want — if we tried and - // failed before, we might succeed now. - const didRows = await db - .prepare("SELECT DISTINCT did FROM backfills") - .all<{ did: string }>(); - const dids = (didRows.results ?? []) - .map((r) => r.did) - .filter((d) => isDid(d)); - - const usersTotal = dids.length; - let usersComplete = 0; - let usersFailed = 0; - let recordsScanned = 0; - - const ignoreBeforeUs = (Date.now() - ignoreWindowMs) * 1000; - - const processDid = async (did: string): Promise<void> => { - let client: Client; - try { - client = await withTimeout( - () => getClient(did as Did, db), - requestTimeout - ); - } catch { - usersFailed++; - return; - } - - for (const nsid of nsids) { - if (!isNsid(nsid)) continue; - let cursor: string | undefined; - while (true) { - let pageRecords: Array<{ uri: string; cid: string; value: unknown }>; - let nextCursor: string | undefined; - try { - // Retry listRecords: transient PDS failures are expected during refresh - let attempt = 0; - // eslint-disable-next-line no-constant-condition - while (true) { - try { - const res = await withTimeout( - () => - client.get("com.atproto.repo.listRecords", { - params: { - repo: did as Did, - collection: nsid as Nsid, - limit: PAGE_SIZE, - cursor, - }, - }), - requestTimeout - ); - if (!res.ok) { - // 400s on a collection the user doesn't have are fine; stop - // paging this collection for this user. - pageRecords = []; - nextCursor = undefined; - break; - } - pageRecords = res.data.records; - nextCursor = res.data.cursor ?? undefined; - break; - } catch (err) { - if (attempt >= maxRetries) throw err; - attempt++; - await new Promise((r) => setTimeout(r, 500 * 2 ** attempt)); - } - } - } catch { - // Give up on this collection for this user; keep going. - break; - } - - if (pageRecords.length === 0) break; - - const now = Date.now(); - const events: IngestEvent[] = pageRecords.map((r) => ({ - uri: r.uri, - did, - collection: nsid, - rkey: r.uri.split("/").pop()!, - operation: "create" as const, - cid: r.cid, - record: JSON.stringify(r.value), - time_us: now * 1000, - indexed_at: now * 1000, - })); - - const existing = await lookupExistingRecords( - db, - events.map((e) => ({ uri: e.uri, collection: e.collection })), - false, - config - ); - - for (const ev of events) { - const ex = existing.get(ev.uri); - if (!ex) { - byCollection[nsid].missing++; - total.missing++; - } else if (ex.cid !== ev.cid) { - const inWindow = - ex.indexed_at !== null && ex.indexed_at >= ignoreBeforeUs; - if (inWindow) { - byCollection[nsid].inSync++; - total.inSync++; - } else { - byCollection[nsid].staleUpdates++; - total.staleUpdates++; - } - } else { - byCollection[nsid].inSync++; - total.inSync++; - } - } - - // Upsert everything — even records "inside the ignore window" - // might genuinely have a new CID; we just don't count them as a - // miss-signal. Skip feed fanout since this is a catch-up, not a - // user-visible write. - await applyEvents(db, events, config, { skipFeedFanout: true }); - recordsScanned += events.length; - - cursor = nextCursor; - if (!cursor) break; - } - } - - usersComplete++; - options?.onProgress?.({ - usersComplete, - usersTotal, - usersFailed, - recordsScanned, - }); - }; - - for (let i = 0; i < dids.length; i += concurrency) { - const batch = dids.slice(i, i + concurrency); - await Promise.allSettled(batch.map(processDid)); - } - - return { - byCollection, - total, - usersScanned: usersComplete, - usersFailed, - ignoreWindowMs, - elapsedMs: Date.now() - startedAt, - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/admin.ts b/packages/contrail/src/core/router/admin.ts index a3b754b..551c495 100644 --- a/packages/contrail/src/core/router/admin.ts +++ b/packages/contrail/src/core/router/admin.ts @@ -1,44 +1 @@ -import type { Hono } from "hono"; -import type { ContrailConfig, Database } from "../types"; -import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../types"; -import { getLastCursor } from "../db"; - -export function registerAdminRoutes( - app: Hono, - db: Database, - config: ContrailConfig -): void { - const ns = config.namespace; - - app.get(`/xrpc/${ns}.getCursor`, async (c) => { - const cursor = await getLastCursor(db); - if (cursor === null) return c.json({ cursor: null }); - - const dateMs = Math.floor(cursor / 1000); - return c.json({ - time_us: cursor, - date: new Date(dateMs).toISOString(), - seconds_ago: Math.floor((Date.now() - dateMs) / 1000), - }); - }); - - app.get(`/xrpc/${ns}.getOverview`, async (c) => { - const collections: { collection: string; records: number; unique_users: number }[] = []; - - for (const short of getCollectionShortNames(config)) { - const table = recordsTableName(short); - const nsid = nsidForShortName(config, short) ?? short; - const row = await db - .prepare(`SELECT COUNT(*) as records, COUNT(DISTINCT did) as unique_users FROM ${table}`) - .first<{ records: number; unique_users: number }>(); - if (row) { - collections.push({ collection: nsid, records: row.records, unique_users: row.unique_users }); - } - } - - return c.json({ - total_records: collections.reduce((sum, col) => sum + col.records, 0), - collections, - }); - }); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/collection.ts b/packages/contrail/src/core/router/collection.ts index 6d43d00..551c495 100644 --- a/packages/contrail/src/core/router/collection.ts +++ b/packages/contrail/src/core/router/collection.ts @@ -1,1225 +1 @@ -import type { Context, Hono } from "hono"; -import type { ContrailConfig, ResolvedContrailConfig, Database, RecordRow, QueryableField, RecordSource, RelationConfig } from "../types"; -import { - getCollectionShortNames, - countColumnName, - groupedCountColumnName, - recordsTableName, - nsidForShortName, - getCollectionMethods, -} from "../types"; -import { queryRecords, queryAcrossSources } from "../db"; -import type { SortOption } from "../db/records"; -import { backfillUser } from "../backfill"; -import { resolveHydrates, resolveReferences, parseHydrateParams } from "./hydrate"; -import { resolveProfiles, collectDids } from "./profiles"; -import { resolveActor } from "../identity"; -import type { FormattedRecord } from "./helpers"; -import { formatRecord, parseIntParam, fieldToParam } from "./helpers"; -import { selectAcceptedLabelers } from "../labels/select"; -import { hydrateLabels } from "../labels/hydrate"; -import { verifyServiceAuthRequest, extractInviteToken, checkInviteReadGrant } from "../spaces/auth"; -import { checkAccess } from "../spaces/acl"; -import { hashInviteToken } from "../invite/token"; -import type { SpacesContext } from "."; -import type { Nsid } from "@atcute/lexicons"; -import type { RealtimeEvent } from "../realtime/types"; -import { sseResponse } from "../realtime/sse"; -import { spaceTopic, communityTopic, parseSpaceTopic } from "../realtime/types"; -import type { SubscriberQuerySpec } from "../realtime/durable-object"; -import { DurableObjectPubSub } from "../realtime/durable-object"; -import { TicketSigner, type TicketQuerySpec } from "../realtime/ticket"; -import { resolveTopicForCaller } from "../realtime/resolve"; -import { mergeAsyncIterables } from "../realtime/merge"; -import type { CommunityProbe } from "../community-integration"; -import { getRelationField, getNestedValue } from "../types"; - -/** Scope of a watch stream. - * - `space`: single permissioned space — one `space:<uri>` topic. - * - `actor`: records authored by `actor` across multiple spaces — the - * resolver expanded these to a per-caller subset of space topics (plus - * `actor:<did>` for public records). Events outside `allowedSpaces` - * are filtered out. */ -type WatchScope = - | { kind: "space"; spaceUri: string } - | { - kind: "actor"; - actor: string; - /** Concrete pubsub topics to subscribe to (from resolveTopicForCaller). */ - topics: string[]; - /** Space URIs the caller can see. Events with `space` outside this - * set are dropped. Undefined `space` on an event (public record) - * is allowed only when `actor` topic is in `topics`. */ - allowedSpaces: Set<string>; - }; - -/** Shared implementation of the watchRecords snapshot+live loop. Called by - * both transport branches (SSE and Worker-terminated WS). The caller owns - * the actual socket/stream and provides a `send(kind, data)` closure. */ -async function runQueryStream(opts: { - send: (kind: string, data: unknown) => void; - abort: AbortController; - scope: WatchScope; - callerDid: string | undefined; - params: URLSearchParams; - db: Database; - config: ContrailConfig; - collection: string; - colNsid: string; - pubsub: import("../realtime/types").PubSub; - relations: Record<string, import("../types").RelationConfig>; - references: Record<string, import("../types").ReferenceConfig>; - childCollectionMap: Map< - string, - { relName: string; matchField: string; matchMode: "uri" | "did" } - >; -}): Promise<void> { - const { - send, - abort, - scope, - callerDid, - params, - db, - config, - collection, - colNsid, - pubsub, - relations, - references, - childCollectionMap - } = opts; - - // Predicate: does this event belong in the caller's scope? - const inScope = (space: string | undefined): boolean => { - if (scope.kind === "space") return space === scope.spaceUri; - if (space == null) return false; // actor mode: require space for now (app topic) - return scope.allowedSpaces.has(space); - }; - - const hydrateSpec = parseHydrateParams(params, relations, references); - const trackHydration = Object.keys(hydrateSpec.relations).length > 0; - const parentUris = new Set<string>(); - const parentDids = new Set<string>(); - const childToParent = new Map<string, { parentUri: string; relName: string }>(); - - const primaryUri = (payload: { uri: string }) => payload.uri; - - const handleChildEvent = (event: RealtimeEvent) => { - if (!trackHydration) return; - if (event.kind !== "record.created" && event.kind !== "record.deleted") return; - const meta = childCollectionMap.get(event.payload.collection); - if (!meta) return; - if (!(hydrateSpec.relations as Record<string, number>)[meta.relName]) return; - if (!inScope(event.payload.space)) return; - // Actor mode: additionally require the record's author match our actor - // (the caller might share spaces with other authors — we only surface - // records by the actor under watch). - if (scope.kind === "actor" && event.payload.did !== scope.actor) return; - - if (event.kind === "record.created") { - const matched = getNestedValue(event.payload.record, meta.matchField); - if (matched == null) return; - const parent = - meta.matchMode === "did" - ? parentDids.has(String(matched)) - ? `at://${String(matched)}/${colNsid}/_` - : null - : parentUris.has(String(matched)) - ? String(matched) - : null; - if (!parent) return; - childToParent.set(event.payload.rkey, { - parentUri: parent, - relName: meta.relName - }); - send("hydration.added", { - parentUri: parent, - relation: meta.relName, - child: { - uri: primaryUri(event.payload), - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - value: event.payload.record, - space: event.payload.space - } - }); - } else { - const info = childToParent.get(event.payload.rkey); - if (!info) return; - childToParent.delete(event.payload.rkey); - send("hydration.removed", { - parentUri: info.parentUri, - relation: info.relName, - childRkey: event.payload.rkey, - childDid: event.payload.did - }); - } - }; - - const handleLive = (event: RealtimeEvent) => { - if (abort.signal.aborted) return; - if (event.kind === "member.removed" && event.payload.did === callerDid) { - send("member.removed", event.payload); - abort.abort(); - return; - } - if (event.kind !== "record.created" && event.kind !== "record.deleted") return; - if (!inScope(event.payload.space)) return; - if (scope.kind === "actor" && event.payload.did !== scope.actor) return; - - if (event.payload.collection !== colNsid) { - handleChildEvent(event); - return; - } - - const nowUs = event.ts * 1000; - const uri = primaryUri(event.payload); - if (event.kind === "record.created") { - parentUris.add(uri); - parentDids.add(event.payload.did); - send("record.created", { - record: { - uri, - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - value: event.payload.record, - time_us: nowUs, - indexed_at: event.ts, - space: event.payload.space - } - }); - } else { - parentUris.delete(uri); - send("record.deleted", { - uri, - did: event.payload.did, - rkey: event.payload.rkey - }); - } - }; - - // Subscribe: one topic for space-scoped, merge across all topics for - // actor-scoped. `mergeAsyncIterables` exists for exactly this case. - let iter: AsyncIterable<RealtimeEvent>; - if (scope.kind === "space") { - iter = pubsub.subscribe(spaceTopic(scope.spaceUri), abort.signal); - } else { - const sources = scope.topics.map((t) => pubsub.subscribe(t, abort.signal)); - iter = mergeAsyncIterables(sources, abort.signal); - } - - const buffered: RealtimeEvent[] = []; - let snapshotDone = false; - - const pump = (async () => { - try { - for await (const event of iter) { - if (abort.signal.aborted) break; - if (!snapshotDone) buffered.push(event); - else handleLive(event); - } - } catch { - /* aborted or errored */ - } - })(); - - try { - send( - "snapshot.start", - scope.kind === "space" - ? { spaceUri: scope.spaceUri, collection: colNsid } - : { actor: scope.actor, collection: colNsid } - ); - const snapshotSpaces = - scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); - const result = await runPipeline(db, config, collection, params, undefined, snapshotSpaces); - for (const record of result.records) { - if (abort.signal.aborted) break; - if (typeof record.uri === "string") parentUris.add(record.uri); - if (typeof record.did === "string") parentDids.add(record.did); - for (const [relName] of Object.entries(hydrateSpec.relations)) { - const hydratedGroups = (record as Record<string, unknown>)[relName]; - if (!hydratedGroups) continue; - const flat: Array<{ rkey?: string }> = Array.isArray(hydratedGroups) - ? (hydratedGroups as Array<{ rkey?: string }>) - : (Object.values(hydratedGroups as Record<string, unknown>).flat() as Array<{ - rkey?: string; - }>); - for (const child of flat) { - if (child?.rkey) { - childToParent.set(child.rkey, { - parentUri: record.uri as string, - relName - }); - } - } - } - send("snapshot.record", { record }); - } - send("snapshot.end", { cursor: result.cursor }); - snapshotDone = true; - for (const event of buffered) handleLive(event); - } catch (err) { - send("error", { - message: err instanceof Error ? err.message : String(err) - }); - abort.abort(); - } - - await pump.catch(() => {}); -} - -export async function runPipeline( - db: Database, - config: ContrailConfig, - collection: string, - params: URLSearchParams, - source?: RecordSource, - spaceUris?: string[], - /** Optional headers from the originating request — used for label - * hydration (`atproto-accept-labelers`). Other entry points pass nothing - * and labels are gated by `?labelers=` / config defaults. */ - headers?: Headers -): Promise<{ records: FormattedRecord[]; cursor?: string; profiles?: any[]; labelersApplied?: string[] }> { - const colConfig = config.collections[collection]; - if (!colConfig) throw new Error(`Unknown collection: ${collection}`); - - const relations = colConfig.relations ?? {}; - const references = colConfig.references ?? {}; - const queryableFields: Record<string, QueryableField> = - (config as ResolvedContrailConfig)._resolved?.queryable[collection] ?? colConfig.queryable ?? {}; - - const limit = parseIntParam(params.get("limit"), 50); - const cursor = params.get("cursor") || undefined; - const actor = params.get("actor") || params.get("did") || undefined; - const wantProfiles = params.get("profiles") === "true"; - - let did: string | undefined; - if (actor) { - const resolved = await resolveActor(db, actor); - if (!resolved) throw new Error("Could not resolve actor"); - did = resolved; - // backfillUser expects the record NSID (for PDS calls), not the short name. - const nsid = nsidForShortName(config, collection) ?? collection; - await backfillUser(db, did, nsid, Date.now() + 3_000, config, { - maxRetries: 0, - requestTimeout: 3_000, - }); - } - - const filters: Record<string, string> = {}; - const rangeFilters: Record<string, { min?: string; max?: string }> = {}; - for (const [field, fieldConfig] of Object.entries(queryableFields)) { - const param = fieldToParam(field); - if (fieldConfig.type === "range") { - const min = params.get(`${param}Min`); - const max = params.get(`${param}Max`); - if (min || max) { - rangeFilters[field] = {}; - if (min) rangeFilters[field].min = min; - if (max) rangeFilters[field].max = max; - } - } else { - const value = params.get(param); - if (value) filters[field] = value; - } - } - - const countFilters: Record<string, number> = {}; - const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; - for (const [relName, rel] of Object.entries(relations)) { - const totalMin = parseIntParam(params.get(`${relName}CountMin`)); - if (totalMin != null) countFilters[rel.collection] = totalMin; - const mapping = relMap[relName]; - if (mapping) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - for (const [shortName, fullToken] of Object.entries(mapping.groups)) { - const val = parseIntParam(params.get(`${relName}${capitalize(shortName)}CountMin`)); - if (val != null) countFilters[fullToken] = val; - } - } - } - - let sort: SortOption | undefined; - const sortParam = params.get("sort"); - if (sortParam) { - const orderParam = params.get("order"); - - const fieldEntry = Object.entries(queryableFields).find( - ([field]) => fieldToParam(field) === sortParam - ); - if (fieldEntry) { - const defaultDir = fieldEntry[1].type === "range" ? "desc" : "asc"; - const direction = orderParam === "asc" ? "asc" as const : orderParam === "desc" ? "desc" as const : defaultDir as "asc" | "desc"; - sort = { recordField: fieldEntry[0], direction }; - } else { - const direction = orderParam === "asc" ? "asc" as const : "desc" as const; - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - for (const [relName, rel] of Object.entries(relations)) { - if (sortParam === `${relName}Count`) { - sort = { countType: rel.collection, direction }; - break; - } - const mapping = relMap[relName]; - if (mapping) { - for (const [shortName, fullToken] of Object.entries(mapping.groups)) { - if (sortParam === `${relName}${capitalize(shortName)}Count`) { - sort = { countType: fullToken, direction }; - break; - } - } - if (sort) break; - } - } - } - } - - const search = params.get("search") || undefined; - const spaceUri = params.get("spaceUri") || undefined; - - const queryOpts = { - collection, - did, - limit, - cursor, - filters, - rangeFilters, - countFilters, - sort, - search, - source, - spaceUri, - }; - const result = spaceUris && spaceUris.length > 0 && !spaceUri - ? await queryAcrossSources(db, config, queryOpts, spaceUris) - : await queryRecords(db, config, queryOpts); - - const rows = result.records; - const hydrateRequested = parseHydrateParams(params, relations, references); - const hydrates = await resolveHydrates( - db, - relations, - hydrateRequested.relations, - rows, - config - ); - const refs = await resolveReferences( - db, - references, - hydrateRequested.references, - rows, - config - ); - - const formattedRecords: FormattedRecord[] = rows.map((row) => { - const formatted = formatRecord(row); - flattenCounts(formatted, row.counts, relations); - const h = hydrates[row.uri]; - if (h) { - for (const [relName, groups] of Object.entries(h)) { - formatted[relName] = groups; - } - } - const r = refs[row.uri]; - if (r) { - for (const [refName, record] of Object.entries(r)) { - formatted[refName] = record; - } - } - return formatted; - }); - - const allDids = collectDids(rows, hydrates); - const profileMap = wantProfiles - ? await resolveProfiles(db, config, allDids) - : undefined; - - let labelersApplied: string[] | undefined; - if (config.labels) { - const sel = selectAcceptedLabelers( - headers?.get("atproto-accept-labelers") ?? null, - params.get("labelers"), - config.labels, - ); - if (sel.accepted.length > 0) { - const subjects: string[] = [ - ...formattedRecords.map((r) => r.uri), - ...allDids, - ]; - const cidByUri = new Map<string, string | null>(); - for (const r of formattedRecords) cidByUri.set(r.uri, r.cid); - const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); - for (const fr of formattedRecords) { - const ls = labelsByUri[fr.uri]; - if (ls && ls.length > 0) fr.labels = ls; - } - if (profileMap) { - for (const entries of Object.values(profileMap)) { - for (const entry of entries) { - const ls = labelsByUri[entry.did]; - if (ls && ls.length > 0) entry.labels = ls; - } - } - } - labelersApplied = sel.accepted; - } - } - - return { - records: formattedRecords, - cursor: result.cursor, - ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), - ...(labelersApplied ? { labelersApplied } : {}), - }; -} - -/** Serialize a runPipeline result as JSON, echoing - * `atproto-content-labelers` when labels were applied. The result's - * `labelersApplied` field never appears in the response body — it's a - * side channel for the route to read and turn into a header. */ -function jsonWithLabelers(c: Context, result: { labelersApplied?: string[] } & Record<string, unknown>) { - const { labelersApplied, ...body } = result; - if (labelersApplied && labelersApplied.length > 0) { - c.header("atproto-content-labelers", labelersApplied.join(",")); - } - return c.json(body); -} - -export function registerCollectionRoutes( - app: Hono, - db: Database, - config: ContrailConfig, - spacesCtx?: SpacesContext | null, - options: { - pubsub?: import("../realtime/types").PubSub | null; - community?: CommunityProbe | null; - } = {} -): void { - const ns = config.namespace; - const pubsub = options.pubsub ?? null; - const community = options.community ?? null; - - /** When a per-collection endpoint receives `?spaceUri=...`, verify the JWT, - * resolve membership, run the space ACL, and return the caller DID if allowed. - * Returns null if the spaces subsystem isn't available; the handler should - * then treat the spaceUri as invalid. - * Throws by returning a Response (caller checks via `instanceof Response`). */ - async function gateSpaceAccess( - c: Context, - spaceUri: string, - op: "read" - ): Promise<Response | { callerDid?: string; clientId?: string; viaInviteToken?: boolean }> { - if (!spacesCtx) { - return c.json( - { error: "InvalidRequest", message: "spaces not configured on this service" }, - 501 - ); - } - - // Read-token path: anonymous bearer access via `?inviteToken=...` (or - // `Authorization: Bearer atmo-invite:<token>`). Token must exist, be - // unexpired/unrevoked, scoped to this space, and have a kind that grants - // read (`read` or `read-join`). Token kind cannot grant write — caller must - // separately redeem to become a member for any non-read op. - if (op === "read") { - const rawToken = extractInviteToken(c.req.raw); - if (rawToken) { - const ok = await checkInviteReadGrant( - spacesCtx.adapter, - rawToken, - spaceUri, - hashInviteToken - ); - if (ok) { - const space = await spacesCtx.adapter.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - return { viaInviteToken: true }; - } - return c.json( - { error: "Forbidden", reason: "invalid-invite-token" }, - 403 - ); - } - } - - const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; - const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); - if (!auth) { - return c.json( - { error: "AuthRequired", message: "spaceUri requires a valid service-auth JWT or read-grant invite token" }, - 401 - ); - } - const space = await spacesCtx.adapter.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await spacesCtx.adapter.getMember(spaceUri, auth.issuer); - const result = checkAccess({ - op, - space, - callerDid: auth.issuer, - member, - clientId: auth.clientId, - }); - if (!result.allow) { - return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - return { callerDid: auth.issuer, clientId: auth.clientId }; - } - - for (const collection of getCollectionShortNames(config)) { - const colConfig = config.collections[collection]; - const methods = getCollectionMethods(colConfig); - - if (methods.includes("listRecords")) { - app.get(`/xrpc/${ns}.${collection}.listRecords`, async (c) => { - const params = new URL(c.req.url).searchParams; - const spaceUri = params.get("spaceUri") || undefined; - - if (spaceUri) { - const gated = await gateSpaceAccess(c, spaceUri, "read"); - if (gated instanceof Response) return gated; - // Route through runPipeline with a single-element space list so the - // full filter / sort / hydrate / reference surface works on per-space - // queries too, not just on the cross-space union path. - try { - const result = await runPipeline(db, config, collection, params, undefined, [spaceUri], c.req.raw.headers); - return jsonWithLabelers(c, result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - } - - // Union path: when the caller is authenticated, fold in records from - // spaces they're a member of. Anonymous callers just get public results. - let spaceUris: string[] | undefined; - const hasAuthHeader = !!c.req.header("Authorization"); - if (spacesCtx) { - const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; - const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); - if (auth) { - const { spaces } = await spacesCtx.adapter.listSpaces({ - memberDid: auth.issuer, - limit: 200, - }); - spaceUris = spaces.map((s) => s.uri); - } else if (hasAuthHeader) { - // Had an auth header but it was invalid — reject rather than - // silently downgrading to public results. - return c.json( - { error: "AuthRequired", message: "invalid service-auth JWT" }, - 401 - ); - } - } - - try { - const result = await runPipeline(db, config, collection, params, undefined, spaceUris, c.req.raw.headers); - return jsonWithLabelers(c, result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - }); - - // Streaming variant — same query shape, SSE'd forever. Opted in via the - // presence of the realtime module; no explicit method config needed. - if (pubsub && spacesCtx) { - const colNsid = colConfig.collection; - const relations = colConfig.relations ?? {}; - const references = colConfig.references ?? {}; - // Map child-NSID → { relName, matchField } so we can route child - // events to hydration deltas without re-parsing config per event. - const childCollectionMap = new Map< - string, - { relName: string; matchField: string; matchMode: "uri" | "did" } - >(); - for (const [relName, rel] of Object.entries(relations)) { - const childNsid = nsidForShortName(config, rel.collection) ?? rel.collection; - childCollectionMap.set(childNsid, { - relName, - matchField: getRelationField(rel), - matchMode: rel.match ?? "uri" - }); - } - - // TicketSigner for watch-scoped tickets. Minted on `mode=ws` handshake - // so the subsequent WS upgrade can auth with just `?ticket=...` (no - // cookie or JWT needed — enables cross-origin + stateless clients). - const ticketSigner = config.realtime?.ticketSecret - ? new TicketSigner(config.realtime.ticketSecret) - : null; - const ticketTtl = config.realtime?.ticketTtlMs ?? 120_000; - - app.get(`/xrpc/${ns}.${collection}.watchRecords`, async (c) => { - const params = new URL(c.req.url).searchParams; - const spaceUri = params.get("spaceUri"); - const actorParam = params.get("actor"); - - if (!spaceUri && !actorParam) { - return c.json( - { error: "InvalidRequest", message: "spaceUri or actor required" }, - 400 - ); - } - - // Resolve the caller and their scope. Two parallel paths: - // - space-scoped: single `space:<uri>` topic, per-space ACL gate. - // - actor-scoped: caller's reachable spaces in the actor's - // community (v1 only supports community DIDs as the actor). - // Events are delivered via N `space:<uri>` topics and filtered - // to `did === actor`. - let callerDid: string | undefined; - let scope: WatchScope; - let scopeTopics: string[]; // for ticket signing - let ticketSpec: TicketQuerySpec | null = null; - - const providedTicket = params.get("ticket"); - if (providedTicket && ticketSigner) { - const payload = await ticketSigner.verify(providedTicket); - if (payload?.querySpec && payload.querySpec.collection === colNsid) { - const ts = payload.querySpec; - if (spaceUri && ts.spaceUri === spaceUri) { - if (payload.topics.includes(spaceTopic(spaceUri))) { - callerDid = payload.did; - ticketSpec = { - collection: ts.collection, - spaceUri: ts.spaceUri, - ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) - }; - } - } else if (actorParam && ts.actor === actorParam) { - callerDid = payload.did; - ticketSpec = { - collection: ts.collection, - actor: ts.actor, - ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) - }; - } - } - } - - const hydrateSpec = parseHydrateParams(params, relations, references); - const hydrateForSpec = Object.keys(hydrateSpec.relations).length > 0 - ? Object.fromEntries( - Object.entries(hydrateSpec.relations).map(([relName]) => { - const rel = relations[relName]!; - const childNsid = - nsidForShortName(config, rel.collection) ?? rel.collection; - return [ - relName, - { childCollection: childNsid, matchField: getRelationField(rel) } - ]; - }) - ) - : undefined; - - if (spaceUri) { - if (!ticketSpec) { - const gated = await gateSpaceAccess(c, spaceUri, "read"); - if (gated instanceof Response) return gated; - callerDid = "callerDid" in gated ? gated.callerDid : undefined; - } - scope = { kind: "space", spaceUri }; - scopeTopics = [spaceTopic(spaceUri)]; - } else { - // Actor-scoped path — v1 only supports community DIDs. - const actor = actorParam!; - if (!community || !spacesCtx) { - return c.json( - { error: "NotSupported", reason: "community-module-disabled" }, - 400 - ); - } - const isCommunity = !!(await community.getCommunity(actor)); - if (!isCommunity) { - return c.json( - { error: "InvalidRequest", reason: "actor-must-be-community-did", message: "cross-space watch currently only supports community DIDs as actor" }, - 400 - ); - } - - if (!ticketSpec) { - // Verify the caller via the same JWT/in-process path used for - // per-space queries, then resolve the community topic to the - // caller's accessible space topics. - const nsidLxm = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; - const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsidLxm); - if (!auth) { - return c.json( - { error: "AuthRequired", message: "service-auth JWT or in-process principal required" }, - 401 - ); - } - callerDid = auth.issuer; - } - const resolved = await resolveTopicForCaller(communityTopic(actor), callerDid!, { - spaces: spacesCtx.adapter, - community - }); - if (!resolved.ok) { - const status = - resolved.error === "NotFound" ? 404 : - resolved.error === "Forbidden" ? 403 : 400; - return c.json({ error: resolved.error, reason: resolved.reason }, status); - } - const allowedSpaces = new Set<string>(); - for (const t of resolved.topics) { - const uri = parseSpaceTopic(t); - if (uri) allowedSpaces.add(uri); - } - scope = { kind: "actor", actor, topics: resolved.topics, allowedSpaces }; - scopeTopics = resolved.topics; - } - - const querySpec: TicketQuerySpec = ticketSpec ?? { - collection: colNsid, - ...(spaceUri ? { spaceUri } : { actor: actorParam! }), - ...(hydrateForSpec ? { hydrate: hydrateForSpec } : {}) - }; - - // Upgrade-to-WS path — forward directly to the DO with the spec, - // so the DO terminates the socket and hibernates when idle. - // Requires snapshot to be fetched separately (see `mode=ws` JSON - // handshake below) or accepted as lossy-on-connect for a plain WS - // upgrade. - const isUpgrade = c.req.header("Upgrade")?.toLowerCase() === "websocket"; - const isWsMode = params.get("mode") === "ws"; - - if (isWsMode && !isUpgrade) { - // Handshake: return snapshot + a ticket the client uses to - // upgrade. Ticket carries the (did, topics, querySpec) signed - // so the WS-upgrade route skips any other auth. - try { - const sinceTs = Date.now(); - const snapshotSpaces = - scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); - const result = await runPipeline( - db, - config, - collection, - params, - undefined, - snapshotSpaces, - c.req.raw.headers - ); - let ticket: string | undefined; - if (ticketSigner && callerDid) { - ticket = await ticketSigner.sign({ - topics: scopeTopics, - did: callerDid, - ttlMs: ticketTtl, - querySpec - }); - } - const wsUrl = (() => { - const u = new URL(c.req.url); - u.searchParams.delete("mode"); - if (ticket) u.searchParams.set("ticket", ticket); - u.searchParams.set("sinceTs", String(sinceTs)); - return u.pathname + u.search; - })(); - return c.json({ - transport: "ws", - snapshot: { records: result.records, cursor: result.cursor }, - querySpec, - ticket, - ticketTtlMs: ticketTtl, - sinceTs, - wsUrl - }); - } catch (err) { - return c.json( - { error: "SnapshotFailed", message: err instanceof Error ? err.message : String(err) }, - 500 - ); - } - } - - if (isUpgrade && pubsub instanceof DurableObjectPubSub && scope.kind === "space") { - // Forward the WS upgrade to the DO. The DO owns the socket from - // here and hibernates when idle. Replays any events buffered - // since the handshake `sinceTs` so the client closes the gap. - // - // Actor-scoped queries fall through to the worker-terminated - // path below — the DO binding is single-topic today; extending - // it to fan out over N topics is future work. - const sinceTsParam = params.get("sinceTs"); - const sinceTs = sinceTsParam ? Number(sinceTsParam) : 0; - return pubsub.forwardSubscribe(spaceTopic(scope.spaceUri), c.req.raw, { - did: callerDid, - querySpec: { - collection: querySpec.collection, - spaceUri: scope.spaceUri, - ...(querySpec.hydrate ? { hydrate: querySpec.hydrate } : {}) - }, - sinceTs: Number.isFinite(sinceTs) ? sinceTs : 0 - }); - } - - const ac = new AbortController(); - const reqSignal = c.req.raw.signal; - if (reqSignal) { - if (reqSignal.aborted) ac.abort(); - else reqSignal.addEventListener("abort", () => ac.abort(), { once: true }); - } - - // Worker-terminated WebSocket — used when pubsub isn't DO-backed - // (dev InMemoryPubSub). Same query-filter loop as SSE; different - // transport. Runs in the same isolate so no cost benefit, but - // matches the prod protocol. - if (isUpgrade) { - const WsPair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; - if (!WsPair) { - return c.json( - { error: "NotSupported", reason: "websockets-require-workers-runtime" }, - 426 - ); - } - const pair = new WsPair(); - const clientWs = pair[0] as WebSocket; - const serverWs = pair[1] as WebSocket & { accept?: () => void }; - serverWs.accept?.(); - - const sendWs = (kind: string, data: unknown) => { - try { - serverWs.send(JSON.stringify({ kind, data })); - } catch { - ac.abort(); - } - }; - serverWs.addEventListener?.("close", () => ac.abort()); - serverWs.addEventListener?.("error", () => ac.abort()); - - void runQueryStream({ - send: sendWs, - abort: ac, - scope, - callerDid, - params, - db, - config, - collection, - colNsid, - pubsub, - relations, - references, - childCollectionMap - }).finally(() => { - try { - serverWs.close(); - } catch { - /* ignore */ - } - }); - - return new Response(null, { - status: 101, - webSocket: clientWs - } as ResponseInit & { webSocket: unknown }); - } - - // SSE fallback. - const encoder = new TextEncoder(); - const stream = new ReadableStream<Uint8Array>({ - start(controller) { - let closed = false; - const close = () => { - if (closed) return; - closed = true; - try { - controller.close(); - } catch { - /* already closed */ - } - }; - ac.signal.addEventListener("abort", close, { once: true }); - - const send = (kind: string, data: unknown) => { - if (closed) return; - try { - controller.enqueue( - encoder.encode(`event: ${kind}\ndata: ${JSON.stringify(data)}\n\n`) - ); - } catch { - close(); - } - }; - - const keepalive = setInterval(() => { - if (closed) return; - try { - controller.enqueue(encoder.encode(`: keepalive\n\n`)); - } catch { - close(); - } - }, 15_000); - ac.signal.addEventListener( - "abort", - () => clearInterval(keepalive), - { once: true } - ); - - void runQueryStream({ - send, - abort: ac, - scope, - callerDid, - params, - db, - config, - collection, - colNsid, - pubsub, - relations, - references, - childCollectionMap - }).finally(() => close()); - }, - cancel() { - ac.abort(); - }, - }); - - return new Response(stream, { - status: 200, - headers: { - "content-type": "text/event-stream", - "cache-control": "no-cache, no-transform", - connection: "keep-alive", - "x-accel-buffering": "no", - }, - }); - }); - } - } - - if (!methods.includes("getRecord")) { - // Skip getRecord + custom queries unless listRecords-only was explicitly requested. - for (const [queryName, handler] of Object.entries(colConfig.queries ?? {})) { - app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { - const params = new URL(c.req.url).searchParams; - return handler(db, params, config); - }); - } - continue; - } - - app.get(`/xrpc/${ns}.${collection}.getRecord`, async (c) => { - const uri = c.req.query("uri"); - if (!uri) return c.json({ error: "uri parameter required" }, 400); - - // Spaces path — `?spaceUri=` routes to the per-space store + ACL gate. - const spaceUri = c.req.query("spaceUri") || undefined; - if (spaceUri) { - const gated = await gateSpaceAccess(c, spaceUri, "read"); - if (gated instanceof Response) return gated; - - // Parse author + rkey from the record uri `at://<did>/<collection>/<rkey>` - const m = uri.match(/^at:\/\/([^/]+)\/[^/]+\/([^/]+)$/); - if (!m) return c.json({ error: "InvalidRequest", message: "uri must be at://<did>/<collection>/<rkey>" }, 400); - const authorDid = m[1]; - const rkey = m[2]; - - const nsid = colConfig.collection; - const record = await spacesCtx!.adapter.getRecord(spaceUri, nsid, authorDid, rkey); - if (!record) return c.json({ error: "NotFound" }, 404); - return c.json({ record }); - } - - const relations = colConfig.relations ?? {}; - const references = colConfig.references ?? {}; - const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; - - const table = recordsTableName(collection); - const countCols = getRelationCountColumns(relations, relMap); - const selectCols = `uri, did, rkey, cid, record, time_us, indexed_at${countCols.length > 0 ? ", " + countCols.map(c => c.column).join(", ") : ""}`; - const row = await db - .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri = ?`) - .bind(uri) - .first<any>(); - - if (!row) return c.json({ error: "Record not found" }, 404); - - const nsid = nsidForShortName(config, collection) ?? collection; - const formatted = formatRecord({ ...row, collection: nsid }); - const counts = extractCounts(row, relations); - if (counts) flattenCounts(formatted, counts, relations); - - const params = new URL(c.req.url).searchParams; - const wantProfilesSingle = params.get("profiles") === "true"; - - const hydrateRequested = parseHydrateParams(params, relations, references); - const hydrates = await resolveHydrates( - db, - relations, - hydrateRequested.relations, - [row], - config - ); - const refs = await resolveReferences( - db, - references, - hydrateRequested.references, - [row], - config - ); - const h = hydrates[row.uri]; - if (h) { - for (const [relName, groups] of Object.entries(h)) { - (formatted as any)[relName] = groups; - } - } - const r = refs[row.uri]; - if (r) { - for (const [refName, record] of Object.entries(r)) { - (formatted as any)[refName] = record; - } - } - - const allDids = collectDids([row], hydrates); - const profileMap = wantProfilesSingle - ? await resolveProfiles(db, config, allDids) - : undefined; - - let labelersApplied: string[] | undefined; - if (config.labels) { - const sel = selectAcceptedLabelers( - c.req.raw.headers.get("atproto-accept-labelers"), - params.get("labelers"), - config.labels, - ); - if (sel.accepted.length > 0) { - const subjects: string[] = [row.uri, ...allDids]; - const cidByUri = new Map<string, string | null>([[row.uri, row.cid]]); - const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); - const ls = labelsByUri[row.uri]; - if (ls && ls.length > 0) (formatted as Record<string, unknown>).labels = ls; - if (profileMap) { - for (const entries of Object.values(profileMap)) { - for (const entry of entries) { - const els = labelsByUri[entry.did]; - if (els && els.length > 0) entry.labels = els; - } - } - } - labelersApplied = sel.accepted; - } - } - if (labelersApplied) { - c.header("atproto-content-labelers", labelersApplied.join(",")); - } - - return c.json({ - ...formatted, - ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), - }); - }); - - for (const [queryName, handler] of Object.entries( - colConfig.queries ?? {} - )) { - app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { - const params = new URL(c.req.url).searchParams; - return handler(db, params, config); - }); - } - - for (const [queryName, handler] of Object.entries( - colConfig.pipelineQueries ?? {} - )) { - app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { - const params = new URL(c.req.url).searchParams; - try { - const source = await handler(db, params, config); - const result = await runPipeline(db, config, collection, params, source, undefined, c.req.raw.headers); - return jsonWithLabelers(c, result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - }); - } - } -} - -function getRelationCountColumns( - relations: Record<string, RelationConfig>, - relMap: Record<string, any> -): { column: string }[] { - const cols: { column: string }[] = []; - for (const [relName, rel] of Object.entries(relations)) { - if (rel.count === false) continue; - cols.push({ column: countColumnName(rel.collection) }); - const mapping = relMap[relName]; - if (mapping?.groups) { - for (const groupKey of Object.keys(mapping.groups as Record<string, string>)) { - cols.push({ column: groupedCountColumnName(rel.collection, groupKey) }); - } - } - } - return cols; -} - -function extractCounts( - row: any, - relations: Record<string, any> -): Record<string, number> | undefined { - const counts: Record<string, number> = {}; - - for (const [, rel] of Object.entries(relations)) { - if (rel.count === false) continue; - const totalCol = countColumnName(rel.collection); - const val = row[totalCol]; - if (val != null && val !== 0) counts[rel.collection] = val; - - if (rel.groups) { - for (const [groupKey, fullToken] of Object.entries(rel.groups as Record<string, string>)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - const gval = row[groupCol]; - if (gval != null && gval !== 0) counts[fullToken] = gval; - } - } - } - - return Object.keys(counts).length > 0 ? counts : undefined; -} - -function flattenCounts( - formatted: FormattedRecord, - counts: Record<string, number> | undefined, - relations: Record<string, any> -): void { - if (!counts) return; - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - - const collectionToRelName: Record<string, string> = {}; - const tokenToField: Record<string, string> = {}; - for (const [relName, rel] of Object.entries(relations)) { - collectionToRelName[rel.collection] = relName; - if (rel.groups) { - for (const [shortName, fullToken] of Object.entries(rel.groups as Record<string, string>)) { - tokenToField[fullToken] = `${relName}${capitalize(shortName)}Count`; - } - } - } - - for (const [type, count] of Object.entries(counts)) { - if (collectionToRelName[type]) { - formatted[`${collectionToRelName[type]}Count`] = count; - } else if (tokenToField[type]) { - formatted[tokenToField[type]] = count; - } - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/feed.ts b/packages/contrail/src/core/router/feed.ts index 0430292..551c495 100644 --- a/packages/contrail/src/core/router/feed.ts +++ b/packages/contrail/src/core/router/feed.ts @@ -1,134 +1 @@ -import type { Hono } from "hono"; -import type { ContrailConfig, Database, FeedConfig } from "../types"; -import { getDialect } from "../dialect"; -import { DEFAULT_FEED_MAX_ITEMS, recordsTableName } from "../types"; -import { resolveActor } from "../identity"; -import { backfillUser } from "../backfill"; -import { runPipeline } from "./collection"; - -async function maybeBackfillFeed( - db: Database, - config: ContrailConfig, - actor: string, - feedName: string, - feedConfig: FeedConfig -): Promise<void> { - const status = await db - .prepare("SELECT completed FROM feed_backfills WHERE actor = ? AND feed = ?") - .bind(actor, feedName) - .first<{ completed: number }>(); - - if (status?.completed) return; - - // Ensure the user's follow records are backfilled first - await backfillUser(db, actor, feedConfig.follow, Date.now() + 3_000, config, { - maxRetries: 0, - requestTimeout: 3_000, - }); - - // Mark as in-progress (idempotent) - await db - .prepare( - "INSERT INTO feed_backfills (actor, feed, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" - ) - .bind(actor, feedName) - .run(); - - const maxItems = feedConfig.maxItems ?? DEFAULT_FEED_MAX_ITEMS; - - // Populate feed from existing records by followed users - const followTable = recordsTableName(feedConfig.follow); - for (const targetCol of feedConfig.targets) { - const targetTable = recordsTableName(targetCol); - await db - .prepare( - getDialect(db).insertOrIgnore( - `INSERT INTO feed_items (actor, uri, collection, time_us) - SELECT ?, r.uri, ?, r.time_us - FROM ${targetTable} r - WHERE r.did IN ( - SELECT ${getDialect(db).jsonExtract('f.record', 'subject')} - FROM ${followTable} f - WHERE f.did = ? - ) - ORDER BY r.time_us DESC - LIMIT ?` - ) - ) - .bind(actor, targetCol, actor, maxItems) - .run(); - } - - // Prune oldest items beyond the cap - await db - .prepare( - `DELETE FROM feed_items WHERE actor = ? AND uri NOT IN ( - SELECT uri FROM feed_items WHERE actor = ? ORDER BY time_us DESC LIMIT ? - )` - ) - .bind(actor, actor, maxItems) - .run(); - - await db - .prepare("UPDATE feed_backfills SET completed = 1 WHERE actor = ? AND feed = ?") - .bind(actor, feedName) - .run(); -} - -export function registerFeedRoutes( - app: Hono, - db: Database, - config: ContrailConfig -): void { - if (!config.feeds) return; - - const ns = config.namespace; - - app.get(`/xrpc/${ns}.getFeed`, async (c) => { - const params = new URL(c.req.url).searchParams; - const feedName = params.get("feed"); - const actor = params.get("actor"); - - if (!feedName || !actor) { - return c.json({ error: "feed and actor parameters required" }, 400); - } - - const feedConfig = config.feeds![feedName]; - if (!feedConfig) { - return c.json({ error: "Unknown feed" }, 404); - } - - const did = await resolveActor(db, actor); - if (!did) return c.json({ error: "Could not resolve actor" }, 400); - - await maybeBackfillFeed(db, config, did, feedName, feedConfig); - - const collection = params.get("collection") || feedConfig.targets[0]; - if (!feedConfig.targets.includes(collection)) { - return c.json({ error: "Collection not in feed targets" }, 400); - } - - // Strip feed-specific params so runPipeline doesn't misinterpret them - // (e.g. "actor" in feeds means "whose feed", not "filter by record creator") - const pipelineParams = new URLSearchParams(params); - pipelineParams.delete("feed"); - pipelineParams.delete("actor"); - pipelineParams.delete("collection"); - - const source = { - joins: "JOIN feed_items f ON r.uri = f.uri", - conditions: ["f.actor = ?"], - params: [did], - }; - - try { - const result = await runPipeline(db, config, collection, pipelineParams, source); - return c.json(result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - }); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/helpers.ts b/packages/contrail/src/core/router/helpers.ts index 7ff372e..551c495 100644 --- a/packages/contrail/src/core/router/helpers.ts +++ b/packages/contrail/src/core/router/helpers.ts @@ -1,67 +1 @@ -import type { Database, RecordRow } from "../types"; - -export interface FormattedRecord { - uri: string; - cid: string | null; - value: unknown; - did: string; - collection: string; - rkey: string; - time_us: number; - [key: string]: unknown; -} - -export function formatRecord(row: RecordRow): FormattedRecord { - let value: unknown = null; - if (row.record) { - try { - value = JSON.parse(row.record); - } catch { - value = row.record; - } - } - return { - uri: row.uri, - cid: row.cid, - value, - did: row.did, - collection: row.collection, - rkey: row.rkey, - time_us: row.time_us, - ...(row.space ? { space: row.space } : {}), - }; -} - -export function parseIntParam( - value: string | null | undefined, - defaultValue?: number -): number | undefined { - if (!value) return defaultValue; - const parsed = parseInt(value, 10); - return isNaN(parsed) ? defaultValue : parsed; -} - -export function fieldToParam(field: string): string { - return field.replace(/\.(\w)/g, (_, c) => c.toUpperCase()); -} - -const BATCH_SIZE = 50; - -export async function batchedInQuery<T>( - db: Database, - sql: string, - prefixBindings: (string | number)[], - inValues: string[] -): Promise<T[]> { - const results: T[] = []; - for (let i = 0; i < inValues.length; i += BATCH_SIZE) { - const chunk = inValues.slice(i, i + BATCH_SIZE); - const query = sql.replace("__IN__", chunk.map(() => "?").join(",")); - const rows = await db - .prepare(query) - .bind(...prefixBindings, ...chunk) - .all<T>(); - results.push(...(rows.results ?? [])); - } - return results; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/hydrate.ts b/packages/contrail/src/core/router/hydrate.ts index 8c1ae63..551c495 100644 --- a/packages/contrail/src/core/router/hydrate.ts +++ b/packages/contrail/src/core/router/hydrate.ts @@ -1,223 +1 @@ -import type { RelationConfig, ReferenceConfig, RecordRow, Database, ContrailConfig } from "../types"; -import { getDialect } from "../dialect"; -import { - getNestedValue, - getRelationField, - recordsTableName, - spacesRecordsTableName, - nsidForShortName, -} from "../types"; -import { batchedInQuery, formatRecord } from "./helpers"; - -/** Group rows by their origin: public (undefined key) or a specific spaceUri. */ -function groupBySource<T extends { space?: string }>(rows: T[]): Map<string | undefined, T[]> { - const groups = new Map<string | undefined, T[]>(); - for (const r of rows) { - const key = r.space; - const g = groups.get(key); - if (g) g.push(r); - else groups.set(key, [r]); - } - return groups; -} - -// --- Hydration: embed related records --- - -export function parseHydrateParams( - params: URLSearchParams, - relations: Record<string, RelationConfig>, - references: Record<string, ReferenceConfig> -): { relations: Record<string, number>; references: Set<string> } { - const relHydrates: Record<string, number> = {}; - const refHydrates = new Set<string>(); - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - - for (const relName of Object.keys(relations)) { - const val = params.get(`hydrate${capitalize(relName)}`); - if (val) { - const limit = parseInt(val, 10); - if (!isNaN(limit) && limit > 0) { - relHydrates[relName] = Math.min(limit, 50); - } - } - } - - for (const refName of Object.keys(references)) { - const val = params.get(`hydrate${capitalize(refName)}`); - if (val === "true" || val === "1") { - refHydrates.add(refName); - } - } - - return { relations: relHydrates, references: refHydrates }; -} - -// Per-relation hydrate result: array for ungrouped, Record<group, array> for grouped -export type HydrateResult = Record<string, Record<string, any[] | Record<string, any[]>>>; - -export async function resolveHydrates( - db: Database, - relations: Record<string, RelationConfig>, - requested: Record<string, number>, - records: RecordRow[], - config?: ContrailConfig -): Promise<HydrateResult> { - if (Object.keys(requested).length === 0 || records.length === 0) return {}; - - const grouped: Record<string, Record<string, Record<string, any[]>>> = {}; - - const sourceGroups = groupBySource(records); - - for (const [relName, hydrateLimit] of Object.entries(requested)) { - const rel = relations[relName]; - const field = getRelationField(rel); - const matchMode = rel.match ?? "uri"; - - for (const [sourceSpace, sourceRecords] of sourceGroups) { - const matchValues = matchMode === "did" - ? [...new Set(sourceRecords.map((r) => r.did))] - : sourceRecords.map((r) => r.uri); - - if (matchValues.length === 0) continue; - - const groupCount = rel.groupBy ? 10 : 1; - const maxRows = matchValues.length * hydrateLimit * groupCount; - - const table = sourceSpace - ? spacesRecordsTableName(rel.collection) - : recordsTableName(rel.collection); - const where = sourceSpace - ? `space_uri = ? AND ${getDialect(db).jsonExtract('record', field)} IN (__IN__)` - : `${getDialect(db).jsonExtract('record', field)} IN (__IN__)`; - const prefix = sourceSpace ? [sourceSpace] : []; - - const relatedRows = await batchedInQuery<Omit<RecordRow, "collection">>( - db, - `SELECT uri, did, rkey, record, time_us FROM ${table} - WHERE ${where} - ORDER BY time_us DESC - LIMIT ${maxRows}`, - prefix, - matchValues - ); - - for (const row of relatedRows) { - const record = row.record ? JSON.parse(row.record) : null; - const matchedValue = getNestedValue(record, field); - if (!matchedValue) continue; - - const parentUris = matchMode === "did" - ? sourceRecords.filter((r) => r.did === matchedValue).map((r) => r.uri) - : [matchedValue]; - - const groupValue = rel.groupBy - ? String(getNestedValue(record, rel.groupBy) ?? "other") - : "_flat"; - - for (const parentUri of parentUris) { - const targetUri = matchMode === "did" ? parentUri : matchedValue; - - if (!grouped[targetUri]) grouped[targetUri] = {}; - if (!grouped[targetUri][relName]) grouped[targetUri][relName] = {}; - if (!grouped[targetUri][relName][groupValue]) grouped[targetUri][relName][groupValue] = []; - - const group = grouped[targetUri][relName][groupValue]; - if (group.length < hydrateLimit) { - const childNsid = config - ? nsidForShortName(config, rel.collection) ?? rel.collection - : rel.collection; - group.push( - formatRecord({ - ...(row as any), - collection: childNsid, - ...(sourceSpace ? { space: sourceSpace } : {}), - } as RecordRow) - ); - } - } - } - } - } - - const result: HydrateResult = {}; - for (const [uri, rels] of Object.entries(grouped)) { - result[uri] = {}; - for (const [relName, groups] of Object.entries(rels)) { - if (relations[relName].groupBy) { - result[uri][relName] = groups; - } else { - result[uri][relName] = groups["_flat"] ?? []; - } - } - } - - return result; -} - -// --- References: embed records that our records point at --- - -export type ReferenceResult = Record<string, Record<string, any>>; - -export async function resolveReferences( - db: Database, - references: Record<string, ReferenceConfig>, - requested: Set<string>, - records: RecordRow[], - config?: ContrailConfig -): Promise<ReferenceResult> { - if (requested.size === 0 || records.length === 0) return {}; - - const result: ReferenceResult = {}; - - const sourceGroups = groupBySource(records); - - for (const refName of requested) { - const ref = references[refName]; - if (!ref) continue; - - const refNsid = config - ? nsidForShortName(config, ref.collection) ?? ref.collection - : ref.collection; - - for (const [sourceSpace, sourceRecords] of sourceGroups) { - const targetMap = new Map<string, string[]>(); - for (const r of sourceRecords) { - const parsed = r.record ? JSON.parse(r.record) : null; - const targetValue = parsed ? getNestedValue(parsed, ref.field) : null; - if (!targetValue) continue; - if (!targetMap.has(targetValue)) targetMap.set(targetValue, []); - targetMap.get(targetValue)!.push(r.uri); - } - - const targetUris = [...targetMap.keys()]; - if (targetUris.length === 0) continue; - - const table = sourceSpace - ? spacesRecordsTableName(ref.collection) - : recordsTableName(ref.collection); - const where = sourceSpace ? `space_uri = ? AND uri IN (__IN__)` : `uri IN (__IN__)`; - const prefix = sourceSpace ? [sourceSpace] : []; - - const rows = await batchedInQuery<Omit<RecordRow, "collection">>( - db, - `SELECT uri, did, rkey, record, time_us FROM ${table} WHERE ${where}`, - prefix, - targetUris - ); - - for (const row of rows) { - const parentUris = targetMap.get(row.uri) ?? []; - for (const parentUri of parentUris) { - if (!result[parentUri]) result[parentUri] = {}; - result[parentUri][refName] = formatRecord({ - ...(row as any), - collection: refNsid, - ...(sourceSpace ? { space: sourceSpace } : {}), - } as RecordRow); - } - } - } - } - - return result; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 0c5e058..551c495 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -1,247 +1 @@ -import { Hono } from "hono"; -import { cors } from "hono/cors"; -import type { Database, ContrailConfig } from "../types"; -import { normalizeProfileConfig } from "../types"; -import { registerAdminRoutes } from "./admin"; -import { registerCollectionRoutes } from "./collection"; -import { registerFeedRoutes } from "./feed"; -import { registerNotifyRoute } from "./notify"; -import { registerSpacesRoutes } from "../spaces/router"; -import type { SpacesRoutesOptions } from "../spaces/router"; -import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; -import { HostedAdapter } from "../spaces/adapter"; -import type { StorageAdapter } from "../spaces/types"; -import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; -import type { CommunityIntegration } from "../community-integration"; -import { registerRealtimeRoutes } from "../realtime/router"; -import type { RealtimeRoutesOptions } from "../realtime/router"; -import { registerInviteRoutes } from "../invite/router"; -import { InMemoryPubSub } from "../realtime/in-memory"; -import { wrapWithPublishing } from "../realtime/publishing-adapter"; -import type { PubSub } from "../realtime/types"; -import { resolveActor } from "../identity"; -import { resolveProfiles } from "./profiles"; -import { backfillUser } from "../backfill"; -import { selectAcceptedLabelers } from "../labels/select"; -import { hydrateLabels } from "../labels/hydrate"; -import type { MiddlewareHandler } from "hono"; - -export interface SpacesContext { - adapter: StorageAdapter; - verifier: ServiceJwtVerifier; -} - -export interface CreateAppOptions { - spaces?: SpacesRoutesOptions; - /** Pre-built community integration. Construct via the community package's - * `createCommunityIntegration({ ... })`. When set, contrail wires - * community whoami extension, invite handler, route registration, etc. - * When omitted, deployment runs without community features. */ - community?: CommunityIntegration | null; - /** Auth middleware override for community routes (rare — mostly for tests). */ - communityAuthMiddleware?: MiddlewareHandler; - realtime?: Partial<RealtimeRoutesOptions>; - /** Separate DB for the spaces tables. Defaults to `db`. */ - spacesDb?: Database; - /** Full spaces context override (escape hatch for tests). */ - spacesCtx?: SpacesContext | null; - /** Lexicon JSONs to serve at `/lexicons` so consumer apps can fetch + - * typegen against this deployment. Emit with `contrail-lex generate` — - * its `lexicons/generated/index.ts` exports the right shape. If omitted, - * the endpoint returns `404`. */ - lexicons?: object[]; -} - -export function createApp( - db: Database, - config: ContrailConfig, - options: CreateAppOptions = {} -): Hono { - const app = new Hono(); - app.use("*", cors()); - - app.get("/", (c) => c.json({ status: "ok" })); - app.get("/health", (c) => c.json({ status: "ok" })); - app.get("/xrpc/_health", (c) => c.json({ status: "ok" })); - - const ns = config.namespace; - - // Lexicon manifest — lets consumer apps fetch every lexicon this - // deployment speaks (generated + pulled + custom) over HTTP and - // typegen clients, without needing a PDS or DNS resolution. Only - // registered when the caller passed bundled lexicons at build time - // via `contrail-lex generate`. - if (options.lexicons && options.lexicons.length > 0) { - const lexicons = options.lexicons; - app.get(`/xrpc/${ns}.lexicons`, (c) => c.json({ lexicons })); - } - - app.get(`/xrpc/${ns}.getProfile`, async (c) => { - const actor = c.req.query("actor"); - if (!actor) return c.json({ error: "actor parameter required" }, 400); - - const did = await resolveActor(db, actor); - if (!did) return c.json({ error: "Could not resolve actor" }, 400); - - // Ensure profile records are backfilled - const profileConfigs = (config.profiles ?? []).map(normalizeProfileConfig); - for (const pc of profileConfigs) { - await backfillUser(db, did, pc.collection, Date.now() + 3_000, config, { - maxRetries: 0, - requestTimeout: 3_000, - }); - } - - const profileMap = await resolveProfiles(db, config, [did]); - const profiles = profileMap[did]; - if (!profiles || profiles.length === 0) return c.json({ error: "Profile not found" }, 404); - - if (config.labels) { - const params = new URL(c.req.url).searchParams; - const sel = selectAcceptedLabelers( - c.req.raw.headers.get("atproto-accept-labelers"), - params.get("labelers"), - config.labels, - ); - if (sel.accepted.length > 0) { - const labelsByUri = await hydrateLabels(db, [did], sel.accepted); - const ls = labelsByUri[did]; - if (ls && ls.length > 0) { - for (const entry of profiles) { - entry.labels = ls; - } - } - c.header("atproto-content-labelers", sel.accepted.join(",")); - } - } - - return c.json({ profiles }); - }); - - // Shared spaces context — verifier + adapter — reused by both the per-collection - // routes (for `?spaceUri=...` dispatch) and the `<ns>.space.*` routes. - // Built when an authority is configured (spaces are gated on the authority, - // not the record host — a record-host-only deployment still needs an - // authority somewhere, just possibly external). - const spacesDb = options.spacesDb ?? db; - let spacesCtx: SpacesContext | null = - options.spacesCtx !== undefined - ? options.spacesCtx - : config.spaces?.authority - ? { - adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), - verifier: buildVerifier(config.spaces.authority), - } - : null; - - // Community is provided as a pre-built integration — contrail core never - // imports from the community package. The integration object is opaque; - // we just pass through its probe / whoamiExtension / inviteHandler / - // registerRoutes hooks at the right wiring points. - const community = options.community ?? null; - - // Realtime pubsub is built whenever realtime is configured — independent of - // spaces. With spaces, the spaces adapter is wrapped so private record/member - // events publish to space:/community: topics. Without spaces, only public - // topics (collection:/actor:) see traffic — those are published from - // applyEvents (jetstream ingestion), not from here. - let realtimePubsub: PubSub | null = null; - if (config.realtime) { - realtimePubsub = - options.realtime?.pubsub ?? config.realtime.pubsub ?? new InMemoryPubSub({ - queueBound: config.realtime.queueBound, - }); - if (spacesCtx) { - const isCommunityDid = community - ? cachedIsCommunityDid(community.probe) - : undefined; - spacesCtx = { - ...spacesCtx, - adapter: wrapWithPublishing(spacesCtx.adapter, realtimePubsub, { isCommunityDid }), - }; - } - } - - registerAdminRoutes(app, db, config); - - registerCollectionRoutes(app, db, config, spacesCtx, { - pubsub: realtimePubsub, - community: community?.probe ?? null, - }); - registerFeedRoutes(app, db, config); - registerNotifyRoute(app, db, config); - - // Spaces routes — get a whoami extension from the community integration - // when one's wired so community-owned spaces get an `accessLevel` field. - const spacesOptions = { - ...options.spaces, - whoamiExtension: - options.spaces?.whoamiExtension ?? community?.whoamiExtension, - }; - registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); - - if (community && spacesCtx) { - // Community routes reuse the spaces service-auth middleware (same JWT verifier). - const authMiddleware = - options.communityAuthMiddleware ?? - options.spaces?.authMiddleware ?? - createServiceAuthMiddleware(spacesCtx.verifier); - community.registerRoutes(app, { authMiddleware }); - } - - if (config.spaces?.authority && spacesCtx) { - // Unified invite surface: one `<ns>.invite.*` family that dispatches on - // space ownership (user-owned → addMember; community-owned → grant via - // the integration's invite handler). - const authMiddleware = - options.spaces?.authMiddleware ?? - createServiceAuthMiddleware(spacesCtx.verifier); - registerInviteRoutes( - app, - config, - spacesCtx.adapter, - community?.inviteHandler ?? null, - { authMiddleware } - ); - } - - if (config.realtime && realtimePubsub) { - // The ticket endpoint still needs a JWT verifier — but that verifier only - // exists when spaces is configured. Without spaces, private-topic ticket - // minting simply isn't offered; public subscriptions (collection:/actor:) - // require no auth and still work. - const authMiddleware = spacesCtx - ? options.realtime?.authMiddleware ?? - options.spaces?.authMiddleware ?? - createServiceAuthMiddleware(spacesCtx.verifier) - : null; - registerRealtimeRoutes( - app, - config, - spacesCtx?.adapter ?? null, - community?.probe ?? null, - { - authMiddleware, - pubsub: realtimePubsub, - } - ); - } - - return app; -} - -function cachedIsCommunityDid( - probe: import("../community-integration").CommunityProbe -): (did: string) => Promise<boolean> { - const TTL = 60_000; - const cache = new Map<string, { value: boolean; expires: number }>(); - return async (did: string) => { - const now = Date.now(); - const hit = cache.get(did); - if (hit && hit.expires > now) return hit.value; - const row = await probe.getCommunity(did); - const value = row != null; - cache.set(did, { value, expires: now + TTL }); - return value; - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/notify.ts b/packages/contrail/src/core/router/notify.ts index 5ff630b..551c495 100644 --- a/packages/contrail/src/core/router/notify.ts +++ b/packages/contrail/src/core/router/notify.ts @@ -1,179 +1 @@ -import type { Hono } from "hono"; -import type { Database, ContrailConfig, IngestEvent } from "../types"; -import { shortNameForNsid } from "../types"; -import { applyEvents, lookupExistingRecords } from "../db/records"; -import { getPDS } from "../client"; -import type { Did } from "@atcute/lexicons"; - -/** Parse an AT URI into its components. */ -export function parseAtUri(uri: string): { did: string; collection: string; rkey: string } | null { - const match = uri.match(/^at:\/\/(did:[^/]+)\/([^/]+)\/([^/]+)$/); - if (!match) return null; - return { did: match[1], collection: match[2], rkey: match[3] }; -} - -/** - * Fetch a single record from the user's PDS. - * Returns the record + cid on success, null if not found. - */ -async function fetchRecordFromPDS( - pds: string, - did: string, - collection: string, - rkey: string -): Promise<{ value: unknown; cid: string } | null> { - const url = new URL(`/xrpc/com.atproto.repo.getRecord`, pds); - url.searchParams.set("repo", did); - url.searchParams.set("collection", collection); - url.searchParams.set("rkey", rkey); - - const res = await fetch(url.toString()); - if (!res.ok) return null; - - const data = (await res.json()) as { value?: unknown; cid?: string }; - if (!data.value || !data.cid) return null; - return { value: data.value, cid: data.cid }; -} - -export interface NotifyResult { - indexed: number; - deleted: number; - errors?: string[]; -} - -/** - * Process notify URIs: fetch from PDS, detect changes, apply events. - * Shared by both the Hono route and the Contrail.notify() method. - */ -export async function processNotifyUris( - db: Database, - config: ContrailConfig, - uris: string[] -): Promise<NotifyResult> { - const events: IngestEvent[] = []; - const errors: string[] = []; - - // Validate and parse all URIs first - const validUris: { uri: string; parsed: { did: string; collection: string; rkey: string } }[] = []; - for (const uri of uris) { - const parsed = parseAtUri(uri); - if (!parsed) { - errors.push(`invalid AT URI: ${uri}`); - continue; - } - // `parsed.collection` is an NSID; look up the matching short name. - if (!shortNameForNsid(config, parsed.collection)) { - errors.push(`collection not tracked: ${parsed.collection}`); - continue; - } - validUris.push({ uri, parsed }); - } - - // Single batch lookup for all existing records (cid + record in one query) - const existing = await lookupExistingRecords( - db, - validUris.map(({ uri, parsed }) => ({ uri, collection: parsed.collection })), - true, - config - ); - - for (const { uri, parsed } of validUris) { - const pds = await getPDS(parsed.did as Did, db); - if (!pds) { - errors.push(`could not resolve PDS for ${parsed.did}`); - continue; - } - - const result = await fetchRecordFromPDS( - pds, - parsed.did, - parsed.collection, - parsed.rkey - ); - - const now = Date.now() * 1000; // microseconds - const existingInfo = existing.get(uri); - - if (result) { - if (existingInfo?.cid === result.cid) { - // Same CID — nothing changed - continue; - } - - events.push({ - uri, - did: parsed.did, - collection: parsed.collection, - rkey: parsed.rkey, - operation: existingInfo ? "update" : "create", - cid: result.cid, - record: JSON.stringify(result.value), - time_us: now, - indexed_at: now, - }); - } else if (existingInfo) { - // Record gone from PDS but exists locally — delete it. - events.push({ - uri, - did: parsed.did, - collection: parsed.collection, - rkey: parsed.rkey, - operation: "delete", - cid: null, - record: existingInfo.record, - time_us: now, - indexed_at: now, - }); - } - } - - if (events.length > 0) { - // Pass pre-fetched existing records so applyEvents skips re-querying - await applyEvents(db, events, config, { existing }); - } - - return { - indexed: events.filter((e) => e.operation === "create" || e.operation === "update").length, - deleted: events.filter((e) => e.operation === "delete").length, - errors: errors.length > 0 ? errors : undefined, - }; -} - -export function registerNotifyRoute( - app: Hono, - db: Database, - config: ContrailConfig -) { - // Endpoint is off by default. Set config.notify to true or a secret string to enable. - if (!config.notify) return; - - const ns = config.namespace; - const secret = typeof config.notify === "string" ? config.notify : null; - - app.post(`/xrpc/${ns}.notifyOfUpdate`, async (c) => { - if (secret) { - const auth = c.req.header("Authorization"); - if (auth !== `Bearer ${secret}`) { - return c.json({ error: "unauthorized" }, 401); - } - } - - const body = await c.req.json<{ uri?: string; uris?: string[] }>().catch(() => null); - const uris: string[] = []; - - if (body?.uris && Array.isArray(body.uris)) { - uris.push(...body.uris); - } else if (body?.uri) { - uris.push(body.uri); - } else { - return c.json({ error: "uri or uris required" }, 400); - } - - if (uris.length > 25) { - return c.json({ error: "max 25 URIs per request" }, 400); - } - - const result = await processNotifyUris(db, config, uris); - return c.json(result); - }); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/profiles.ts b/packages/contrail/src/core/router/profiles.ts index 5552357..551c495 100644 --- a/packages/contrail/src/core/router/profiles.ts +++ b/packages/contrail/src/core/router/profiles.ts @@ -1,181 +1 @@ -import type { Database, ContrailConfig, RecordRow, ProfileConfig } from "../types"; -import { recordsTableName, normalizeProfileConfig } from "../types"; -import { resolveIdentities } from "../identity"; -import { getPDS } from "../client"; -import type { Did } from "@atcute/lexicons"; -import { batchedInQuery } from "./helpers"; - -export interface ProfileEntry { - did: string; - handle: string | null; - uri?: string; - cid?: string | null; - value?: unknown; - collection?: string; - rkey?: string; - /** Hydrated by the labels module when the caller has accepted-labelers - * active and there are matching labels on this DID. */ - labels?: unknown; -} - -export function collectDids( - records: RecordRow[], - hydrates: Record<string, Record<string, any[] | Record<string, any[]>>> -): string[] { - const dids = new Set(records.map((r) => r.did)); - for (const rels of Object.values(hydrates)) { - for (const value of Object.values(rels)) { - const items = Array.isArray(value) - ? value - : Object.values(value).flat(); - for (const item of items) { - if (item.did) dids.add(item.did); - } - } - } - return [...dids]; -} - -export async function resolveProfiles( - db: Database, - config: ContrailConfig, - dids: string[] -): Promise<Record<string, ProfileEntry[]>> { - if (dids.length === 0 || !config.profiles || config.profiles.length === 0) { - return {}; - } - - const profileConfigs = config.profiles.map(normalizeProfileConfig); - const result: Record<string, ProfileEntry[]> = {}; - - // Batch-lookup profile records for each configured profile collection - for (const pc of profileConfigs) { - const { collection, rkey: configRkey, shortName } = pc; - const rkey = configRkey ?? "self"; - const table = recordsTableName(shortName ?? collection); - const uris = dids.map((did) => `at://${did}/${collection}/${rkey}`); - - const rows = await batchedInQuery<Omit<RecordRow, "collection">>( - db, - `SELECT uri, did, rkey, cid, record FROM ${table} WHERE uri IN (__IN__)`, - [], - uris - ); - - for (const row of rows) { - let value: unknown = null; - if (row.record) { - try { - value = JSON.parse(row.record); - } catch { - value = row.record; - } - } - if (!result[row.did]) result[row.did] = []; - result[row.did].push({ - did: row.did, - handle: null, // filled below - uri: row.uri, - collection, - rkey: row.rkey, - cid: row.cid, - value, - }); - } - } - - // Resolve identities for all DIDs - const identities = await resolveIdentities(db, dids); - - // Fetch missing profile records from PDS on demand - const missingDids = dids.filter((d) => !result[d]); - if (missingDids.length > 0 && profileConfigs.length > 0) { - const fetched = await fetchMissingProfiles(db, config, missingDids); - for (const [did, entries] of Object.entries(fetched)) { - if (!result[did]) result[did] = []; - result[did].push(...entries); - } - } - - // Fill in handles and create entries for DIDs without profile records - for (const did of dids) { - const identity = identities.get(did); - const handle = identity?.handle ?? null; - - if (result[did]) { - for (const entry of result[did]) { - entry.handle = handle; - } - } else { - result[did] = [{ did, handle }]; - } - } - - return result; -} - -/** - * Fetch profile records from PDS for DIDs not yet in the index. - * Fetches in parallel across all configured profile collections, - * indexes the results into D1 for future requests. - */ -async function fetchMissingProfiles( - db: Database, - config: ContrailConfig, - dids: string[] -): Promise<Record<string, ProfileEntry[]>> { - const result: Record<string, ProfileEntry[]> = {}; - const profileConfigs = config.profiles!.map(normalizeProfileConfig); - - await Promise.all( - dids.flatMap((did) => - profileConfigs.map(async (pc) => { - const { collection, rkey: configRkey, shortName } = pc; - const rkey = configRkey ?? "self"; - const table = recordsTableName(shortName ?? collection); - try { - const pds = await getPDS(did as Did, db); - if (!pds) return; - - const url = new URL("/xrpc/com.atproto.repo.getRecord", pds); - url.searchParams.set("repo", did); - url.searchParams.set("collection", collection); - url.searchParams.set("rkey", rkey); - - const res = await fetch(url.toString()); - if (!res.ok) return; - - const data = (await res.json()) as { uri?: string; value?: unknown; cid?: string }; - if (!data.value || !data.cid) return; - - const uri = data.uri ?? `at://${did}/${collection}/${rkey}`; - const record = data.value; - const cid = data.cid; - - // Index into D1 for future requests - await db - .prepare( - `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, indexed_at = excluded.indexed_at` - ) - .bind(uri, did, rkey, cid, JSON.stringify(record), Date.now() * 1000, Date.now()) - .run(); - - if (!result[did]) result[did] = []; - result[did].push({ - did, - handle: null, - uri, - collection, - rkey, - cid, - value: record, - }); - } catch { - // Skip failures silently - } - }) - ) - ); - - return result; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/search.ts b/packages/contrail/src/core/search.ts index a33dc5e..551c495 100644 --- a/packages/contrail/src/core/search.ts +++ b/packages/contrail/src/core/search.ts @@ -1,31 +1 @@ -import type { CollectionConfig } from "./types"; -import { getNestedValue } from "./types"; - -/** - * Resolve which fields are searchable for a collection. - * Returns null if search is disabled or no fields found. - */ -export function getSearchableFields( - collection: string, - colConfig: CollectionConfig -): string[] | null { - if (!Array.isArray(colConfig.searchable)) return null; - return colConfig.searchable.length > 0 ? colConfig.searchable : null; -} - -/** Sanitized FTS table name for a collection. */ -export function ftsTableName(collection: string): string { - return `fts_${collection.replace(/[^a-zA-Z0-9]/g, "_")}`; -} - -/** Extract searchable field values from a record and join them into a single string. */ -export function buildFtsContent(record: unknown, fields: string[]): string | null { - const parts: string[] = []; - for (const field of fields) { - const value = getNestedValue(record, field); - if (typeof value === "string" && value.length > 0) { - parts.push(value); - } - } - return parts.length > 0 ? parts.join(" ") : null; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/acl.ts b/packages/contrail/src/core/spaces/acl.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/acl.ts +++ b/packages/contrail/src/core/spaces/acl.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/adapter.ts b/packages/contrail/src/core/spaces/adapter.ts index 50fb33d..551c495 100644 --- a/packages/contrail/src/core/spaces/adapter.ts +++ b/packages/contrail/src/core/spaces/adapter.ts @@ -1,505 +1 @@ -/** Contrail's all-in-one default adapter — extends the authority package's - * {@link HostedAuthorityAdapter} (which owns space lifecycle, member list, - * invites) and adds the record-host methods (records, blobs, enrollment). - * - * Phase 7a step 3 will lift the record-host methods into a separate - * HostedRecordHostAdapter, at which point this class becomes a thin - * composition / re-export. For now we keep both roles in one class so - * consumers can wire a single object that satisfies the full StorageAdapter. */ - -import type { ContrailConfig, RelationConfig, ResolvedContrailConfig } from "../types"; -import { - shortNameForNsid, - spacesRecordsTableName, - countColumnName, - groupedCountColumnName, - getRelationField, - getNestedValue, -} from "../types"; -import { getDialect } from "../dialect"; -import type { - BlobMetaRow, - CollectionCount, - EnrollmentRow, - ListBlobsOptions, - ListBlobsResult, - ListOptions, - ListResult, - StorageAdapter, - StoredRecord, -} from "./types"; -import type { Database } from "../types"; -import { buildRecordUri } from "./uri"; -import { HostedAuthorityAdapter, parseJson, toNum } from "@atmo-dev/contrail-authority"; - -function mapBlobMetaRow(row: any): BlobMetaRow { - return { - spaceUri: row.space_uri, - cid: row.cid, - mimeType: row.mime_type, - size: Number(row.size), - authorDid: row.author_did, - createdAt: toNum(row.created_at), - }; -} - -function mapEnrollmentRow(row: any): EnrollmentRow { - return { - spaceUri: row.space_uri, - authorityDid: row.authority_did, - enrolledAt: toNum(row.enrolled_at), - enrolledBy: row.enrolled_by, - }; -} - -/** Row mapper for per-collection spaces_records_<short> tables. - * `collection` is injected by the caller (known from the table name). */ -function mapRecordRow(row: any, collection: string): StoredRecord { - return { - spaceUri: row.space_uri, - collection, - authorDid: row.did, - rkey: row.rkey, - cid: row.cid ?? null, - record: parseJson<Record<string, unknown>>(row.record) ?? {}, - createdAt: toNum(row.time_us), - }; -} - -export class HostedAdapter extends HostedAuthorityAdapter implements StorageAdapter { - /** Resolve the per-collection spaces table name, or throw if the collection - * isn't configured (and therefore has no table). */ - private tableFor(collection: string): string { - if (!this.config) { - throw new Error( - `HostedAdapter: config not provided; cannot resolve table for collection ${collection}` - ); - } - const short = shortNameForNsid(this.config, collection); - if (!short) { - throw new Error( - `HostedAdapter: collection ${collection} is not configured in this deployment` - ); - } - return spacesRecordsTableName(short); - } - - // ---- Enrollment ---- - - async enroll(input: EnrollmentRow): Promise<void> { - await this.db - .prepare( - `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) - VALUES (?, ?, ?, ?) - ON CONFLICT (space_uri) DO UPDATE SET - authority_did = excluded.authority_did, - enrolled_at = excluded.enrolled_at, - enrolled_by = excluded.enrolled_by` - ) - .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) - .run(); - } - - async getEnrollment(spaceUri: string): Promise<EnrollmentRow | null> { - const row = await this.db - .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) - .bind(spaceUri) - .first<any>(); - return row ? mapEnrollmentRow(row) : null; - } - - async listEnrollments( - options: { authorityDid?: string; limit?: number } = {} - ): Promise<EnrollmentRow[]> { - const limit = Math.min(options.limit ?? 200, 1000); - if (options.authorityDid) { - const { results } = await this.db - .prepare( - `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` - ) - .bind(options.authorityDid, limit) - .all<any>(); - return results.map(mapEnrollmentRow); - } - const { results } = await this.db - .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) - .bind(limit) - .all<any>(); - return results.map(mapEnrollmentRow); - } - - async removeEnrollment(spaceUri: string): Promise<void> { - await this.db - .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) - .bind(spaceUri) - .run(); - } - - // ---- Records ---- - - async putRecord(record: StoredRecord): Promise<void> { - const table = this.tableFor(record.collection); - const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); - - const childShort = this.config ? shortNameForNsid(this.config, record.collection) : null; - const prev = childShort - ? await this.db - .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) - .bind(record.spaceUri, record.authorDid, record.rkey) - .first<{ record: unknown } | null>() - : null; - const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); - - await this.db - .prepare( - `INSERT INTO ${table} (space_uri, uri, did, rkey, cid, record, time_us, indexed_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (space_uri, did, rkey) DO UPDATE SET - uri = excluded.uri, - cid = excluded.cid, - record = excluded.record, - time_us = excluded.time_us, - indexed_at = excluded.indexed_at` - ) - .bind( - record.spaceUri, - uri, - record.authorDid, - record.rkey, - record.cid, - JSON.stringify(record.record), - record.createdAt, - Date.now() - ) - .run(); - - if (childShort && this.config) { - await this.recountParentsForSpace( - record.spaceUri, - childShort, - beforeRecord, - record.record, - record.authorDid - ); - } - } - - async getRecord( - spaceUri: string, - collection: string, - authorDid: string, - rkey: string - ): Promise<StoredRecord | null> { - const table = this.tableFor(collection); - const row = await this.db - .prepare( - `SELECT * FROM ${table} - WHERE space_uri = ? AND did = ? AND rkey = ?` - ) - .bind(spaceUri, authorDid, rkey) - .first<any>(); - return row ? mapRecordRow(row, collection) : null; - } - - async listRecords( - spaceUri: string, - collection: string, - options: ListOptions = {} - ): Promise<ListResult> { - const table = this.tableFor(collection); - const limit = Math.min(options.limit ?? 50, 200); - const clauses: string[] = ["space_uri = ?"]; - const params: any[] = [spaceUri]; - - if (options.byUser) { - clauses.push("did = ?"); - params.push(options.byUser); - } - if (options.cursor) { - clauses.push("time_us < ?"); - params.push(Number(options.cursor)); - } - - const sql = `SELECT * FROM ${table} - WHERE ${clauses.join(" AND ")} - ORDER BY time_us DESC - LIMIT ?`; - params.push(limit + 1); - - const { results } = await this.db.prepare(sql).bind(...params).all<any>(); - const records = results.map((r) => mapRecordRow(r, collection)); - let cursor: string | undefined; - if (records.length > limit) { - const next = records.pop()!; - cursor = String(next.createdAt); - } - return { records, cursor }; - } - - async deleteRecord( - spaceUri: string, - collection: string, - authorDid: string, - rkey: string - ): Promise<void> { - const table = this.tableFor(collection); - - const childShort = this.config ? shortNameForNsid(this.config, collection) : null; - const prev = childShort - ? await this.db - .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) - .bind(spaceUri, authorDid, rkey) - .first<{ record: unknown } | null>() - : null; - const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); - - await this.db - .prepare( - `DELETE FROM ${table} - WHERE space_uri = ? AND did = ? AND rkey = ?` - ) - .bind(spaceUri, authorDid, rkey) - .run(); - - if (childShort && this.config) { - await this.recountParentsForSpace(spaceUri, childShort, beforeRecord, null, authorDid); - } - } - - /** Recompute count columns on parent records in the same space, scoped to the - * targets derived from before/after versions of the written/deleted child record. */ - private async recountParentsForSpace( - spaceUri: string, - childShort: string, - before: Record<string, unknown> | null, - after: Record<string, unknown> | null, - childDid: string - ): Promise<void> { - if (!this.config) return; - const config = this.config; - const resolved = (config as ResolvedContrailConfig)._resolved; - const childTable = spacesRecordsTableName(childShort); - - type Inbound = { parentShort: string; relationName: string; rel: RelationConfig }; - const inbound: Inbound[] = []; - for (const [parentShort, parentCfg] of Object.entries(config.collections)) { - if (parentCfg.allowInSpaces === false) continue; - for (const [relName, rel] of Object.entries(parentCfg.relations ?? {})) { - if (rel.count === false) continue; - if (rel.collection !== childShort) continue; - inbound.push({ parentShort, relationName: relName, rel }); - } - } - if (inbound.length === 0) return; - - // Deduplicate (parent, relation, target) across before/after. - const keyed = new Map<string, { parentShort: string; relationName: string; rel: RelationConfig; target: string }>(); - for (const { parentShort, relationName, rel } of inbound) { - const field = getRelationField(rel); - const collectTarget = (rec: Record<string, unknown> | null) => { - if (!rec) return; - if (rel.match === "did") { - keyed.set(`${parentShort}:${relationName}:${childDid}`, { - parentShort, relationName, rel, target: childDid, - }); - return; - } - const v = getNestedValue(rec, field); - if (typeof v === "string" && v.length > 0) { - keyed.set(`${parentShort}:${relationName}:${v}`, { - parentShort, relationName, rel, target: v, - }); - } - }; - collectTarget(before); - collectTarget(after); - } - if (keyed.size === 0) return; - - const dialect = getDialect(this.db); - const stmts: ReturnType<Database["prepare"]>[] = []; - - for (const { parentShort, relationName, rel, target } of keyed.values()) { - const parentTable = spacesRecordsTableName(parentShort); - const matchColumn = rel.match === "did" ? "did" : "uri"; - const field = getRelationField(rel); - const countExpr = rel.countDistinct - ? `COUNT(DISTINCT ${rel.countDistinct})` - : "COUNT(*)"; - - const setClauses: string[] = []; - const binds: (string | number)[] = []; - - const totalCol = countColumnName(rel.collection); - setClauses.push( - `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ?)` - ); - binds.push(spaceUri, target); - - if (rel.groupBy) { - const mapping = resolved?.relations[parentShort]?.[relationName]; - if (mapping?.groups) { - for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - setClauses.push( - `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ? AND ${dialect.jsonExtract("record", rel.groupBy)} = ?)` - ); - binds.push(spaceUri, target, fullToken); - } - } - } - - binds.push(spaceUri, target); - stmts.push( - this.db - .prepare( - `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE space_uri = ? AND ${matchColumn} = ?` - ) - .bind(...binds) - ); - } - - if (stmts.length > 0) await this.db.batch(stmts); - } - - async listCollections( - spaceUri: string, - options: { byUser?: string } = {} - ): Promise<CollectionCount[]> { - if (!this.config) return []; - const results: CollectionCount[] = []; - for (const [short, colConfig] of Object.entries(this.config.collections)) { - if (colConfig.allowInSpaces === false) continue; - const table = spacesRecordsTableName(short); - const clauses: string[] = ["space_uri = ?"]; - const params: any[] = [spaceUri]; - if (options.byUser) { - clauses.push("did = ?"); - params.push(options.byUser); - } - try { - const row = await this.db - .prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${clauses.join(" AND ")}`) - .bind(...params) - .first<{ count: number }>(); - const count = Number(row?.count ?? 0); - if (count > 0) results.push({ collection: colConfig.collection, count }); - } catch { - // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip - } - } - return results; - } - - // ---- Blobs ---- - - async putBlobMeta(row: BlobMetaRow): Promise<void> { - const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT (space_uri, cid) DO NOTHING`; - await this.db - .prepare(sql) - .bind(row.spaceUri, row.cid, row.mimeType, row.size, row.authorDid, row.createdAt) - .run(); - } - - async getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null> { - const r = await this.db - .prepare(`SELECT * FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) - .bind(spaceUri, cid) - .first<any>(); - return r ? mapBlobMetaRow(r) : null; - } - - async listBlobMeta( - spaceUri: string, - options: ListBlobsOptions = {} - ): Promise<ListBlobsResult> { - const limit = Math.min(options.limit ?? 50, 200); - const clauses: string[] = ["space_uri = ?"]; - const params: any[] = [spaceUri]; - if (options.byUser) { - clauses.push("author_did = ?"); - params.push(options.byUser); - } - if (options.cursor) { - clauses.push("created_at < ?"); - params.push(Number(options.cursor)); - } - const sql = `SELECT * FROM spaces_blobs - WHERE ${clauses.join(" AND ")} - ORDER BY created_at DESC - LIMIT ?`; - params.push(limit + 1); - const { results } = await this.db.prepare(sql).bind(...params).all<any>(); - const blobs = results.map(mapBlobMetaRow); - let cursor: string | undefined; - if (blobs.length > limit) { - const next = blobs.pop()!; - cursor = String(next.createdAt); - } - return { blobs, cursor }; - } - - async deleteBlobMeta(spaceUri: string, cid: string): Promise<void> { - await this.db - .prepare(`DELETE FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) - .bind(spaceUri, cid) - .run(); - } - - async findOrphanBlobs( - spaceUri: string, - cutoff: number, - limit: number - ): Promise<BlobMetaRow[]> { - if (!this.config) return []; - // Gather candidate blobs older than cutoff, then filter out any whose CID - // appears in any record JSON in this space. We use a cheap substring probe - // (LIKE) per collection — false positives are OK because an orphan that - // survives GC just gets collected next cycle; false negatives (deleting - // a referenced blob) would be a bug, and substring search over the full - // CID is safe enough for that. - const { results } = await this.db - .prepare( - `SELECT * FROM spaces_blobs - WHERE space_uri = ? AND created_at < ? - ORDER BY created_at ASC - LIMIT ?` - ) - .bind(spaceUri, cutoff, limit) - .all<any>(); - const candidates = results.map(mapBlobMetaRow); - if (candidates.length === 0) return []; - - const tables: string[] = []; - for (const [short, colConfig] of Object.entries(this.config.collections)) { - if (colConfig.allowInSpaces === false) continue; - tables.push(spacesRecordsTableName(short)); - } - - const orphans: BlobMetaRow[] = []; - for (const blob of candidates) { - let referenced = false; - const pattern = `%${blob.cid}%`; - for (const table of tables) { - try { - const row = await this.db - .prepare( - `SELECT 1 FROM ${table} WHERE space_uri = ? AND record LIKE ? LIMIT 1` - ) - .bind(spaceUri, pattern) - .first<any>(); - if (row) { - referenced = true; - break; - } - } catch { - // table missing — ignore - } - } - if (!referenced) orphans.push(blob); - } - return orphans; - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/binding.ts +++ b/packages/contrail/src/core/spaces/binding.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/blob-adapter.ts b/packages/contrail/src/core/spaces/blob-adapter.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/blob-adapter.ts +++ b/packages/contrail/src/core/spaces/blob-adapter.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/blob-gc.ts b/packages/contrail/src/core/spaces/blob-gc.ts index d523fcd..551c495 100644 --- a/packages/contrail/src/core/spaces/blob-gc.ts +++ b/packages/contrail/src/core/spaces/blob-gc.ts @@ -1,38 +1 @@ -import type { BlobAdapter } from "./blob-adapter"; -import { blobKey } from "./blob-adapter"; -import type { StorageAdapter } from "./types"; - -export interface BlobGcOptions { - /** Orphan rows created before this timestamp are eligible for deletion. */ - olderThan: number; - /** Maximum number of blobs to delete in this pass. Defaults to 500. */ - batchSize?: number; -} - -export interface BlobGcResult { - deleted: number; - cids: string[]; -} - -/** Delete blob bytes + metadata for any blob older than `olderThan` that - * is not referenced by any record in the space. Safe to run periodically. */ -export async function gcOrphanBlobs( - storage: StorageAdapter, - blobs: BlobAdapter, - spaceUri: string, - options: BlobGcOptions -): Promise<BlobGcResult> { - const batchSize = options.batchSize ?? 500; - const orphans = await storage.findOrphanBlobs(spaceUri, options.olderThan, batchSize); - if (orphans.length === 0) return { deleted: 0, cids: [] }; - - const keys: string[] = []; - for (const row of orphans) { - keys.push(await blobKey(row.spaceUri, row.cid)); - } - await blobs.delete(keys); - for (const row of orphans) { - await storage.deleteBlobMeta(row.spaceUri, row.cid); - } - return { deleted: orphans.length, cids: orphans.map((o) => o.cid) }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/blob-refs.ts b/packages/contrail/src/core/spaces/blob-refs.ts index 62cf693..551c495 100644 --- a/packages/contrail/src/core/spaces/blob-refs.ts +++ b/packages/contrail/src/core/spaces/blob-refs.ts @@ -1,27 +1 @@ -/** - * Walk a record JSON and collect every atproto blob ref. - * - * Blob refs look like: - * { "$type": "blob", "ref": { "$link": "<cid>" }, "mimeType": "...", "size": N } - * - * We return the CID strings. - */ -export function collectBlobCids(value: unknown, out: Set<string> = new Set()): Set<string> { - if (value == null) return out; - if (Array.isArray(value)) { - for (const v of value) collectBlobCids(v, out); - return out; - } - if (typeof value !== "object") return out; - - const obj = value as Record<string, unknown>; - if (obj["$type"] === "blob") { - const ref = obj["ref"] as { $link?: unknown } | undefined; - if (ref && typeof ref["$link"] === "string") out.add(ref["$link"]); - // Don't descend — a blob ref's own shape has no nested blobs. - return out; - } - - for (const v of Object.values(obj)) collectBlobCids(v, out); - return out; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/credentials.ts +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/in-process.ts b/packages/contrail/src/core/spaces/in-process.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/in-process.ts +++ b/packages/contrail/src/core/spaces/in-process.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index dadb7c9..551c495 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -1,1027 +1 @@ -import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig, Database } from "../types"; -import { HostedAdapter } from "./adapter"; -import { checkAccess } from "./acl"; -import type { ServiceAuth } from "./auth"; -import { - buildVerifier, - checkInviteReadGrant, - createServiceAuthMiddleware, - extractInviteToken, - extractSpaceCredential, -} from "./auth"; -import { nextTid } from "./tid"; -import { hashInviteToken } from "../invite/token"; -import { buildSpaceUri, parseSpaceUri } from "./uri"; -import { - DEFAULT_BLOB_MAX_SIZE, - DEFAULT_CREDENTIAL_TTL_MS, - type AuthorityConfig, - type RecordHostConfig, - type RecordHost, - type SpaceAuthority, - type SpaceRow, - type StorageAdapter, -} from "./types"; -import { blobKey } from "./blob-adapter"; -import { collectBlobCids } from "./blob-refs"; -import { - createBindingCredentialVerifier, - decodeUnverifiedClaims, - issueCredential, - verifyCredential, - type CredentialClaims, - type CredentialScope, - type CredentialVerifier, -} from "./credentials"; -import { - createCompositeBindingResolver, - createEnrollmentBindingResolver, - createLocalBindingResolver, - createLocalKeyResolver, -} from "./binding"; -import { create as createCid, toString as cidToString } from "@atcute/cid"; - -/** Optional hook to extend `<ns>.spaceExt.whoami` with extra fields when a - * module above spaces (e.g. community) wants to override the default - * binary-membership response. If the hook returns a non-null object, that - * object is the entire response body. If null, falls through to the - * default behavior (just `isOwner`/`isMember`). - * - * Spaces stays community-agnostic: any consumer can plug in here. */ -export type WhoamiExtension = (input: { - spaceUri: string; - callerDid: string; - isOwner: boolean; - ownerDid: string; -}) => Promise<Record<string, unknown> | null>; - -export interface SpacesRoutesOptions { - /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ - authMiddleware?: MiddlewareHandler; - /** Storage adapter override. Defaults to HostedAdapter(db). */ - adapter?: StorageAdapter; - /** Optional whoami extension; see {@link WhoamiExtension}. */ - whoamiExtension?: WhoamiExtension; - /** Optional credential verifier for the record host. When omitted, a - * default in-process binding verifier is built from the authority's - * signing config (Local binding + Local key resolvers). Override to - * accept credentials from external authorities — wire in PDS-record / - * DID-doc binding resolvers and a DID-doc key resolver. */ - credentialVerifier?: CredentialVerifier; -} - -/** Umbrella registration: wires both the authority and the record-host - * routes against the same adapter. Today's deployments enable both via - * `config.spaces.authority` and `config.spaces.recordHost`. Either may be - * omitted in future split deployments — phase 5 lifts the assumption that - * one process runs both. */ -export function registerSpacesRoutes( - app: Hono, - db: Database, - config: ContrailConfig, - options: SpacesRoutesOptions = {}, - ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null -): void { - const spacesConfig = config.spaces; - if (!spacesConfig) return; - const authorityConfig = spacesConfig.authority; - if (!authorityConfig) return; - - const adapter = options.adapter ?? ctx?.adapter ?? new HostedAdapter(db, config); - const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); - const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - - // When the record host is colocated, hand it to the authority so - // createSpace can auto-enroll. This is the in-process default — - // single-call createSpace gives you a usable space without requiring an - // explicit `recordHost.enroll` afterward. Split deployments don't get - // this convenience; their createSpace caller (or operator) has to call - // enroll on the remote host explicitly. - const localRecordHost = spacesConfig.recordHost ? adapter : null; - registerAuthorityRoutes( - app, - adapter, - authorityConfig, - config, - auth, - options.whoamiExtension, - localRecordHost - ); - - if (spacesConfig.recordHost) { - // Default in-process verifier: enrollment is the canonical binding - // source (matches the host's local consent), with Local-binding as a - // fallback for the case where the authority's createSpace ran but - // auto-enroll was bypassed. Local key resolver knows the local - // authority's public key. Caller overrides via - // `options.credentialVerifier` to accept external authorities — wire - // in DID-doc key resolvers there. - const verifier = - options.credentialVerifier ?? - (authorityConfig.signing - ? createBindingCredentialVerifier({ - bindings: createCompositeBindingResolver([ - createEnrollmentBindingResolver({ recordHost: adapter }), - createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), - ]), - keys: createLocalKeyResolver({ - authorityDid: authorityConfig.serviceDid, - publicKey: authorityConfig.signing.publicKey, - }), - }) - : undefined); - registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, verifier); - } -} - -/** Register the **space authority** XRPC surface — space lifecycle, member - * list, app policy, whoami. Does NOT touch records or blobs. - * - * When a `localRecordHost` is passed, the authority's `createSpace` handler - * also enrolls the new space on that host — convenient for in-process - * deployments where the same operator runs both roles. Split deployments - * pass `null` and arrange enrollment explicitly via `recordHost.enroll`. */ -export function registerAuthorityRoutes( - app: Hono, - authority: SpaceAuthority, - authorityConfig: AuthorityConfig, - config: ContrailConfig, - auth: MiddlewareHandler, - whoamiExtension?: WhoamiExtension, - localRecordHost?: RecordHost | null -): void { - /** Space endpoints are emitted per-deployment under the configured namespace; - * the deployment owns and publishes its own lexicons. The library ships - * templates at `lexicon-templates/spaces/*` that the generator instantiates - * under `<ns>.space.*` (spec-aligned) and `<ns>.spaceExt.*` (contrail - * extras — invites, whoami — that the permissioned-data spec doesn't cover). */ - const SPACE = `${config.namespace}.space`; - const SPACE_EXT = `${config.namespace}.spaceExt`; - - // ---- Read endpoints ---- - - app.get(`/xrpc/${SPACE}.listSpaces`, auth, async (c) => { - const sa = getAuth(c); - const scope = c.req.query("scope") ?? "member"; // "member" | "owner" - const type = c.req.query("type") ?? undefined; - const owner = c.req.query("owner") ?? undefined; - const cursor = c.req.query("cursor") ?? undefined; - const limit = c.req.query("limit") ? Number(c.req.query("limit")) : undefined; - - const opts: Parameters<typeof authority.listSpaces>[0] = { type, cursor, limit }; - if (scope === "owner") opts.ownerDid = sa.issuer; - else { - opts.memberDid = sa.issuer; - if (owner) opts.ownerDid = owner; // narrow to spaces owned by this DID - } - - const result = await authority.listSpaces(opts); - return c.json({ - spaces: result.spaces.map((s) => publicSpaceView(s, s.ownerDid === sa.issuer)), - cursor: result.cursor, - }); - }); - - app.get(`/xrpc/${SPACE}.listMembers`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - const member = isOwner ? null : await authority.getMember(spaceUri, sa.issuer); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - const members = await authority.listMembers(spaceUri); - return c.json({ members }); - }); - - /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is - * present so anonymous bearer reads don't 401 before the route handler can - * validate the token. */ - const readAuth: MiddlewareHandler = async (c, next) => { - if (extractInviteToken(c.req.raw)) { - await next(); - return; - } - return auth(c, next); - }; - - app.get(`/xrpc/${SPACE}.getSpace`, readAuth, async (c) => { - const uri = c.req.query("uri"); - if (!uri) return c.json({ error: "InvalidRequest", message: "uri required" }, 400); - const space = await authority.getSpace(uri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const authz = await authorizeRead(c, authority, uri); - if (authz instanceof Response) return authz; - - if (authz.via === "token") { - // Anonymous read-token bearer — show non-owner space view. - return c.json({ space: publicSpaceView(space, false) }); - } - - if (authz.via === "credential") { - // Credential proves membership; derive isOwner from sub vs ownerDid. - const isOwner = authz.claims.sub === space.ownerDid; - return c.json({ space: publicSpaceView(space, isOwner) }); - } - - const sa = authz.sa; - const isOwner = sa.issuer === space.ownerDid; - const member = isOwner ? null : await authority.getMember(uri, sa.issuer); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - return c.json({ space: publicSpaceView(space, isOwner) }); - }); - - // ---- Space management (owner-gated) ---- - - app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => ({}))) as { - type?: string; - key?: string; - appPolicy?: SpaceRow["appPolicy"]; - appPolicyRef?: string; - }; - - const type = body.type ?? authorityConfig.type; - const key = body.key ?? nextTid(); - const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); - - const existing = await authority.getSpace(uri); - if (existing) return c.json({ error: "AlreadyExists", uri }, 409); - - const space = await authority.createSpace({ - uri, - ownerDid: sa.issuer, - type, - key, - serviceDid: authorityConfig.serviceDid, - appPolicyRef: body.appPolicyRef ?? null, - appPolicy: body.appPolicy ?? authorityConfig.defaultAppPolicy ?? null, - }); - // Owner is implicit; we still write a row so membership queries are uniform. - await authority.addMember(uri, sa.issuer, sa.issuer); - - // Auto-enroll the space on the colocated record host (if any). Without - // this, putRecord/listRecords would 404 with "not-enrolled" until the - // caller explicitly hit recordHost.enroll. Idempotent (ON CONFLICT - // UPDATE), so calling enroll again later is harmless. - if (localRecordHost) { - await localRecordHost.enroll({ - spaceUri: uri, - authorityDid: authorityConfig.serviceDid, - enrolledAt: Date.now(), - enrolledBy: sa.issuer, - }); - } - - return c.json({ space: publicSpaceView(space, true) }); - }); - - app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - await authority.addMember(body.spaceUri, body.did, sa.issuer); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - if (body.did === space.ownerDid) { - return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); - } - await authority.removeMember(body.spaceUri, body.did); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid === sa.issuer) { - return c.json( - { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, - 400 - ); - } - await authority.removeMember(body.spaceUri, sa.issuer); - return c.json({ ok: true }); - }); - - // Unified whoami — `<ns>.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, - // ... }. Extra fields (e.g. accessLevel for community-owned spaces) come - // from the optional whoamiExtension hook; without one, response is binary. - app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - - if (whoamiExtension) { - const ext = await whoamiExtension({ - spaceUri, - callerDid: sa.issuer, - isOwner, - ownerDid: space.ownerDid, - }); - if (ext) return c.json(ext); - } - - // Default: binary membership. - if (isOwner) return c.json({ isOwner: true, isMember: true }); - const member = await authority.getMember(spaceUri, sa.issuer); - return c.json({ isOwner: false, isMember: !!member }); - }); - - // ---- Credential endpoints ---- - - /** Mint a space credential for a member of `spaceUri`. Caller is identified - * by the JWT issuer; the credential's `sub` is set to that DID. */ - app.post(`/xrpc/${SPACE}.getCredential`, auth, async (c) => { - if (!authorityConfig.signing) { - return c.json( - { error: "NotImplemented", message: "authority is not configured to sign credentials" }, - 501 - ); - } - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - const member = isOwner ? null : await authority.getMember(body.spaceUri, sa.issuer); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - - // App policy is checked at credential-issuance time. Existing credentials - // remain valid until expiry — that's the spec contract (revocation - // bounded by TTL, not synchronous). - if (space.appPolicy) { - const allowed = checkClientId(space.appPolicy, sa.clientId); - if (!allowed) return c.json({ error: "Forbidden", reason: "app-not-allowed" }, 403); - } - - const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; - const { credential, expiresAt } = await issueCredential( - { - iss: authorityConfig.serviceDid, - sub: sa.issuer, - space: body.spaceUri, - scope: "rw", - ttlMs: ttl, - }, - authorityConfig.signing - ); - return c.json({ credential, expiresAt }); - }); - - /** Refresh an unexpired credential. Used by long-running clients to extend - * their access without going back through the JWT mint dance. The current - * credential must verify; the bearer must still be a member. */ - app.post(`/xrpc/${SPACE}.refreshCredential`, async (c) => { - if (!authorityConfig.signing) { - return c.json( - { error: "NotImplemented", message: "authority is not configured to sign credentials" }, - 501 - ); - } - const body = (await c.req.json().catch(() => null)) as { credential?: string } | null; - if (!body?.credential) { - return c.json({ error: "InvalidRequest", message: "credential required" }, 400); - } - const signing = authorityConfig.signing; - const claims = await verifyAndAuthorizeRefresh(body.credential, authorityConfig); - if ("error" in claims) return c.json(claims, claims.status); - - const space = await authority.getSpace(claims.space); - if (!space) return c.json({ error: "NotFound" }, 404); - const isOwner = space.ownerDid === claims.sub; - const member = isOwner ? null : await authority.getMember(claims.space, claims.sub); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - - const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; - const { credential, expiresAt } = await issueCredential( - { - iss: authorityConfig.serviceDid, - sub: claims.sub, - space: claims.space, - scope: claims.scope, - ttlMs: ttl, - }, - signing - ); - return c.json({ credential, expiresAt }); - }); -} - -/** Verify a credential presented at refreshCredential. Returns the claims, or - * an error envelope ready to relay. Different from the record-host verifier - * in two ways: (a) we don't have the expectedSpace yet — we read it from the - * credential itself; (b) we don't enforce a scope. */ -async function verifyAndAuthorizeRefresh( - credential: string, - authorityConfig: AuthorityConfig -): Promise<CredentialClaims | { error: string; reason?: string; message?: string; status: 400 | 401 }> { - const peek = decodeUnverifiedClaims(credential); - if (!peek) return { error: "InvalidRequest", reason: "malformed", status: 400 }; - if (peek.iss !== authorityConfig.serviceDid) { - return { error: "Forbidden", reason: "wrong-issuer", status: 401 }; - } - if (!authorityConfig.signing) { - return { error: "InvalidState", status: 401 }; - } - const signing = authorityConfig.signing; - const result = await verifyCredential(credential, { - expectedSpace: peek.space, - resolveKey: async (iss) => (iss === authorityConfig.serviceDid ? signing.publicKey : null), - }); - if (!result.ok) { - return { error: "InvalidCredential", reason: result.reason, status: 401 }; - } - return result.claims; -} - -/** App-policy check using just `clientId`. Mirrors `acl.ts:checkAppPolicy` - * but inlined here so the credential-issuance path doesn't need to construct - * a full AclInput. */ -function checkClientId( - appPolicy: NonNullable<SpaceRow["appPolicy"]>, - clientId: string | undefined -): boolean { - const listed = clientId ? appPolicy.apps.includes(clientId) : false; - if (appPolicy.mode === "allow") return !listed; - return listed; -} - -/** Register the **record host** XRPC surface — record + blob CRUD. - * - * Auth precedence on every route: - * 1. `X-Space-Credential` header (if a verifier is wired and the credential - * is valid) — caller DID = credential `sub`, no clientId. - * 2. Read-route invite token (`?inviteToken=` or `Bearer atmo-invite:...`). - * 3. Service-auth JWT (existing behavior) — caller DID = JWT issuer. - * - * When a credential is presented, the record host trusts it: no member - * check, no app-policy check (those happen at issuance time on the - * authority side). Service-auth requests still consult the authority — that - * bridge is what phase 5 cuts when the host/authority split goes runtime. */ -export function registerRecordHostRoutes( - app: Hono, - recordHost: RecordHost, - authority: SpaceAuthority, - recordHostConfig: RecordHostConfig, - config: ContrailConfig, - auth: MiddlewareHandler, - /** Optional credential verifier. When present, the record host accepts - * `X-Space-Credential` as an alternative to a service-auth JWT. */ - credentialVerifier?: CredentialVerifier -): void { - const SPACE = `${config.namespace}.space`; - - /** Auth wrapper: tries credential first, then delegates to JWT auth. */ - const authWithCredential: MiddlewareHandler = async (c, next) => { - const credToken = extractSpaceCredential(c.req.raw); - if (credToken) { - if (!credentialVerifier) { - return c.json( - { error: "AuthRequired", reason: "credential-verifier-not-configured" }, - 401 - ); - } - const result = await credentialVerifier.verify(credToken); - if (!result.ok) { - return c.json({ error: "AuthRequired", reason: result.reason }, 401); - } - c.set("spaceCredential", result.claims); - await next(); - return; - } - return auth(c, next); - }; - - /** Read-route auth: like {@link authWithCredential} but also short-circuits - * on a read-grant invite token. Token presence skips both credential and - * JWT middlewares; the route handler validates the token via authorizeRead. */ - const readAuth: MiddlewareHandler = async (c, next) => { - if (extractInviteToken(c.req.raw)) { - await next(); - return; - } - return authWithCredential(c, next); - }; - - /** Hard gate on every record-host operation: the space must be enrolled. - * Returns the enrollment row, or a Response to relay. */ - async function requireEnrollment( - c: Context, - spaceUri: string - ): Promise<{ authorityDid: string } | Response> { - const enrollment = await recordHost.getEnrollment(spaceUri); - if (!enrollment) { - return c.json( - { error: "NotFound", reason: "not-enrolled", message: "space is not enrolled on this record host" }, - 404 - ); - } - return enrollment; - } - - // ---- Enrollment endpoint ---- - // - // Authorize a space onto this record host. Caller must be either the space - // owner OR the declared authority — the host treats either as a sufficient - // signal of consent. Idempotent: re-enrolling the same space updates the - // authority binding. - const RECORD_HOST = `${config.namespace}.recordHost`; - - app.post(`/xrpc/${RECORD_HOST}.enroll`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; authority?: string } - | null; - if (!body?.spaceUri || !body.authority) { - return c.json({ error: "InvalidRequest", message: "spaceUri and authority required" }, 400); - } - const parts = parseSpaceUri(body.spaceUri); - if (!parts) { - return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); - } - const callerIsOwner = sa.issuer === parts.ownerDid; - const callerIsAuthority = sa.issuer === body.authority; - if (!callerIsOwner && !callerIsAuthority) { - return c.json( - { error: "Forbidden", reason: "not-owner-or-authority" }, - 403 - ); - } - await recordHost.enroll({ - spaceUri: body.spaceUri, - authorityDid: body.authority, - enrolledAt: Date.now(), - enrolledBy: sa.issuer, - }); - return c.json({ ok: true }); - }); - - app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { - const spaceUri = c.req.query("spaceUri"); - const collection = c.req.query("collection"); - if (!spaceUri || !collection) { - return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const authz = await authorizeRead(c, authority, spaceUri); - if (authz instanceof Response) return authz; - - if (authz.via === "jwt") { - // JWT path consults the authority for member + app-policy checks. - // Split deployments where the host has no live authority will degrade - // here (getSpace returns null) and return 404, which is correct: a - // host that can't reach an authority can't validate JWT membership. - const sa = authz.sa; - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const result = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!result.allow) { - return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - } - // Credential and token paths are pre-authorized — credential's signature - // proves the authority granted access; token validation already happened. - - const list = await recordHost.listRecords(spaceUri, collection, { - byUser: c.req.query("byUser") ?? undefined, - cursor: c.req.query("cursor") ?? undefined, - limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, - }); - return c.json(list); - }); - - app.get(`/xrpc/${SPACE}.getRecord`, readAuth, async (c) => { - const spaceUri = c.req.query("spaceUri"); - const collection = c.req.query("collection"); - const author = c.req.query("author"); - const rkey = c.req.query("rkey"); - if (!spaceUri || !collection || !author || !rkey) { - return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const authz = await authorizeRead(c, authority, spaceUri); - if (authz instanceof Response) return authz; - - if (authz.via === "jwt") { - const sa = authz.sa; - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const result = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - targetAuthorDid: author, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - - const record = await recordHost.getRecord(spaceUri, collection, author, rkey); - if (!record) return c.json({ error: "NotFound" }, 404); - return c.json({ record }); - }); - - // Write endpoints - app.post(`/xrpc/${SPACE}.putRecord`, authWithCredential, async (c) => { - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; collection?: string; rkey?: string; record?: Record<string, unknown> } - | null; - if (!body?.spaceUri || !body.collection || !body.record) { - return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); - } - const enrollment = await requireEnrollment(c, body.spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, body.spaceUri, "rw"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - // JWT path needs authority access for member + app-policy checks. - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, caller.callerDid); - const result = checkAccess({ - op: "write", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - - // Validate that every blob referenced by this record has already been - // uploaded into this space. This mirrors how PDSes require uploadBlob - // before putRecord, and prevents forging refs to blobs the caller never - // actually claimed. - if (recordHostConfig.blobs) { - const cids = collectBlobCids(body.record); - for (const cid of cids) { - const meta = await recordHost.getBlobMeta(body.spaceUri, cid); - if (!meta) { - return c.json( - { - error: "InvalidRequest", - reason: "unknown-blob-ref", - message: `Record references blob ${cid} that has not been uploaded to this space.`, - }, - 400 - ); - } - } - } - - const rkey = body.rkey ?? nextTid(); - const now = Date.now(); - await recordHost.putRecord({ - spaceUri: body.spaceUri, - collection: body.collection, - authorDid: caller.callerDid, - rkey, - cid: null, - record: body.record, - createdAt: now, - }); - return c.json({ rkey, authorDid: caller.callerDid, createdAt: now }); - }); - - app.post(`/xrpc/${SPACE}.deleteRecord`, authWithCredential, async (c) => { - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; collection?: string; rkey?: string } - | null; - if (!body?.spaceUri || !body.collection || !body.rkey) { - return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); - } - const enrollment = await requireEnrollment(c, body.spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, body.spaceUri, "rw"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, caller.callerDid); - const result = checkAccess({ - op: "delete", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - targetAuthorDid: caller.callerDid, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - // Credential path: scope=rw is checked in resolveCaller. Delete remains - // author-scoped — the credential's `sub` is the caller, and we only - // delete records authored by that DID. - - await recordHost.deleteRecord(body.spaceUri, body.collection, caller.callerDid, body.rkey); - return c.json({ ok: true }); - }); - - // Blobs (only registered when a blob adapter is configured) - if (recordHostConfig.blobs) { - const blobsCfg = recordHostConfig.blobs; - const blobAdapter = blobsCfg.adapter; - const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; - const accept = blobsCfg.accept; - - app.post(`/xrpc/${SPACE}.uploadBlob`, authWithCredential, async (c) => { - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, spaceUri, "rw"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, caller.callerDid); - const aclResult = checkAccess({ - op: "write", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); - } - } - - const mimeType = c.req.header("content-type") ?? "application/octet-stream"; - if (accept && !accept.includes(mimeType)) { - return c.json( - { error: "InvalidMimeType", message: `MIME type ${mimeType} is not accepted.` }, - 400 - ); - } - - const declaredLen = c.req.header("content-length"); - if (declaredLen && Number(declaredLen) > maxSize) { - return c.json( - { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, - 413 - ); - } - - const buf = await c.req.arrayBuffer(); - const bytes = new Uint8Array(buf); - if (bytes.byteLength > maxSize) { - return c.json( - { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, - 413 - ); - } - - const cid = await createCid(0x55, bytes); - const cidString = cidToString(cid); - const key = await blobKey(spaceUri, cidString); - - await blobAdapter.put(key, bytes, { mimeType, size: bytes.byteLength }); - await recordHost.putBlobMeta({ - spaceUri, - cid: cidString, - mimeType, - size: bytes.byteLength, - authorDid: caller.callerDid, - createdAt: Date.now(), - }); - - return c.json({ - blob: { - $type: "blob", - ref: { $link: cidString }, - mimeType, - size: bytes.byteLength, - }, - }); - }); - - app.get(`/xrpc/${SPACE}.getBlob`, readAuth, async (c) => { - const spaceUri = c.req.query("spaceUri"); - const cid = c.req.query("cid"); - if (!spaceUri || !cid) { - return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const authz = await authorizeRead(c, authority, spaceUri); - if (authz instanceof Response) return authz; - - if (authz.via === "jwt") { - const sa = authz.sa; - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const aclResult = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); - } - } - - const meta = await recordHost.getBlobMeta(spaceUri, cid); - if (!meta) return c.json({ error: "NotFound" }, 404); - const key = await blobKey(spaceUri, cid); - const bytes = await blobAdapter.get(key); - if (!bytes) return c.json({ error: "NotFound" }, 404); - - return new Response(bytes, { - headers: { - "content-type": meta.mimeType, - "content-length": String(meta.size), - }, - }); - }); - - app.get(`/xrpc/${SPACE}.listBlobs`, authWithCredential, async (c) => { - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, spaceUri, "read"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, caller.callerDid); - const aclResult = checkAccess({ - op: "read", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); - } - } - - const result = await recordHost.listBlobMeta(spaceUri, { - byUser: c.req.query("byUser") ?? undefined, - cursor: c.req.query("cursor") ?? undefined, - limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, - }); - return c.json(result); - }); - } -} - -/** Authorize a read request — three valid paths: a verified space credential - * (set by the credential middleware), a read-grant invite token, or a - * service-auth JWT. - * - * Credential and token paths skip the membership check downstream — the - * credential or token IS the proof. The JWT path requires a member check - * in the route handler. */ -async function authorizeRead( - c: Context, - authority: SpaceAuthority, - spaceUri: string -): Promise< - | { via: "credential"; claims: CredentialClaims } - | { via: "token" } - | { via: "jwt"; sa: ServiceAuth } - | Response -> { - const cred = c.get("spaceCredential") as CredentialClaims | undefined; - if (cred) { - if (cred.space !== spaceUri) { - return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); - } - return { via: "credential", claims: cred }; - } - const rawToken = extractInviteToken(c.req.raw); - if (rawToken) { - const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); - if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); - return { via: "token" }; - } - const sa = c.get("serviceAuth") as ServiceAuth | undefined; - if (sa) return { via: "jwt", sa }; - return c.json( - { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, - 401 - ); -} - -/** Unified caller resolution for write/manage paths on the record host. - * Either a verified credential (set by middleware) or a service-auth JWT. - * When a credential is present, also enforces space-match and the requested - * scope. Returns either a caller envelope or a Response to relay. */ -function resolveCaller( - c: Context, - requestSpace: string, - requiredScope: CredentialScope -): { callerDid: string; clientId: string | undefined; viaCredential: boolean } | Response { - const cred = c.get("spaceCredential") as CredentialClaims | undefined; - if (cred) { - if (cred.space !== requestSpace) { - return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); - } - if (requiredScope === "rw" && cred.scope !== "rw") { - return c.json({ error: "Forbidden", reason: "credential-wrong-scope" }, 403); - } - return { callerDid: cred.sub, clientId: undefined, viaCredential: true }; - } - const sa = c.get("serviceAuth") as ServiceAuth | undefined; - if (!sa) return c.json({ error: "AuthRequired", reason: "no-auth" }, 401); - return { callerDid: sa.issuer, clientId: sa.clientId, viaCredential: false }; -} - -function getAuth(c: Parameters<MiddlewareHandler>[0]): ServiceAuth { - const auth = c.get("serviceAuth") as ServiceAuth | undefined; - if (!auth) throw new Error("service auth not set"); - return auth; -} - -function publicSpaceView(space: SpaceRow, forOwner: boolean) { - return { - uri: space.uri, - ownerDid: space.ownerDid, - type: space.type, - key: space.key, - serviceDid: space.serviceDid, - appPolicyRef: space.appPolicyRef, - createdAt: space.createdAt, - ...(forOwner ? { appPolicy: space.appPolicy } : {}), - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/schema.ts b/packages/contrail/src/core/spaces/schema.ts index ae33a7c..551c495 100644 --- a/packages/contrail/src/core/spaces/schema.ts +++ b/packages/contrail/src/core/spaces/schema.ts @@ -1,103 +1 @@ -import type { ContrailConfig, Database } from "../types"; -import type { SqlDialect } from "../dialect"; -import { getDialect } from "../dialect"; -import { - buildCollectionTables, - buildDynamicIndexes, - buildFtsTables, - buildCountColumns, -} from "../db/schema"; - -/** Spaces metadata tables — spaces, members, invites. No per-collection tables. */ -export function buildSpacesBaseSchema(dialect: SqlDialect): string[] { - return [ - `CREATE TABLE IF NOT EXISTS spaces ( - uri TEXT PRIMARY KEY, - owner_did TEXT NOT NULL, - type TEXT NOT NULL, - key TEXT NOT NULL, - service_did TEXT NOT NULL, - app_policy_ref TEXT, - app_policy ${dialect.recordColumnType}, - created_at ${dialect.bigintType} NOT NULL, - deleted_at ${dialect.bigintType} - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_owner ON spaces(owner_did)`, - `CREATE INDEX IF NOT EXISTS idx_spaces_type ON spaces(type)`, - - `CREATE TABLE IF NOT EXISTS spaces_members ( - space_uri TEXT NOT NULL, - did TEXT NOT NULL, - added_at ${dialect.bigintType} NOT NULL, - added_by TEXT, - PRIMARY KEY (space_uri, did) - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_members_did ON spaces_members(did)`, - - `CREATE TABLE IF NOT EXISTS spaces_blobs ( - space_uri TEXT NOT NULL, - cid TEXT NOT NULL, - mime_type TEXT NOT NULL, - size INTEGER NOT NULL, - author_did TEXT NOT NULL, - created_at ${dialect.bigintType} NOT NULL, - PRIMARY KEY (space_uri, cid) - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_author ON spaces_blobs(space_uri, author_did)`, - `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_created ON spaces_blobs(space_uri, created_at)`, - - `CREATE TABLE IF NOT EXISTS spaces_invites ( - token_hash TEXT PRIMARY KEY, - space_uri TEXT NOT NULL, - kind TEXT NOT NULL DEFAULT 'join', - expires_at ${dialect.bigintType}, - max_uses INTEGER, - used_count INTEGER NOT NULL DEFAULT 0, - created_by TEXT NOT NULL, - created_at ${dialect.bigintType} NOT NULL, - revoked_at ${dialect.bigintType}, - note TEXT - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, - - // Record-host enrollment table — the host's local cache of "spaces I - // accept records for, and which authority signs credentials for each." - // Filled by the recordHost.enroll endpoint, or auto-populated by the - // authority's createSpace when both roles run in the same process. - `CREATE TABLE IF NOT EXISTS record_host_enrollments ( - space_uri TEXT PRIMARY KEY, - authority_did TEXT NOT NULL, - enrolled_at ${dialect.bigintType} NOT NULL, - enrolled_by TEXT NOT NULL - )`, - `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, - ]; -} - -/** Full spaces schema (base + per-collection tables + indexes). For callers - * that need a single array of statements. Note: this does NOT include FTS - * virtual tables or ALTER TABLE count columns — those must be applied with - * try/catch fallbacks and are handled by `initSchema`. */ -export function buildSpacesSchema(db: Database, config?: ContrailConfig): string[] { - const dialect = getDialect(db); - const base = buildSpacesBaseSchema(dialect); - if (!config) return base; - return [ - ...base, - ...buildCollectionTables(config, dialect, { forSpaces: true }), - ...buildDynamicIndexes(config, dialect, { forSpaces: true }), - ]; -} - -export async function initSpacesSchema(db: Database, config?: ContrailConfig): Promise<void> { - const dialect = getDialect(db); - const stmts = buildSpacesSchema(db, config); - await db.batch(stmts.map((s) => db.prepare(s))); - if (!config) return; - for (const stmt of buildFtsTables(config, dialect, { forSpaces: true })) { - try { await db.prepare(stmt).run(); } catch { /* ignore */ } - } - for (const stmt of buildCountColumns(config, { forSpaces: true })) { - try { await db.prepare(stmt).run(); } catch { /* ignore */ } - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/tid.ts b/packages/contrail/src/core/spaces/tid.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/tid.ts +++ b/packages/contrail/src/core/spaces/tid.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/uri.ts b/packages/contrail/src/core/spaces/uri.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/uri.ts +++ b/packages/contrail/src/core/spaces/uri.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/tests/persistent.test.ts b/packages/contrail/tests/persistent.test.ts index f248cf0..a1811c0 100644 --- a/packages/contrail/tests/persistent.test.ts +++ b/packages/contrail/tests/persistent.test.ts @@ -5,10 +5,14 @@ import { runPersistent } from "../src/core/persistent"; import { getLastCursor, queryRecords } from "../src/core/db/records"; import { initSchema } from "../src/core/db/schema"; -// Mock identity resolution to avoid network calls in tests -vi.mock("../src/core/identity", () => ({ - refreshStaleIdentities: vi.fn().mockResolvedValue(undefined), -})); +// Identity helpers live in @atmo-dev/contrail-base post-split. Mock there. +vi.mock("@atmo-dev/contrail-base", async (importOriginal) => { + const actual = await importOriginal<typeof import("@atmo-dev/contrail-base")>(); + return { + ...actual, + refreshStaleIdentities: vi.fn().mockResolvedValue(undefined), + }; +}); let db: Database; diff --git a/packages/contrail/tests/refresh.test.ts b/packages/contrail/tests/refresh.test.ts index 2f9179c..67f8e6f 100644 --- a/packages/contrail/tests/refresh.test.ts +++ b/packages/contrail/tests/refresh.test.ts @@ -8,21 +8,27 @@ import type { Database } from "../src/core/types"; // the `pages` map below. const pages = new Map<string, Array<{ uri: string; cid: string; value: object }>>(); -vi.mock("../src/core/client", () => ({ - getClient: vi.fn(async (did: string) => ({ - get: async ( - _method: string, - opts: { params: { repo: string; collection: string; cursor?: string } } - ) => { - const key = `${opts.params.repo}|${opts.params.collection}`; - // Single page per (did, collection); cursor triggers empty page = done. - if (opts.params.cursor) return { ok: true, data: { records: [], cursor: undefined } }; - const records = pages.get(key) ?? []; - return { ok: true, data: { records, cursor: undefined } }; - }, - })), - getPDS: vi.fn(), -})); +// `getClient` lives in @atmo-dev/contrail-base after the package split. +// `refresh` (now in contrail-appview) imports it via its own shim that +// ultimately resolves to base — so we mock the base export directly. +vi.mock("@atmo-dev/contrail-base", async (importOriginal) => { + const actual = await importOriginal<typeof import("@atmo-dev/contrail-base")>(); + return { + ...actual, + getClient: vi.fn(async (_did: string) => ({ + get: async ( + _method: string, + opts: { params: { repo: string; collection: string; cursor?: string } } + ) => { + const key = `${opts.params.repo}|${opts.params.collection}`; + if (opts.params.cursor) return { ok: true, data: { records: [], cursor: undefined } }; + const records = pages.get(key) ?? []; + return { ok: true, data: { records, cursor: undefined } }; + }, + })), + getPDS: vi.fn(), + }; +}); const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 9c32af0..996e91f 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -3,6 +3,7 @@ import path from "node:path"; const baseSrc = path.resolve(__dirname, "../contrail-base/src"); const authoritySrc = path.resolve(__dirname, "../contrail-authority/src"); +const recordHostSrc = path.resolve(__dirname, "../contrail-record-host/src"); export default defineConfig({ test: { @@ -18,6 +19,7 @@ export default defineConfig({ "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), "@atmo-dev/contrail-authority": path.join(authoritySrc, "index.ts"), + "@atmo-dev/contrail-record-host": path.join(recordHostSrc, "index.ts"), }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6e26e5a..2d68ee6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -376,12 +376,18 @@ importers: '@atcute/xrpc-server': specifier: ^0.1.12 version: 0.1.12 + '@atmo-dev/contrail-appview': + specifier: workspace:* + version: link:../contrail-appview '@atmo-dev/contrail-authority': specifier: workspace:* version: link:../contrail-authority '@atmo-dev/contrail-base': specifier: workspace:* version: link:../contrail-base + '@atmo-dev/contrail-record-host': + specifier: workspace:* + version: link:../contrail-record-host cac: specifier: ^7.0.0 version: 7.0.0 @@ -417,6 +423,58 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-appview: + dependencies: + '@atcute/atproto': + specifier: ^3.1.10 + version: 3.1.11 + '@atcute/cbor': + specifier: ^2.3.2 + version: 2.3.2 + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atcute/client': + specifier: ^4.2.1 + version: 4.2.1 + '@atcute/identity': + specifier: ^1.1.4 + version: 1.1.4 + '@atcute/identity-resolver': + specifier: ^1.2.2 + version: 1.2.2(@atcute/identity@1.1.4) + '@atcute/jetstream': + specifier: ^1.0.2 + version: 1.1.2 + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atcute/xrpc-server': + specifier: ^0.1.12 + version: 0.1.12 + '@atmo-dev/contrail-authority': + specifier: workspace:* + version: link:../contrail-authority + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base + '@atmo-dev/contrail-record-host': + specifier: workspace:* + version: link:../contrail-record-host + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + '@types/node': + specifier: ^25.5.0 + version: 25.6.0 + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/contrail-authority: dependencies: '@atcute/cid': @@ -522,6 +580,25 @@ importers: specifier: ^4.1.0 version: 4.1.5(@types/node@25.6.0)(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.6.1)(tsx@4.21.0)) + packages/contrail-record-host: + dependencies: + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/lexicons: dependencies: '@atcute/lex-cli': -- 2.51.2 From e7149dcd5ba61c2e8d44af77fe32c181b7e140a3 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 02:38:16 +0200 Subject: [PATCH 11/25] phase 7b --- packages/contrail-appview/src/index.ts | 11 + packages/contrail-appview/src/sync.ts | 229 ++++++++++++ packages/contrail-record-host/src/index.ts | 3 + packages/contrail-record-host/src/sync.ts | 331 ++++++++++++++++++ packages/contrail/tests/sync-e2e.test.ts | 304 ++++++++++++++++ packages/contrail/tests/sync-host.test.ts | 282 +++++++++++++++ .../lexicon-templates/recordHost/sync.json | 86 +++++ 7 files changed, 1246 insertions(+) create mode 100644 packages/contrail-appview/src/sync.ts create mode 100644 packages/contrail-record-host/src/sync.ts create mode 100644 packages/contrail/tests/sync-e2e.test.ts create mode 100644 packages/contrail/tests/sync-host.test.ts create mode 100644 packages/lexicons/lexicon-templates/recordHost/sync.json diff --git a/packages/contrail-appview/src/index.ts b/packages/contrail-appview/src/index.ts index 1aad674..dce992a 100644 --- a/packages/contrail-appview/src/index.ts +++ b/packages/contrail-appview/src/index.ts @@ -16,6 +16,17 @@ export * from "@atmo-dev/contrail-base"; export * from "@atmo-dev/contrail-authority"; export * from "@atmo-dev/contrail-record-host"; +// Record-sync ingestion (consumer side of recordHost.sync) +export { + runRecordHostSync, + buildRecordSyncSchema, + applyRecordSyncSchema, +} from "./sync"; +export type { + RecordHostSyncSource, + RecordHostSyncOptions, +} from "./sync"; + // Indexing pipeline (jetstream, persistent, backfill, refresh, ingest helpers) export * from "./core/jetstream"; export * from "./core/persistent"; diff --git a/packages/contrail-appview/src/sync.ts b/packages/contrail-appview/src/sync.ts new file mode 100644 index 0000000..e2526fc --- /dev/null +++ b/packages/contrail-appview/src/sync.ts @@ -0,0 +1,229 @@ +/** Appview-side ingestion loop for the recordHost.sync streaming endpoint. + * + * Opens an SSE connection to a remote host's `<ns>.recordHost.sync` endpoint + * for a (host, space) pair, parses the event stream, writes each + * `record.created` / `record.deleted` event into the local record host's + * tables, and persists the cursor after every checkpoint. + * + * Designed to be called per-subscription. Reconnect logic is the caller's — + * this function returns when the stream ends or an error throws. Wrap it in + * a retry-with-backoff loop in your worker / persistent-process. */ + +import type { + ContrailConfig, + Database, + RecordHost, + SqlDialect, +} from "@atmo-dev/contrail-base"; +import { getDialect } from "@atmo-dev/contrail-base"; + +export interface RecordHostSyncSource { + /** Remote host's base URL, e.g. "https://contrail-a.example.com". */ + hostUrl: string; + /** Space we want to sync. */ + spaceUri: string; + /** Authority DID — used for auto-enrolling locally on first connect. */ + authorityDid: string; + /** Credential the appview presents to read this space's stream. */ + credential: string; + /** Sync endpoint NSID; defaults to "<config.namespace>.recordHost.sync". */ + endpointNsid?: string; +} + +export interface RecordHostSyncOptions { + /** Local DB the records go into (same DB the appview's RecordHost adapter uses). */ + db: Database; + /** Resolved config. Used to derive the remote endpoint NSID and to find + * collection short names for table writes. */ + config: ContrailConfig; + /** Local record host — the destination for ingested events. The function + * calls `putRecord` / `deleteRecord` / `enroll` on this. */ + recordHost: RecordHost; + /** fetch implementation — pass the remote host app's fetch directly for + * in-process tests; defaults to globalThis.fetch. */ + fetch?: typeof fetch; + /** Aborts the stream when triggered. */ + signal?: AbortSignal; + /** Called on each cursor checkpoint, after persistence. */ + onCursor?: (cursor: string) => void; +} + +/** Run sync for a single (host, space) source until the stream ends or the + * signal aborts. Reads the prior cursor from `record_sync_subscriptions` if + * present; persists the new cursor as it advances. Auto-enrolls the space + * locally on first connect using the source's `authorityDid`. */ +export async function runRecordHostSync( + source: RecordHostSyncSource, + options: RecordHostSyncOptions +): Promise<void> { + const fetchImpl = options.fetch ?? globalThis.fetch; + await ensureSyncSchema(options.db); + + // Auto-enroll the space so the local record host accepts subsequent + // queries against it. + const existing = await options.recordHost.getEnrollment(source.spaceUri); + if (!existing) { + await options.recordHost.enroll({ + spaceUri: source.spaceUri, + authorityDid: source.authorityDid, + enrolledAt: Date.now(), + enrolledBy: source.authorityDid, + }); + } + + // Resume from the last persisted cursor for this subscription. + const since = await readCursor(options.db, source.hostUrl, source.spaceUri); + + const endpoint = + source.endpointNsid ?? `${options.config.namespace}.recordHost.sync`; + const url = new URL(`${source.hostUrl}/xrpc/${endpoint}`); + url.searchParams.set("spaceUri", source.spaceUri); + if (since) url.searchParams.set("since", since); + + const res = await fetchImpl(url.toString(), { + headers: { + "X-Space-Credential": source.credential, + accept: "text/event-stream", + }, + signal: options.signal, + }); + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`recordHost.sync ${res.status}: ${body}`); + } + if (!res.body) { + throw new Error("recordHost.sync response has no body"); + } + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buf = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + while (true) { + const idx = buf.indexOf("\n\n"); + if (idx < 0) break; + const block = buf.slice(0, idx); + buf = buf.slice(idx + 2); + const dataLine = block.split("\n").find((l) => l.startsWith("data:")); + if (!dataLine) continue; + const json = dataLine.slice(5).trim(); + if (!json) continue; + let event: any; + try { + event = JSON.parse(json); + } catch { + continue; + } + await applyEvent(event, options); + if (event.kind === "cursor" && typeof event.value === "string") { + await persistCursor( + options.db, + source.hostUrl, + source.spaceUri, + event.value + ); + options.onCursor?.(event.value); + } + } + } + } finally { + await reader.cancel().catch(() => {}); + } +} + +async function applyEvent( + event: any, + options: RecordHostSyncOptions +): Promise<void> { + if (event.kind === "record.created") { + const p = event.payload; + await options.recordHost.putRecord({ + spaceUri: p.space, + collection: p.collection, + authorDid: p.did, + rkey: p.rkey, + cid: p.cid ?? null, + record: p.record ?? {}, + // The host's time_us is microseconds; createdAt on putRecord is ms-ish + // historically. Keep the host's ordering by passing it through; the + // host adapter writes time_us = createdAt * 1000 internally so this + // round-trips. We store the source's time_us directly to preserve + // ordering across hosts. + createdAt: p.time_us != null ? Math.floor(p.time_us / 1000) : Date.now(), + }); + } else if (event.kind === "record.deleted") { + const p = event.payload; + await options.recordHost.deleteRecord(p.space, p.collection, p.did, p.rkey); + } + // cursor events are handled by the caller for persistence +} + +// ---- Schema + cursor persistence ---- + +const SYNC_SCHEMA_APPLIED = new WeakSet<object>(); + +/** Idempotent: applies the `record_sync_subscriptions` table on first call + * for a given DB. Tracks per-DB (by reference) so repeated calls in tests + * don't re-issue DDL each time. */ +async function ensureSyncSchema(db: Database): Promise<void> { + if (SYNC_SCHEMA_APPLIED.has(db as unknown as object)) return; + const dialect = getDialect(db); + const stmts = buildRecordSyncSchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); + SYNC_SCHEMA_APPLIED.add(db as unknown as object); +} + +export function buildRecordSyncSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS record_sync_subscriptions ( + host_url TEXT NOT NULL, + space_uri TEXT NOT NULL, + cursor TEXT, + last_synced_at ${dialect.bigintType}, + PRIMARY KEY (host_url, space_uri) + )`, + ]; +} + +/** SchemaModule-shaped helper for `initSchema({ extraSchemas: [...] })`. */ +export async function applyRecordSyncSchema(db: Database): Promise<void> { + const dialect = getDialect(db); + const stmts = buildRecordSyncSchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); +} + +async function readCursor( + db: Database, + hostUrl: string, + spaceUri: string +): Promise<string | null> { + const row = await db + .prepare( + `SELECT cursor FROM record_sync_subscriptions WHERE host_url = ? AND space_uri = ?` + ) + .bind(hostUrl, spaceUri) + .first<{ cursor: string | null } | null>(); + return row?.cursor ?? null; +} + +async function persistCursor( + db: Database, + hostUrl: string, + spaceUri: string, + cursor: string +): Promise<void> { + await db + .prepare( + `INSERT INTO record_sync_subscriptions (host_url, space_uri, cursor, last_synced_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (host_url, space_uri) DO UPDATE SET + cursor = excluded.cursor, + last_synced_at = excluded.last_synced_at` + ) + .bind(hostUrl, spaceUri, cursor, Date.now()) + .run(); +} diff --git a/packages/contrail-record-host/src/index.ts b/packages/contrail-record-host/src/index.ts index ff4eacd..df57f36 100644 --- a/packages/contrail-record-host/src/index.ts +++ b/packages/contrail-record-host/src/index.ts @@ -25,3 +25,6 @@ export type { BlobGcOptions, BlobGcResult } from "./blob-gc"; export { collectBlobCids } from "./blob-refs"; export { registerRecordHostRoutes } from "./routes"; + +export { registerRecordHostSyncRoutes } from "./sync"; +export type { RecordHostSyncOptions, SyncEvent } from "./sync"; diff --git a/packages/contrail-record-host/src/sync.ts b/packages/contrail-record-host/src/sync.ts new file mode 100644 index 0000000..00af1e1 --- /dev/null +++ b/packages/contrail-record-host/src/sync.ts @@ -0,0 +1,331 @@ +/** Record-host sync endpoint — streams record events for a specific space. + * + * Two phases per connection: + * 1. **Catch-up**: scan the per-collection `spaces_records_<short>` tables + * for `time_us > since`, emit each row as `record.created`. After each + * batch, emit a `cursor` checkpoint so the client can persist progress. + * 2. **Live**: subscribe to the in-process pubsub for `space:<uri>` and + * forward record.created / record.deleted events. + * + * Phase 7b MVP limitations: + * - Catch-up only sees `record.created`; deletions in the past aren't + * replayed (the row is gone). Live deletions are emitted. + * - Brief race between catch-up end and live subscribe — if a write + * lands in that window, the next reconnect catches it via catch-up. + * - SSE only (no WS). Simpler. WS can be added if needed. + * + * Auth: requires a valid X-Space-Credential whose `space` claim matches + * the requested spaceUri and whose scope is `read` or `rw`. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { + ContrailConfig, + CredentialClaims, + CredentialVerifier, + PubSub, + RealtimeEvent, + RecordHost, +} from "@atmo-dev/contrail-base"; +import { + DEFAULT_KEEPALIVE_MS, + extractSpaceCredential, + shortNameForNsid, + spacesRecordsTableName, + spaceTopic, +} from "@atmo-dev/contrail-base"; +import type { Database } from "@atmo-dev/contrail-base"; + +/** Events emitted on the wire. RealtimeEvent kinds (record.created / + * record.deleted) plus our own `cursor` checkpoint. */ +export type SyncEvent = + | RealtimeEvent + | { kind: "cursor"; value: string }; + +export interface RecordHostSyncOptions { + /** Database the host's record tables live on. */ + db: Database; + /** Optional pubsub for live mode. When omitted, sync is catch-up only — + * the stream ends after catch-up rather than tailing for new writes. */ + pubsub?: PubSub | null; + /** Required: verifier for the X-Space-Credential header. */ + credentialVerifier: CredentialVerifier; + /** Page size for catch-up scans. Default 100. */ + batchSize?: number; + /** SSE keepalive interval in ms. Default uses the realtime module's. */ + keepaliveMs?: number; +} + +export function registerRecordHostSyncRoutes( + app: Hono, + recordHost: RecordHost, + config: ContrailConfig, + options: RecordHostSyncOptions +): void { + const RECORD_HOST = `${config.namespace}.recordHost`; + const batchSize = options.batchSize ?? 100; + const keepaliveMs = options.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + + app.get(`/xrpc/${RECORD_HOST}.sync`, async (c) => { + // ---- Auth: credential required ---- + const credToken = extractSpaceCredential(c.req.raw); + if (!credToken) { + return c.json( + { error: "AuthRequired", reason: "credential-required" }, + 401 + ); + } + const verified = await options.credentialVerifier.verify(credToken); + if (!verified.ok) { + return c.json({ error: "AuthRequired", reason: verified.reason }, 401); + } + const claims = verified.claims; + + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) { + return c.json( + { error: "InvalidRequest", message: "spaceUri required" }, + 400 + ); + } + if (claims.space !== spaceUri) { + return c.json( + { error: "Forbidden", reason: "credential-wrong-space" }, + 403 + ); + } + + const enrollment = await recordHost.getEnrollment(spaceUri); + if (!enrollment) { + return c.json( + { error: "NotFound", reason: "not-enrolled" }, + 404 + ); + } + + const since = parseSince(c.req.query("since")); + + // Build the SSE response with a hand-rolled stream so we can interleave + // catch-up batches and live events under one cursor sequence. + const ac = new AbortController(); + c.req.raw.signal.addEventListener("abort", () => ac.abort(), { once: true }); + + const stream = new ReadableStream<Uint8Array>({ + async start(controller) { + const enc = new TextEncoder(); + let closed = false; + const close = () => { + if (closed) return; + closed = true; + try { controller.close(); } catch { /* already closed */ } + }; + ac.signal.addEventListener("abort", close, { once: true }); + + const keepalive = setInterval(() => { + if (closed) return; + try { + controller.enqueue(enc.encode(`: keepalive\n\n`)); + } catch { + close(); + } + }, keepaliveMs); + + const writeEvent = (e: SyncEvent) => { + if (closed) return false; + try { + controller.enqueue(enc.encode(frameEvent(e))); + return true; + } catch { + close(); + return false; + } + }; + + try { + controller.enqueue(enc.encode(`: open\n\n`)); + + // ---- Phase 1: catch-up ---- + const lastCursor = await streamCatchup({ + db: options.db, + config, + spaceUri, + since, + batchSize, + writeEvent, + isClosed: () => closed, + }); + if (closed) return; + + // ---- Phase 2: live ---- + if (options.pubsub) { + const liveCutoff = lastCursor ?? since; + for await (const event of options.pubsub.subscribe( + spaceTopic(spaceUri), + ac.signal + )) { + if (closed) break; + // Ignore events older than what catch-up already covered. + const ts = (event as any).payload?.time_us ?? null; + if ( + liveCutoff != null && + typeof ts === "number" && + ts <= liveCutoff + ) { + continue; + } + if ( + event.kind !== "record.created" && + event.kind !== "record.deleted" + ) { + continue; + } + const ok = writeEvent(event); + if (!ok) break; + // Emit a cursor checkpoint after each live event so consumers + // can resume from the latest seen point. + if (typeof ts === "number") { + writeEvent({ kind: "cursor", value: String(ts) }); + } + } + } + } catch (err) { + if (!closed) { + try { + controller.enqueue( + enc.encode( + `event: error\ndata: ${JSON.stringify({ + message: err instanceof Error ? err.message : String(err), + })}\n\n` + ) + ); + } catch { + /* already torn down */ + } + } + } finally { + clearInterval(keepalive); + close(); + } + }, + cancel() { + ac.abort(); + }, + }); + + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + }, + }); + }); +} + +/** Catch-up phase: scan every configured per-collection spaces_records table + * for rows with `time_us > since`, emit them in time_us order. Returns the + * highest time_us emitted (or null if nothing emitted). */ +async function streamCatchup(args: { + db: Database; + config: ContrailConfig; + spaceUri: string; + since: number | null; + batchSize: number; + writeEvent: (e: SyncEvent) => boolean; + isClosed: () => boolean; +}): Promise<number | null> { + const { db, config, spaceUri, since, batchSize, writeEvent, isClosed } = args; + let highest = since; + + for (const [_short, colCfg] of Object.entries(config.collections)) { + if (colCfg.allowInSpaces === false) continue; + if (isClosed()) return highest; + + const collectionNsid = colCfg.collection; + const short = shortNameForNsid(config, collectionNsid); + if (!short) continue; + const table = spacesRecordsTableName(short); + + let cursor = since; + while (true) { + if (isClosed()) return highest; + + let rows: any[]; + try { + const sql = + cursor != null + ? `SELECT * FROM ${table} WHERE space_uri = ? AND time_us > ? ORDER BY time_us LIMIT ?` + : `SELECT * FROM ${table} WHERE space_uri = ? ORDER BY time_us LIMIT ?`; + const result = await (cursor != null + ? db.prepare(sql).bind(spaceUri, cursor, batchSize).all<any>() + : db.prepare(sql).bind(spaceUri, batchSize).all<any>()); + rows = result.results; + } catch { + // Table missing — skip this collection silently. + break; + } + + if (rows.length === 0) break; + + for (const row of rows) { + if (isClosed()) return highest; + const time_us = numericish(row.time_us); + const event: RealtimeEvent = { + topic: spaceTopic(spaceUri), + kind: "record.created", + payload: { + uri: row.uri, + did: row.did, + collection: collectionNsid, + rkey: row.rkey, + cid: row.cid ?? null, + record: parseRecordJson(row.record), + time_us, + space: spaceUri, + }, + ts: Date.now(), + }; + if (!writeEvent(event)) return highest; + if (highest == null || time_us > highest) highest = time_us; + cursor = time_us; + } + + // Cursor checkpoint after the batch. + if (highest != null) { + writeEvent({ kind: "cursor", value: String(highest) }); + } + + if (rows.length < batchSize) break; + } + } + + return highest; +} + +function frameEvent(event: SyncEvent): string { + return `event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`; +} + +function parseSince(raw: string | undefined): number | null { + if (!raw) return null; + const n = Number(raw); + if (Number.isNaN(n) || !Number.isFinite(n)) return null; + return n; +} + +function numericish(v: unknown): number { + return typeof v === "string" ? Number(v) : (v as number); +} + +function parseRecordJson(value: unknown): Record<string, unknown> { + if (value == null) return {}; + if (typeof value === "string") { + try { + return JSON.parse(value) as Record<string, unknown>; + } catch { + return {}; + } + } + return value as Record<string, unknown>; +} diff --git a/packages/contrail/tests/sync-e2e.test.ts b/packages/contrail/tests/sync-e2e.test.ts new file mode 100644 index 0000000..154385f --- /dev/null +++ b/packages/contrail/tests/sync-e2e.test.ts @@ -0,0 +1,304 @@ +/** End-to-end multi-host sync test. + * + * Two Contrail instances (host A and appview B) running in one process, + * each with its own SQLite DB. Host A creates a space and writes records. + * Appview B opens a sync stream against host A and ingests the records + * into its own tables. We verify B can query the records locally. + * + * The "network" between them is host A's hono fetch, threaded through + * appview B's fetch parameter. Real deployments would use the actual + * fetch over HTTPS. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { HostedAdapter } from "../src/core/spaces/adapter"; +import { registerRecordHostSyncRoutes } from "@atmo-dev/contrail-record-host"; +import { runRecordHostSync, applyRecordSyncSchema } from "@atmo-dev/contrail-appview"; +import { + generateAuthoritySigningKey, + issueCredential, + createInProcessVerifier, + InMemoryPubSub, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; +import { wrapWithPublishing } from "../src/core/realtime/publishing-adapter"; + +const ALICE = "did:plc:alice"; +const SERVICE_DID = "did:web:test.example#svc"; +const SPACE_TYPE = "tools.atmo.event.space"; +const SPACE_KEY = "main"; +const SPACE_URI = `ats://${ALICE}/${SPACE_TYPE}/${SPACE_KEY}`; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +const HOST_CONFIG: ContrailConfig = { + namespace: "test.sync", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: SPACE_TYPE, serviceDid: SERVICE_DID }, + recordHost: {}, + }, +}; + +async function makeHost(): Promise<{ + app: Hono; + adapter: HostedAdapter; + db: any; +}> { + const db = createSqliteDatabase(":memory:"); + const cfg = { ...HOST_CONFIG }; + cfg.spaces!.authority!.signing = SIGNING; + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + + const baseAdapter = new HostedAdapter(db, resolved); + const pubsub = new InMemoryPubSub(); + const adapter = wrapWithPublishing(baseAdapter, pubsub) as HostedAdapter; + + // Provision a space + enroll on this host. + await adapter.createSpace({ + uri: SPACE_URI, + ownerDid: ALICE, + type: SPACE_TYPE, + key: SPACE_KEY, + serviceDid: SERVICE_DID, + appPolicyRef: null, + appPolicy: null, + }); + await adapter.addMember(SPACE_URI, ALICE, ALICE); + await adapter.enroll({ + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + enrolledAt: Date.now(), + enrolledBy: ALICE, + }); + + const app = new Hono(); + const verifier = createInProcessVerifier({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }); + registerRecordHostSyncRoutes(app, adapter, resolved, { + db, + pubsub, + credentialVerifier: verifier, + keepaliveMs: 60_000, + batchSize: 50, + }); + + return { app, adapter, db }; +} + +async function makeAppview(): Promise<{ db: any; adapter: HostedAdapter; resolved: any }> { + const db = createSqliteDatabase(":memory:"); + const cfg = { ...HOST_CONFIG }; + cfg.spaces!.authority!.signing = SIGNING; + const resolved = resolveConfig(cfg); + await initSchema(db, resolved, { + extraSchemas: [applyRecordSyncSchema], + }); + const adapter = new HostedAdapter(db, resolved); + return { db, adapter, resolved }; +} + +async function mintCredentialForAlice(): Promise<string> { + const { credential } = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: SPACE_URI, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + return credential; +} + +describe("recordHost.sync — end-to-end (host → appview)", () => { + it("appview ingests historical records into its own tables", async () => { + const host = await makeHost(); + const appview = await makeAppview(); + + // Plant 3 records on the host. + const t0 = Date.now(); + for (let i = 0; i < 3; i++) { + await host.adapter.putRecord({ + spaceUri: SPACE_URI, + collection: "app.event.message", + authorDid: ALICE, + rkey: `rk${i}`, + cid: null, + record: { $type: "app.event.message", text: `msg-${i}` }, + createdAt: t0 + i, + }); + } + + // Appview opens a sync stream with a 200ms abort — long enough for + // catch-up to drain, then we tear down before the live phase blocks. + const ac = new AbortController(); + const credential = await mintCredentialForAlice(); + const cursors: string[] = []; + + const syncPromise = runRecordHostSync( + { + hostUrl: "http://host", + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + credential, + }, + { + db: appview.db, + config: appview.resolved, + recordHost: appview.adapter, + // Route fetch to host A's hono app instead of the network. + fetch: ((input: any, init?: any) => { + const req = typeof input === "string" || input instanceof URL + ? new Request(input, init) + : (input as Request); + return host.app.fetch(req); + }) as typeof fetch, + signal: ac.signal, + onCursor: (c) => { + cursors.push(c); + // Once we've seen at least one cursor checkpoint, catch-up is + // making progress. Schedule abort to break out of live mode. + setTimeout(() => ac.abort(), 50); + }, + } + ).catch((err) => { + // AbortError is expected; rethrow other errors. + if (err.name !== "AbortError") throw err; + }); + + await syncPromise; + + // Verify the appview's local tables now have the host's records. + const ingested = await appview.adapter.listRecords( + SPACE_URI, + "app.event.message" + ); + expect(ingested.records).toHaveLength(3); + expect(ingested.records.map((r) => r.rkey).sort()).toEqual([ + "rk0", + "rk1", + "rk2", + ]); + expect(cursors.length).toBeGreaterThan(0); + + // Cursor was persisted in the subscriptions table. + const subRow = await appview.db + .prepare( + `SELECT cursor FROM record_sync_subscriptions WHERE host_url = ? AND space_uri = ?` + ) + .bind("http://host", SPACE_URI) + .first<{ cursor: string | null }>(); + expect(subRow?.cursor).toBeTruthy(); + + // Auto-enrolled on the appview side. + const enrollment = await appview.adapter.getEnrollment(SPACE_URI); + expect(enrollment).toBeTruthy(); + expect(enrollment?.authorityDid).toBe(SERVICE_DID); + }); + + it("appview resumes from persisted cursor — no re-emission of old records", async () => { + const host = await makeHost(); + const appview = await makeAppview(); + const credential = await mintCredentialForAlice(); + + // First batch. + const t0 = Date.now(); + await host.adapter.putRecord({ + spaceUri: SPACE_URI, + collection: "app.event.message", + authorDid: ALICE, + rkey: "first", + cid: null, + record: { text: "first" }, + createdAt: t0, + }); + + const networkFetch = ((input: any, init?: any) => { + const req = typeof input === "string" || input instanceof URL + ? new Request(input, init) + : (input as Request); + return host.app.fetch(req); + }) as typeof fetch; + + const ac1 = new AbortController(); + const seenCount: { count: number } = { count: 0 }; + await runRecordHostSync( + { + hostUrl: "http://host", + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + credential, + }, + { + db: appview.db, + config: appview.resolved, + recordHost: appview.adapter, + fetch: networkFetch, + signal: ac1.signal, + onCursor: () => { + seenCount.count++; + setTimeout(() => ac1.abort(), 30); + }, + } + ).catch((err) => { + if (err.name !== "AbortError") throw err; + }); + + // Insert second batch on the host AFTER the first sync completed. + await host.adapter.putRecord({ + spaceUri: SPACE_URI, + collection: "app.event.message", + authorDid: ALICE, + rkey: "second", + cid: null, + record: { text: "second" }, + createdAt: t0 + 100, + }); + + // Second sync run — should pick up only the new record, using the + // persisted cursor. + const ac2 = new AbortController(); + const ingestedRkeys: string[] = []; + // Wrap putRecord to observe what gets re-ingested. + const origPut = appview.adapter.putRecord.bind(appview.adapter); + appview.adapter.putRecord = async (record) => { + ingestedRkeys.push(record.rkey); + return origPut(record); + }; + + await runRecordHostSync( + { + hostUrl: "http://host", + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + credential, + }, + { + db: appview.db, + config: appview.resolved, + recordHost: appview.adapter, + fetch: networkFetch, + signal: ac2.signal, + onCursor: () => setTimeout(() => ac2.abort(), 30), + } + ).catch((err) => { + if (err.name !== "AbortError") throw err; + }); + + // The second run should have ingested ONLY "second" (not re-ingested "first"). + expect(ingestedRkeys).toEqual(["second"]); + }); +}); diff --git a/packages/contrail/tests/sync-host.test.ts b/packages/contrail/tests/sync-host.test.ts new file mode 100644 index 0000000..e07e5a3 --- /dev/null +++ b/packages/contrail/tests/sync-host.test.ts @@ -0,0 +1,282 @@ +/** Tests for the recordHost.sync SSE endpoint. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { HostedAdapter } from "../src/core/spaces/adapter"; +import { registerRecordHostSyncRoutes } from "@atmo-dev/contrail-record-host"; +import { + generateAuthoritySigningKey, + issueCredential, + createInProcessVerifier, + InMemoryPubSub, + spaceTopic, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; +import { wrapWithPublishing } from "../src/core/realtime/publishing-adapter"; + +const ALICE = "did:plc:alice"; +const SERVICE_DID = "did:web:test.example#svc"; +const SPACE_TYPE = "tools.atmo.event.space"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +const CONFIG: ContrailConfig = { + namespace: "test.sync", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: SPACE_TYPE, + serviceDid: SERVICE_DID, + signing: undefined as any, // filled in beforeAll + }, + recordHost: {}, + }, +}; + +async function makeHost(): Promise<{ + app: Hono; + adapter: HostedAdapter; + pubsub: InMemoryPubSub; + spaceUri: string; +}> { + const db = createSqliteDatabase(":memory:"); + const cfg = { ...CONFIG }; + cfg.spaces!.authority!.signing = SIGNING; + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + + const baseAdapter = new HostedAdapter(db, resolved); + const pubsub = new InMemoryPubSub(); + // Wrap so writes also publish onto pubsub topics — mirrors what the + // umbrella createApp does in realtime mode. + const adapter = wrapWithPublishing(baseAdapter, pubsub) as HostedAdapter; + + // Create a space + enroll + const spaceUri = `ats://${ALICE}/${SPACE_TYPE}/main`; + await adapter.createSpace({ + uri: spaceUri, + ownerDid: ALICE, + type: SPACE_TYPE, + key: "main", + serviceDid: SERVICE_DID, + appPolicyRef: null, + appPolicy: null, + }); + await adapter.addMember(spaceUri, ALICE, ALICE); + await adapter.enroll({ + spaceUri, + authorityDid: SERVICE_DID, + enrolledAt: Date.now(), + enrolledBy: ALICE, + }); + + // Build the SSE app. + const app = new Hono(); + const verifier = createInProcessVerifier({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }); + registerRecordHostSyncRoutes(app, adapter, resolved, { + db, + pubsub, + credentialVerifier: verifier, + keepaliveMs: 60_000, + }); + + return { app, adapter, pubsub, spaceUri }; +} + +async function mintCredential(spaceUri: string): Promise<string> { + const { credential } = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: spaceUri, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + return credential; +} + +/** Read the SSE response, return the first N parsed events. */ +async function readEvents( + res: Response, + count: number, + timeoutMs = 2000 +): Promise<Array<{ kind: string; payload?: any; value?: string }>> { + const reader = res.body!.getReader(); + const decoder = new TextDecoder(); + const events: Array<{ kind: string; payload?: any; value?: string }> = []; + let buf = ""; + const start = Date.now(); + while (events.length < count) { + if (Date.now() - start > timeoutMs) { + throw new Error(`timeout waiting for ${count} events; got ${events.length}`); + } + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + while (true) { + const idx = buf.indexOf("\n\n"); + if (idx < 0) break; + const block = buf.slice(0, idx); + buf = buf.slice(idx + 2); + // Skip comment-only blocks (`: open`, `: keepalive`). + const dataLine = block + .split("\n") + .find((l) => l.startsWith("data:")); + if (!dataLine) continue; + const json = dataLine.slice(5).trim(); + if (!json) continue; + events.push(JSON.parse(json)); + if (events.length >= count) break; + } + } + await reader.cancel().catch(() => {}); + return events; +} + +describe("recordHost.sync — catch-up phase", () => { + it("emits historical records as record.created in time_us order", async () => { + const { app, adapter, spaceUri } = await makeHost(); + + // Plant 3 records with deterministic timestamps. + const now = Date.now(); + for (let i = 0; i < 3; i++) { + await adapter.putRecord({ + spaceUri, + collection: "app.event.message", + authorDid: ALICE, + rkey: `rk${i}`, + cid: null, + record: { $type: "app.event.message", text: `msg-${i}` }, + createdAt: now + i, + }); + } + + const credential = await mintCredential(spaceUri); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}`, + { headers: { "X-Space-Credential": credential } } + ) + ); + expect(res.status).toBe(200); + + // Expect 3 record.created + at least one cursor checkpoint. + const events = await readEvents(res, 4); + const records = events.filter((e) => e.kind === "record.created"); + expect(records).toHaveLength(3); + expect(records.map((e) => e.payload.rkey)).toEqual(["rk0", "rk1", "rk2"]); + expect(records.every((e) => e.payload.space === spaceUri)).toBe(true); + const cursors = events.filter((e) => e.kind === "cursor"); + expect(cursors.length).toBeGreaterThan(0); + }); + + it("respects the since cursor — records at or before are skipped", async () => { + const { app, adapter, spaceUri } = await makeHost(); + + const t0 = 1_700_000_000_000; + await adapter.putRecord({ + spaceUri, + collection: "app.event.message", + authorDid: ALICE, + rkey: "old", + cid: null, + record: { text: "old" }, + createdAt: t0, + }); + await adapter.putRecord({ + spaceUri, + collection: "app.event.message", + authorDid: ALICE, + rkey: "new", + cid: null, + record: { text: "new" }, + createdAt: t0 + 100, + }); + + const credential = await mintCredential(spaceUri); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}&since=${t0}`, + { headers: { "X-Space-Credential": credential } } + ) + ); + expect(res.status).toBe(200); + + const events = await readEvents(res, 2); + const records = events.filter((e) => e.kind === "record.created"); + expect(records).toHaveLength(1); + expect(records[0]!.payload.rkey).toBe("new"); + }); +}); + +describe("recordHost.sync — auth + enrollment guards", () => { + it("rejects without a credential", async () => { + const { app, spaceUri } = await makeHost(); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}` + ) + ); + expect(res.status).toBe(401); + expect((await res.json() as any).reason).toBe("credential-required"); + }); + + it("rejects credential whose space doesn't match", async () => { + const { app, spaceUri } = await makeHost(); + const wrongSpaceCred = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: "ats://did:plc:alice/x/y", + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}`, + { headers: { "X-Space-Credential": wrongSpaceCred.credential } } + ) + ); + expect(res.status).toBe(403); + expect((await res.json() as any).reason).toBe("credential-wrong-space"); + }); + + it("rejects un-enrolled spaces", async () => { + const { app } = await makeHost(); + const otherSpaceUri = `ats://${ALICE}/${SPACE_TYPE}/different`; + const cred = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: otherSpaceUri, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(otherSpaceUri)}`, + { headers: { "X-Space-Credential": cred.credential } } + ) + ); + expect(res.status).toBe(404); + expect((await res.json() as any).reason).toBe("not-enrolled"); + }); +}); diff --git a/packages/lexicons/lexicon-templates/recordHost/sync.json b/packages/lexicons/lexicon-templates/recordHost/sync.json new file mode 100644 index 0000000..795092f --- /dev/null +++ b/packages/lexicons/lexicon-templates/recordHost/sync.json @@ -0,0 +1,86 @@ +{ + "lexicon": 1, + "id": "tools.atmo.recordHost.sync", + "defs": { + "main": { + "type": "subscription", + "description": "SSE stream of record events for a specific space. Emits a catch-up phase (every record with `time_us > since`, in time_us order) followed by a live phase (record.created / record.deleted as they happen). Cursor checkpoints are emitted after each catch-up batch and after each live event so consumers can persist progress and resume on reconnect. Auth: requires X-Space-Credential whose `space` claim matches `spaceUri`.", + "parameters": { + "type": "params", + "required": ["spaceUri"], + "properties": { + "spaceUri": { + "type": "string", + "description": "Space to sync." + }, + "since": { + "type": "string", + "description": "Opaque cursor (last `value` from a `cursor` event). Server interprets as time_us; events with time_us > since are emitted." + } + } + }, + "message": { + "schema": { + "type": "union", + "refs": [ + "#recordCreated", + "#recordDeleted", + "#cursor" + ] + } + } + }, + "recordCreated": { + "type": "object", + "required": ["topic", "kind", "payload", "ts"], + "properties": { + "topic": { "type": "string" }, + "kind": { "type": "string", "const": "record.created" }, + "payload": { + "type": "object", + "required": ["uri", "did", "collection", "rkey", "record", "time_us", "space"], + "properties": { + "uri": { "type": "string" }, + "did": { "type": "string", "format": "did" }, + "collection": { "type": "string", "format": "nsid" }, + "rkey": { "type": "string" }, + "cid": { "type": "string", "format": "cid" }, + "record": { "type": "unknown" }, + "time_us": { "type": "integer" }, + "space": { "type": "string" } + } + }, + "ts": { "type": "integer" } + } + }, + "recordDeleted": { + "type": "object", + "required": ["topic", "kind", "payload", "ts"], + "properties": { + "topic": { "type": "string" }, + "kind": { "type": "string", "const": "record.deleted" }, + "payload": { + "type": "object", + "required": ["uri", "did", "collection", "rkey", "space"], + "properties": { + "uri": { "type": "string" }, + "did": { "type": "string", "format": "did" }, + "collection": { "type": "string", "format": "nsid" }, + "rkey": { "type": "string" }, + "space": { "type": "string" } + } + }, + "ts": { "type": "integer" } + } + }, + "cursor": { + "type": "object", + "required": ["kind", "value"], + "description": "Checkpoint: persist `value` and pass it as `since` on next reconnect to resume.", + "properties": { + "kind": { "type": "string", "const": "cursor" }, + "value": { "type": "string" } + } + } + } +} -- 2.51.2 From 00eb989887c3a9875c7ceedf7aef0c4bc1a7577e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 02:50:16 +0200 Subject: [PATCH 12/25] phase 7c --- packages/contrail-authority/src/routes.ts | 53 ++++ packages/contrail-base/src/index.ts | 3 + packages/contrail-base/src/spaces/manifest.ts | 164 +++++++++++ packages/contrail-base/src/spaces/types.ts | 10 + .../contrail/tests/spaces-manifest.test.ts | 260 ++++++++++++++++++ .../spaces/getMembershipManifest.json | 41 +++ 6 files changed, 531 insertions(+) create mode 100644 packages/contrail-base/src/spaces/manifest.ts create mode 100644 packages/contrail/tests/spaces-manifest.test.ts create mode 100644 packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json diff --git a/packages/contrail-authority/src/routes.ts b/packages/contrail-authority/src/routes.ts index fe80e37..369ac31 100644 --- a/packages/contrail-authority/src/routes.ts +++ b/packages/contrail-authority/src/routes.ts @@ -21,9 +21,11 @@ import { checkInviteReadGrant, decodeUnverifiedClaims, DEFAULT_CREDENTIAL_TTL_MS, + DEFAULT_MANIFEST_TTL_MS, extractInviteToken, hashInviteToken, issueCredential, + issueMembershipManifest, nextTid, verifyCredential, } from "@atmo-dev/contrail-base"; @@ -324,6 +326,57 @@ export function registerAuthorityRoutes( ); return c.json({ credential, expiresAt }); }); + + // ---- Membership manifest ---- + // + // Issues a signed list of every space the caller is a member of (or owns) + // according to this authority. Appviews carry it on inbound requests so + // unioned listRecords queries can be filtered without syncing the full + // member list. Same key material as credentials, different payload. + + app.post(`/xrpc/${SPACE}.getMembershipManifest`, auth, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign manifests" }, + 501 + ); + } + const sa = getAuth(c); + const cap = authorityConfig.manifestMaxSpaces ?? 500; + + // Page through listSpaces — owner-or-member union — up to the cap. + const seen = new Set<string>(); + const drain = async (scope: "owner" | "member"): Promise<void> => { + let cursor: string | undefined; + while (seen.size < cap) { + const result = await authority.listSpaces({ + ...(scope === "owner" ? { ownerDid: sa.issuer } : { memberDid: sa.issuer }), + cursor, + limit: Math.min(200, cap - seen.size), + }); + for (const s of result.spaces) { + if (s.deletedAt == null) seen.add(s.uri); + if (seen.size >= cap) break; + } + if (!result.cursor || seen.size >= cap) break; + cursor = result.cursor; + } + }; + await drain("owner"); + if (seen.size < cap) await drain("member"); + + const ttl = authorityConfig.manifestTtlMs ?? DEFAULT_MANIFEST_TTL_MS; + const { manifest, expiresAt } = await issueMembershipManifest( + { + iss: authorityConfig.serviceDid, + sub: sa.issuer, + spaces: [...seen], + ttlMs: ttl, + }, + authorityConfig.signing + ); + return c.json({ manifest, expiresAt, truncated: seen.size >= cap }); + }); } /** Verify a credential presented at refreshCredential. */ diff --git a/packages/contrail-base/src/index.ts b/packages/contrail-base/src/index.ts index e11f46c..1fde427 100644 --- a/packages/contrail-base/src/index.ts +++ b/packages/contrail-base/src/index.ts @@ -40,6 +40,9 @@ export * from "./spaces/acl"; // Credentials export * from "./spaces/credentials"; +// Membership manifests +export * from "./spaces/manifest"; + // Binding + key resolution export * from "./spaces/binding"; diff --git a/packages/contrail-base/src/spaces/manifest.ts b/packages/contrail-base/src/spaces/manifest.ts new file mode 100644 index 0000000..83eed8a --- /dev/null +++ b/packages/contrail-base/src/spaces/manifest.ts @@ -0,0 +1,164 @@ +/** Membership manifest: a short-lived signed list of spaces a caller is a + * member of, issued by an authority. Lets appviews filter unioned queries + * without syncing the full member list — each user carries their own + * bounded slice as they hit the appview. + * + * Same signing infrastructure as space credentials (ES256 JWT), different + * payload + endpoint. */ + +import { + signCredential, + verifyCredential, + decodeUnverifiedClaims, + type CredentialKeyMaterial, +} from "./credentials"; + +const ALG = "ES256"; +const TYP = "JWT"; +const DEFAULT_KEY_ID = "atproto_space_authority"; + +/** Default manifest TTL — same 2h as credentials. */ +export const DEFAULT_MANIFEST_TTL_MS = 2 * 60 * 60 * 1000; + +export interface MembershipManifestClaims { + /** Authority DID that issued (and signed) the manifest. */ + iss: string; + /** User DID this manifest is for. */ + sub: string; + /** Space URIs the user is a member of (according to this authority). */ + spaces: string[]; + /** Seconds since epoch. */ + iat: number; + exp: number; +} + +/** Sign a manifest using the authority's signing key. */ +export async function signMembershipManifest( + payload: MembershipManifestClaims, + key: CredentialKeyMaterial +): Promise<string> { + // signCredential internally builds the JWT given a CredentialClaims-like + // shape. The manifest payload has different fields (`spaces` instead of + // `space`/`scope`) so we hand-roll the JWT here using the same utilities. + const enc = new TextEncoder(); + const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; + const header = { alg: ALG, typ: TYP, kid }; + const head = base64urlEncode(enc.encode(JSON.stringify(header))); + const body = base64urlEncode(enc.encode(JSON.stringify(payload))); + const signingInput = `${head}.${body}`; + const privateKey = await crypto.subtle.importKey( + "jwk", + key.privateKey, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["sign"] + ); + const sig = await crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + privateKey, + enc.encode(signingInput) + ); + return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; +} + +/** Issue a manifest with current iat/exp. */ +export async function issueMembershipManifest( + args: Omit<MembershipManifestClaims, "iat" | "exp"> & { ttlMs: number }, + key: CredentialKeyMaterial +): Promise<{ manifest: string; expiresAt: number }> { + const now = Math.floor(Date.now() / 1000); + const expSec = now + Math.floor(args.ttlMs / 1000); + const payload: MembershipManifestClaims = { + iss: args.iss, + sub: args.sub, + spaces: args.spaces, + iat: now, + exp: expSec, + }; + const manifest = await signMembershipManifest(payload, key); + return { manifest, expiresAt: expSec * 1000 }; +} + +export type ManifestVerifyOk = { ok: true; claims: MembershipManifestClaims }; +export type ManifestVerifyErr = { + ok: false; + reason: "malformed" | "bad-alg" | "bad-signature" | "expired" | "not-yet-valid" | "unknown-issuer"; +}; + +export interface VerifyManifestOptions { + /** Resolve the issuer's verification key. */ + resolveKey: (iss: string, kid: string | undefined) => Promise<JsonWebKey | null>; + /** Time provider for tests. */ + now?: () => number; +} + +export async function verifyMembershipManifest( + jwt: string, + opts: VerifyManifestOptions +): Promise<ManifestVerifyOk | ManifestVerifyErr> { + const parts = jwt.split("."); + if (parts.length !== 3) return { ok: false, reason: "malformed" }; + const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; + + let header: { alg?: string; kid?: string }; + let claims: MembershipManifestClaims; + try { + header = JSON.parse(new TextDecoder().decode(base64urlDecode(headSeg))); + claims = JSON.parse(new TextDecoder().decode(base64urlDecode(bodySeg))); + } catch { + return { ok: false, reason: "malformed" }; + } + if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; + if (!Array.isArray(claims.spaces)) return { ok: false, reason: "malformed" }; + + const nowMs = (opts.now ?? Date.now)(); + const nowSec = Math.floor(nowMs / 1000); + if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; + if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; + + const jwk = await opts.resolveKey(claims.iss, header.kid); + if (!jwk) return { ok: false, reason: "unknown-issuer" }; + + const publicKey = await crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"] + ); + const sigBytes = base64urlDecode(sigSeg); + const enc = new TextEncoder(); + const valid = await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + sigBytes as BufferSource, + enc.encode(`${headSeg}.${bodySeg}`) + ); + if (!valid) return { ok: false, reason: "bad-signature" }; + return { ok: true, claims }; +} + +/** Peek at claims without verifying — useful for routing decisions. */ +export function decodeUnverifiedManifest(jwt: string): MembershipManifestClaims | null { + const parts = jwt.split("."); + if (parts.length !== 3) return null; + try { + return JSON.parse(new TextDecoder().decode(base64urlDecode(parts[1]!))) as MembershipManifestClaims; + } catch { + return null; + } +} + +// Local base64url helpers — mirror the credentials module so we don't expose +// these as public utilities. +function base64urlEncode(bytes: Uint8Array): string { + const s = btoa(String.fromCharCode(...bytes)); + return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} +function base64urlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} diff --git a/packages/contrail-base/src/spaces/types.ts b/packages/contrail-base/src/spaces/types.ts index 6561773..0e62f6c 100644 --- a/packages/contrail-base/src/spaces/types.ts +++ b/packages/contrail-base/src/spaces/types.ts @@ -52,6 +52,16 @@ export interface AuthorityConfig { signing?: CredentialKeyMaterial; /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ credentialTtlMs?: number; + /** Membership-manifest lifetime in ms. Manifests carry a user's full + * member-of list and let appviews filter unioned queries without syncing + * the authority's full member tables. Same key material as credentials. + * Defaults to {@link DEFAULT_MANIFEST_TTL_MS}. */ + manifestTtlMs?: number; + /** Maximum number of spaces returned in a manifest. The endpoint paginates + * through `listSpaces` up to this cap; users with more spaces get a + * truncated manifest (the remainder won't be unioned in queries). Defaults + * to 500. */ + manifestMaxSpaces?: number; } /** Configuration for the **record host** role: stores per-space records and diff --git a/packages/contrail/tests/spaces-manifest.test.ts b/packages/contrail/tests/spaces-manifest.test.ts new file mode 100644 index 0000000..3b600f5 --- /dev/null +++ b/packages/contrail/tests/spaces-manifest.test.ts @@ -0,0 +1,260 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + generateAuthoritySigningKey, + signMembershipManifest, + verifyMembershipManifest, + decodeUnverifiedManifest, + issueMembershipManifest, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const CHARLIE = "did:plc:charlie"; + +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function makeConfig(overrides?: Partial<ContrailConfig["spaces"] extends { authority?: infer A } ? A : never>): ContrailConfig { + return { + namespace: "test.man", + collections: { + message: { collection: "app.event.message" }, + }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + manifestTtlMs: 60_000, + ...(overrides ?? {}), + }, + recordHost: {}, + }, + }; +} + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; +} + +async function makeApp(cfg: ContrailConfig = makeConfig()): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); +} + +function call(app: Hono, method: string, path: string, did: string | null, body?: any) { + const headers: Record<string, string> = {}; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +async function createSpace(app: Hono, owner: string): Promise<string> { + const res = await call(app, "POST", "/xrpc/test.man.space.createSpace", owner, {}); + expect(res.status).toBe(200); + return ((await res.json()) as any).space.uri; +} + +describe("membership manifest — sign/verify primitives", () => { + it("round-trips via verifyMembershipManifest", async () => { + const { manifest } = await issueMembershipManifest( + { + iss: SERVICE_DID, + sub: ALICE, + spaces: ["ats://a/x/1", "ats://a/x/2"], + ttlMs: 60_000, + }, + SIGNING + ); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async (iss) => (iss === SERVICE_DID ? SIGNING.publicKey : null), + }); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.claims.iss).toBe(SERVICE_DID); + expect(result.claims.sub).toBe(ALICE); + expect(result.claims.spaces).toEqual(["ats://a/x/1", "ats://a/x/2"]); + } + }); + + it("rejects expired manifests", async () => { + const past = Math.floor(Date.now() / 1000) - 10; + const manifest = await signMembershipManifest( + { + iss: SERVICE_DID, + sub: ALICE, + spaces: ["ats://a/x/1"], + iat: past - 60, + exp: past, + }, + SIGNING + ); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("expired"); + }); + + it("rejects bad signatures", async () => { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: [], ttlMs: 60_000 }, + SIGNING + ); + const otherKey = await generateAuthoritySigningKey(); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async () => otherKey.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("bad-signature"); + }); + + it("rejects unknown issuer", async () => { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: [], ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async () => null, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); + + it("decodeUnverifiedManifest peeks without verifying", async () => { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: ["ats://a/x/1"], ttlMs: 60_000 }, + SIGNING + ); + const claims = decodeUnverifiedManifest(manifest); + expect(claims).not.toBeNull(); + expect(claims!.sub).toBe(ALICE); + expect(claims!.spaces).toEqual(["ats://a/x/1"]); + }); +}); + +describe("membership manifest — getMembershipManifest endpoint", () => { + it("returns a manifest covering owned + joined spaces", async () => { + const app = await makeApp(); + // Alice owns 2; Bob joins one of them. + const uriA1 = await createSpace(app, ALICE); + const uriA2 = await createSpace(app, ALICE); + await call(app, "POST", "/xrpc/test.man.space.addMember", ALICE, { + spaceUri: uriA1, + did: BOB, + }); + // Bob also owns one. + const uriB1 = await createSpace(app, BOB); + + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", BOB); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.manifest).toBeTypeOf("string"); + expect(body.expiresAt).toBeTypeOf("number"); + expect(body.truncated).toBe(false); + + // Verify signature + payload. + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async (iss) => (iss === SERVICE_DID ? SIGNING.publicKey : null), + }); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.claims.iss).toBe(SERVICE_DID); + expect(result.claims.sub).toBe(BOB); + expect(result.claims.spaces.sort()).toEqual([uriA1, uriB1].sort()); + } + + // Alice's membership shouldn't be visible to Bob's manifest. + if (result.ok) expect(result.claims.spaces).not.toContain(uriA2); + }); + + it("returns an empty spaces array for a user with no memberships", async () => { + const app = await makeApp(); + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", CHARLIE); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.claims.spaces).toEqual([]); + }); + + it("dedupes — owner who is also a member appears once", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + // createSpace internally addMembers the owner; both scopes will include it. + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", ALICE); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.claims.spaces).toEqual([uri]); + }); + + it("sets truncated:true when over manifestMaxSpaces", async () => { + const cfg = makeConfig(); + cfg.spaces!.authority!.manifestMaxSpaces = 2; + const app = await makeApp(cfg); + await createSpace(app, ALICE); + await createSpace(app, ALICE); + await createSpace(app, ALICE); + + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", ALICE); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.truncated).toBe(true); + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.claims.spaces.length).toBe(2); + }); + + it("requires auth", async () => { + const app = await makeApp(); + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", null); + expect(res.status).toBe(401); + }); + + it("returns 501 when authority is not configured to sign", async () => { + const cfg = makeConfig(); + delete cfg.spaces!.authority!.signing; + const app = await makeApp(cfg); + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", ALICE); + expect(res.status).toBe(501); + }); +}); diff --git a/packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json b/packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json new file mode 100644 index 0000000..7061df7 --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json @@ -0,0 +1,41 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.getMembershipManifest", + "defs": { + "main": { + "type": "procedure", + "description": "Mint a signed list of every space the caller is a member of (or owns) according to this authority. Appviews can carry the manifest on inbound requests so unioned listRecords queries are filtered against the caller's bounded slice without the appview having to sync the authority's full member list. Same key material as `getCredential`; different payload (an array of space URIs rather than a single space + scope). Truncated to `manifestMaxSpaces` (default 500) — the response sets `truncated: true` when the cap was hit.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": {} + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["manifest", "expiresAt", "truncated"], + "properties": { + "manifest": { + "type": "string", + "description": "Compact JWS (ES256) signed by the authority's key. Payload claims: `iss` (authority DID), `sub` (caller DID), `spaces` (array of space URIs), `iat`, `exp`." + }, + "expiresAt": { + "type": "integer", + "description": "Expiry as ms since epoch." + }, + "truncated": { + "type": "boolean", + "description": "True if the caller is in more spaces than fit in one manifest. The remainder won't be unioned in queries until the next refresh." + } + } + } + }, + "errors": [ + { "name": "NotImplemented", "description": "Authority is not configured to sign manifests." } + ] + } + } +} -- 2.51.2 From 30ce51aaa5fd18fc4204bc674b15cda7586e2988 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 19:24:15 +0200 Subject: [PATCH 13/25] phase 7d? --- .../src/core/router/collection.ts | 38 +++- .../contrail-appview/src/core/router/index.ts | 16 ++ packages/contrail-base/src/spaces/manifest.ts | 37 +++ .../tests/spaces-manifest-appview.test.ts | 210 ++++++++++++++++++ packages/contrail/vitest.config.ts | 2 + 5 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 packages/contrail/tests/spaces-manifest-appview.test.ts diff --git a/packages/contrail-appview/src/core/router/collection.ts b/packages/contrail-appview/src/core/router/collection.ts index 6d43d00..9482631 100644 --- a/packages/contrail-appview/src/core/router/collection.ts +++ b/packages/contrail-appview/src/core/router/collection.ts @@ -599,17 +599,45 @@ export function registerCollectionRoutes( // Union path: when the caller is authenticated, fold in records from // spaces they're a member of. Anonymous callers just get public results. + // + // The caller authenticates with service-auth (`Authorization: Bearer + // <jwt>`). Their member-of space list comes from one of: + // 1. `X-Membership-Manifest` header — signed list issued by some + // authority asserting `sub` is in these spaces. The manifest's + // `sub` MUST match the JWT's issuer (DID) — the manifest is not + // a bearer token. Preferred for multi-authority deployments. + // 2. Local listSpaces — appview asks its own authority adapter + // `listSpaces({ memberDid: jwt.issuer })`. Works when the appview + // operator IS the authority. let spaceUris: string[] | undefined; const hasAuthHeader = !!c.req.header("Authorization"); + const manifestHeader = c.req.header("X-Membership-Manifest"); if (spacesCtx) { const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); if (auth) { - const { spaces } = await spacesCtx.adapter.listSpaces({ - memberDid: auth.issuer, - limit: 200, - }); - spaceUris = spaces.map((s) => s.uri); + if (manifestHeader && spacesCtx.manifestVerifier) { + const verified = await spacesCtx.manifestVerifier.verify(manifestHeader); + if (!verified.ok) { + return c.json( + { error: "AuthRequired", reason: verified.reason, message: "invalid membership manifest" }, + 401 + ); + } + if (verified.claims.sub !== auth.issuer) { + return c.json( + { error: "Forbidden", reason: "manifest-sub-mismatch", message: "manifest sub does not match caller" }, + 403 + ); + } + spaceUris = verified.claims.spaces; + } else { + const { spaces } = await spacesCtx.adapter.listSpaces({ + memberDid: auth.issuer, + limit: 200, + }); + spaceUris = spaces.map((s) => s.uri); + } } else if (hasAuthHeader) { // Had an auth header but it was invalid — reject rather than // silently downgrading to public results. diff --git a/packages/contrail-appview/src/core/router/index.ts b/packages/contrail-appview/src/core/router/index.ts index 0c5e058..10e4990 100644 --- a/packages/contrail-appview/src/core/router/index.ts +++ b/packages/contrail-appview/src/core/router/index.ts @@ -12,6 +12,8 @@ import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; import { HostedAdapter } from "../spaces/adapter"; import type { StorageAdapter } from "../spaces/types"; import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import { createManifestVerifier } from "@atmo-dev/contrail-base"; +import type { ManifestVerifier } from "@atmo-dev/contrail-base"; import type { CommunityIntegration } from "../community-integration"; import { registerRealtimeRoutes } from "../realtime/router"; import type { RealtimeRoutesOptions } from "../realtime/router"; @@ -29,6 +31,12 @@ import type { MiddlewareHandler } from "hono"; export interface SpacesContext { adapter: StorageAdapter; verifier: ServiceJwtVerifier; + /** Verifies inbound `X-Membership-Manifest` headers. Built automatically + * when an authority is configured locally with signing keys; deployments + * that aggregate manifests from multiple authorities should construct one + * via {@link createManifestVerifier} with a custom key resolver and pass + * it through `options.spacesCtx`. */ + manifestVerifier?: ManifestVerifier; } export interface CreateAppOptions { @@ -131,6 +139,14 @@ export function createApp( ? { adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), verifier: buildVerifier(config.spaces.authority), + manifestVerifier: config.spaces.authority.signing + ? createManifestVerifier({ + resolveKey: async (iss) => + iss === config.spaces!.authority!.serviceDid + ? config.spaces!.authority!.signing!.publicKey + : null, + }) + : undefined, } : null; diff --git a/packages/contrail-base/src/spaces/manifest.ts b/packages/contrail-base/src/spaces/manifest.ts index 83eed8a..fbc8d31 100644 --- a/packages/contrail-base/src/spaces/manifest.ts +++ b/packages/contrail-base/src/spaces/manifest.ts @@ -138,6 +138,43 @@ export async function verifyMembershipManifest( return { ok: true, claims }; } +/** Stateful manifest verifier with a TTL'd in-memory cache keyed by JWT. + * Returns the same shape as {@link verifyMembershipManifest}; cache hits skip + * the crypto round-trip but still respect `exp`. Reuse one verifier per + * process — never construct per-request. */ +export interface ManifestVerifier { + verify(jwt: string): Promise<ManifestVerifyOk | ManifestVerifyErr>; +} + +/** Build a manifest verifier with caching. The cache is unbounded for now — + * manifests are short-lived (default 2h) and clients typically refresh them, + * so cardinality is bounded by active-user count. */ +export function createManifestVerifier(opts: VerifyManifestOptions): ManifestVerifier { + const cache = new Map<string, ManifestVerifyOk | ManifestVerifyErr>(); + const now = opts.now ?? Date.now; + return { + async verify(jwt) { + const cached = cache.get(jwt); + if (cached) { + if (cached.ok) { + if (cached.claims.exp * 1000 > now()) return cached; + cache.delete(jwt); + } else { + // Negative cache only for permanent failures (signature, alg); + // expired / not-yet-valid will roll over with the clock. + if (cached.reason === "bad-signature" || cached.reason === "bad-alg" || cached.reason === "malformed") { + return cached; + } + cache.delete(jwt); + } + } + const result = await verifyMembershipManifest(jwt, opts); + cache.set(jwt, result); + return result; + }, + }; +} + /** Peek at claims without verifying — useful for routing decisions. */ export function decodeUnverifiedManifest(jwt: string): MembershipManifestClaims | null { const parts = jwt.split("."); diff --git a/packages/contrail/tests/spaces-manifest-appview.test.ts b/packages/contrail/tests/spaces-manifest-appview.test.ts new file mode 100644 index 0000000..4d50cf7 --- /dev/null +++ b/packages/contrail/tests/spaces-manifest-appview.test.ts @@ -0,0 +1,210 @@ +/** Appview-side manifest consumption: cross-space listRecords union path + * honoring `X-Membership-Manifest`. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + generateAuthoritySigningKey, + issueMembershipManifest, + markInProcess, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const CHARLIE = "did:plc:charlie"; + +const SERVICE_DID = "did:web:test.example#svc"; +const SPACE_TYPE = "tools.atmo.event.space"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function makeConfig(): ContrailConfig { + return { + namespace: "test.man2", + collections: { + message: { collection: "app.event.message" }, + }, + spaces: { + authority: { + type: SPACE_TYPE, + serviceDid: SERVICE_DID, + signing: SIGNING, + }, + recordHost: {}, + }, + }; +} + +async function makeApp(): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved); +} + +/** Make a Request marked with an in-process principal so the union path's + * `verifyServiceAuthRequest` returns the right caller without minting a real + * JWT or wiring up a key resolver. */ +function inProc(url: string, did: string, headers: Record<string, string> = {}): Request { + const req = new Request(url, { headers }); + return markInProcess(req, did); +} + +async function createSpace(app: Hono, owner: string, key: string): Promise<string> { + const res = await app.fetch( + markInProcess( + new Request("http://localhost/xrpc/test.man2.space.createSpace", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ key }), + }), + owner + ) + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + return body.space.uri; +} + +async function plant( + app: Hono, + did: string, + spaceUri: string, + text: string +): Promise<void> { + const res = await app.fetch( + markInProcess( + new Request("http://localhost/xrpc/test.man2.space.putRecord", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + spaceUri, + collection: "app.event.message", + record: { $type: "app.event.message", text }, + }), + }), + did + ) + ); + expect(res.status).toBe(200); +} + +async function mintManifest(sub: string, spaces: string[]): Promise<string> { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub, spaces, ttlMs: 60_000 }, + SIGNING + ); + return manifest; +} + +describe("appview union listRecords — manifest-driven", () => { + it("uses manifest space list when valid + sub matches caller", async () => { + const app = await makeApp(); + // Alice owns one space, Bob owns another. + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + const bobSpace = await createSpace(app, BOB, "bob-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + await plant(app, BOB, bobSpace, "from-bob"); + + // Alice presents a manifest covering ONLY bobSpace (doesn't matter that + // she's not actually a member — the manifest is the source of truth here). + // The authority would never sign such a manifest in practice, but the + // appview's contract is: trust the verified manifest's claims. + const manifest = await mintManifest(ALICE, [bobSpace]); + + const res = await app.fetch( + inProc( + "http://localhost/xrpc/test.man2.message.listRecords", + ALICE, + { "X-Membership-Manifest": manifest } + ) + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const texts = body.records.map((r: any) => r.value.text); + expect(texts).toContain("from-bob"); + expect(texts).not.toContain("from-alice"); + }); + + it("rejects manifest whose sub does not match the caller", async () => { + const app = await makeApp(); + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + + // Bob's manifest, presented by Alice → should 403. + const bobManifest = await mintManifest(BOB, [aliceSpace]); + + const res = await app.fetch( + inProc( + "http://localhost/xrpc/test.man2.message.listRecords", + ALICE, + { "X-Membership-Manifest": bobManifest } + ) + ); + expect(res.status).toBe(403); + expect((await res.json() as any).reason).toBe("manifest-sub-mismatch"); + }); + + it("rejects an unsigned/forged manifest", async () => { + const app = await makeApp(); + const otherKey = await generateAuthoritySigningKey(); + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: [], ttlMs: 60_000 }, + otherKey + ); + const res = await app.fetch( + inProc( + "http://localhost/xrpc/test.man2.message.listRecords", + ALICE, + { "X-Membership-Manifest": manifest } + ) + ); + expect(res.status).toBe(401); + expect((await res.json() as any).reason).toBe("bad-signature"); + }); + + it("falls back to local listSpaces when no manifest is present", async () => { + const app = await makeApp(); + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + const bobSpace = await createSpace(app, BOB, "bob-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + await plant(app, BOB, bobSpace, "from-bob"); + + // Alice queries with no manifest → local listSpaces returns aliceSpace only. + const res = await app.fetch( + inProc("http://localhost/xrpc/test.man2.message.listRecords", ALICE) + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const texts = body.records.map((r: any) => r.value.text); + expect(texts).toContain("from-alice"); + expect(texts).not.toContain("from-bob"); + }); + + it("anonymous (no auth, no manifest) gets public results — no 401", async () => { + const app = await makeApp(); + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + + const res = await app.fetch( + new Request("http://localhost/xrpc/test.man2.message.listRecords") + ); + // No auth header, no manifest → drops through to anonymous public path. + expect(res.status).toBe(200); + const body = (await res.json()) as any; + // Private space records aren't visible publicly. + const texts = (body.records ?? []).map((r: any) => r.value?.text); + expect(texts).not.toContain("from-alice"); + }); +}); diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 996e91f..7f32f75 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -4,6 +4,7 @@ import path from "node:path"; const baseSrc = path.resolve(__dirname, "../contrail-base/src"); const authoritySrc = path.resolve(__dirname, "../contrail-authority/src"); const recordHostSrc = path.resolve(__dirname, "../contrail-record-host/src"); +const appviewSrc = path.resolve(__dirname, "../contrail-appview/src"); export default defineConfig({ test: { @@ -20,6 +21,7 @@ export default defineConfig({ "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), "@atmo-dev/contrail-authority": path.join(authoritySrc, "index.ts"), "@atmo-dev/contrail-record-host": path.join(recordHostSrc, "index.ts"), + "@atmo-dev/contrail-appview": path.join(appviewSrc, "index.ts"), }, }, }); -- 2.51.2 From 50d4771056a7bfe5f5aaadf53ab3e6f5e8723fe0 Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Mon, 4 May 2026 12:20:17 -0400 Subject: [PATCH 14/25] test(contrail-e2e): adopt post-PR30 spaces split + community integration Bring the e2e suite back to green under the package split: - makeSpacesConfig() wraps the old flat { type, serviceDid, resolver } shape under authority / recordHost and generates a fresh signing key per call. The config validator now requires spaces.authority whenever community is set, so this is the minimum viable shape for any test that uses spaces. - setupCommunityContrail() wires the community module via createCommunityIntegration({ db, config: resolveConfig(...) }) and passes the result as communityIntegration to the Contrail constructor. The community config field is now opaque to contrail core; routes are registered through the integration option. - 7 test files updated to use the helpers; 4 community tests pick up a workspace dep on @atmo-dev/contrail-community. 35/35 e2e tests pass against devnet. No source-package changes. --- apps/contrail-e2e/package.json | 1 + .../tests/community-delete.test.ts | 12 ++-- .../tests/community-invites.test.ts | 12 ++-- .../tests/community-lifecycle.test.ts | 12 ++-- .../tests/community-publishing.test.ts | 13 ++--- apps/contrail-e2e/tests/helpers.ts | 58 +++++++++++++++++++ apps/contrail-e2e/tests/spaces-auth.test.ts | 8 +-- .../spaces-firehose-invisibility.test.ts | 8 +-- .../tests/spaces-table-isolation.test.ts | 8 +-- pnpm-lock.yaml | 3 + 10 files changed, 85 insertions(+), 50 deletions(-) diff --git a/apps/contrail-e2e/package.json b/apps/contrail-e2e/package.json index ed43978..aa17c4f 100644 --- a/apps/contrail-e2e/package.json +++ b/apps/contrail-e2e/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@atmo-dev/contrail": "workspace:*", + "@atmo-dev/contrail-community": "workspace:*", "pg": "^8.20.0" }, "devDependencies": { diff --git a/apps/contrail-e2e/tests/community-delete.test.ts b/apps/contrail-e2e/tests/community-delete.test.ts index 4e2ffd3..0106c66 100644 --- a/apps/contrail-e2e/tests/community-delete.test.ts +++ b/apps/contrail-e2e/tests/community-delete.test.ts @@ -26,7 +26,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -34,6 +33,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, login, jsonOr, @@ -66,14 +66,10 @@ describe("community.delete e2e (soft-delete + cascade, real DB)", () => { cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/community-invites.test.ts b/apps/contrail-e2e/tests/community-invites.test.ts index e3a5fd0..58b0b31 100644 --- a/apps/contrail-e2e/tests/community-invites.test.ts +++ b/apps/contrail-e2e/tests/community-invites.test.ts @@ -27,7 +27,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -35,6 +34,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, login, jsonOr, @@ -80,14 +80,10 @@ describe("invite e2e (community + user-owned, real JWT)", () => { cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/community-lifecycle.test.ts b/apps/contrail-e2e/tests/community-lifecycle.test.ts index 137991c..7a40f60 100644 --- a/apps/contrail-e2e/tests/community-lifecycle.test.ts +++ b/apps/contrail-e2e/tests/community-lifecycle.test.ts @@ -25,7 +25,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -33,6 +32,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, login, jsonOr, @@ -78,14 +78,10 @@ describe("community lifecycle (mint → grant → list → revoke, + gap probes) cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/community-publishing.test.ts b/apps/contrail-e2e/tests/community-publishing.test.ts index 6087dd7..8a11f89 100644 --- a/apps/contrail-e2e/tests/community-publishing.test.ts +++ b/apps/contrail-e2e/tests/community-publishing.test.ts @@ -22,7 +22,7 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail, runPersistent } from "@atmo-dev/contrail"; +import { runPersistent } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -30,6 +30,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, createAppPasswordFor, devnetRewriteFetch, @@ -85,14 +86,10 @@ describe("community publishing (proxy → PDS → Jetstream → index)", () => { cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/helpers.ts b/apps/contrail-e2e/tests/helpers.ts index 64f493f..264bf67 100644 --- a/apps/contrail-e2e/tests/helpers.ts +++ b/apps/contrail-e2e/tests/helpers.ts @@ -12,6 +12,9 @@ import { PlcDidDocumentResolver, } from "@atcute/identity-resolver"; import type { Did as AtDid, Nsid } from "@atcute/lexicons"; +import { Contrail, generateAuthoritySigningKey, resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig, Database, SpacesConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; export type Did = `did:${string}:${string}`; @@ -46,6 +49,61 @@ export function createDevnetResolver() { }); } +/** + * Build a `spaces` config block in the post-PR30 split shape: authority owns + * ACL + credential signing, recordHost owns storage. Tests that previously + * passed the flat `{ type, serviceDid, resolver }` shape now get this — the + * config validator enforces the split, and `community` requires `authority`. + * + * Pass the `type` NSID for the kind of space (e.g. "rsvp.atmo.event.space"). + * A fresh signing key is generated per call so credential issuance works in + * the auth tests without leaking key material across suites. + */ +export async function makeSpacesConfig(type: string): Promise<SpacesConfig> { + return { + authority: { + type, + serviceDid: CONTRAIL_SERVICE_DID, + signing: await generateAuthoritySigningKey(), + resolver: createDevnetResolver(), + }, + recordHost: {}, + }; +} + +/** + * Build a Contrail wired with a community integration. Post-PR30, community + * routes are not registered by passing a `community` config block alone — the + * caller must construct a `CommunityIntegration` from the resolved config and + * pass it as `communityIntegration` to the `Contrail` constructor (the same + * pattern `createApp({ community })` uses in the contrail-community unit + * tests). + * + * `community` is forwarded into the Contrail config so the integration can + * read `masterKey`, `fetch`, etc. through `config.community`. + */ +export async function setupCommunityContrail(opts: { + db: Database; + baseConfig: ContrailConfig; + spaceType: string; + community: Record<string, unknown>; +}): Promise<Contrail> { + const fullConfig: ContrailConfig = { + ...opts.baseConfig, + spaces: await makeSpacesConfig(opts.spaceType), + community: opts.community, + }; + const integration = createCommunityIntegration({ + db: opts.db, + config: resolveConfig(fullConfig), + }); + return new Contrail({ + ...fullConfig, + db: opts.db, + communityIntegration: integration, + }); +} + /** * Mint an atproto service-auth JWT via the PDS's getServiceAuth endpoint. * Requires the client to already be authed for a user. Returns the raw JWT diff --git a/apps/contrail-e2e/tests/spaces-auth.test.ts b/apps/contrail-e2e/tests/spaces-auth.test.ts index 7e5124a..a9981fc 100644 --- a/apps/contrail-e2e/tests/spaces-auth.test.ts +++ b/apps/contrail-e2e/tests/spaces-auth.test.ts @@ -21,7 +21,7 @@ import { config as baseConfig } from "../config"; import { createTestAccount, createIsolatedSchema, - createDevnetResolver, + makeSpacesConfig, mintServiceAuthJwt, CONTRAIL_SERVICE_DID, PDS_URL, @@ -51,11 +51,7 @@ describe("spaces auth (devnet PDS JWT → Contrail verifier)", () => { const contrail = new Contrail({ ...baseConfig, db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + spaces: await makeSpacesConfig(SPACE_TYPE), }); await contrail.init(); handle = createHandler(contrail); diff --git a/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts b/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts index 07f7827..4d81933 100644 --- a/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts +++ b/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts @@ -24,7 +24,7 @@ import { config as baseConfig } from "../config"; import { createTestAccount, createIsolatedSchema, - createDevnetResolver, + makeSpacesConfig, mintServiceAuthJwt, CONTRAIL_SERVICE_DID, PDS_URL, @@ -57,11 +57,7 @@ describe("spaces firehose invisibility", () => { const contrail = new Contrail({ ...baseConfig, db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + spaces: await makeSpacesConfig(SPACE_TYPE), }); await contrail.init(); handle = createHandler(contrail); diff --git a/apps/contrail-e2e/tests/spaces-table-isolation.test.ts b/apps/contrail-e2e/tests/spaces-table-isolation.test.ts index 9a3c8c8..c61aed5 100644 --- a/apps/contrail-e2e/tests/spaces-table-isolation.test.ts +++ b/apps/contrail-e2e/tests/spaces-table-isolation.test.ts @@ -21,7 +21,7 @@ import { config as baseConfig } from "../config"; import { createTestAccount, createIsolatedSchema, - createDevnetResolver, + makeSpacesConfig, mintServiceAuthJwt, CONTRAIL_SERVICE_DID, PDS_URL, @@ -52,11 +52,7 @@ describe("spaces table isolation", () => { const contrail = new Contrail({ ...baseConfig, db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + spaces: await makeSpacesConfig(SPACE_TYPE), }); await contrail.init(); handle = createHandler(contrail); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d68ee6..adc192d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,6 +45,9 @@ importers: '@atmo-dev/contrail': specifier: workspace:* version: link:../../packages/contrail + '@atmo-dev/contrail-community': + specifier: workspace:* + version: link:../../packages/contrail-community pg: specifier: ^8.20.0 version: 8.20.0 -- 2.51.2 From 00094ee251fdc3ea3402fecf9441eea2f4f76146 Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 09:05:59 -0400 Subject: [PATCH 15/25] fix(record-host): require owner-signed enrollment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recordHost.enroll endpoint accepted either an owner-signed call OR a self-attested authority call. The latter let any DID claim to be the authority for any space's URI and rebind that space's authority. Drop the self-attesting-authority branch. Require the caller to be the space owner (sa.issuer === parts.ownerDid). The owner is still free to designate any authority via body.authority — only the legitimacy of the caller changes. Updates the existing test that codified the old behavior to assert the new rejection, and adds a positive test for the canonical owner-driven split-deployment enrollment flow. --- packages/contrail-record-host/src/routes.ts | 6 +-- .../contrail/tests/spaces-enrollment.test.ts | 46 ++++++++++++++++--- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/packages/contrail-record-host/src/routes.ts b/packages/contrail-record-host/src/routes.ts index 63fc951..25cd2ab 100644 --- a/packages/contrail-record-host/src/routes.ts +++ b/packages/contrail-record-host/src/routes.ts @@ -107,11 +107,9 @@ export function registerRecordHostRoutes( if (!parts) { return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); } - const callerIsOwner = sa.issuer === parts.ownerDid; - const callerIsAuthority = sa.issuer === body.authority; - if (!callerIsOwner && !callerIsAuthority) { + if (sa.issuer !== parts.ownerDid) { return c.json( - { error: "Forbidden", reason: "not-owner-or-authority" }, + { error: "Forbidden", reason: "not-owner" }, 403 ); } diff --git a/packages/contrail/tests/spaces-enrollment.test.ts b/packages/contrail/tests/spaces-enrollment.test.ts index 3b7ef5b..78cc8b0 100644 --- a/packages/contrail/tests/spaces-enrollment.test.ts +++ b/packages/contrail/tests/spaces-enrollment.test.ts @@ -127,7 +127,7 @@ describe("auto-enrollment via createSpace", () => { expect(((await reenroll.json()) as any).ok).toBe(true); }); - it("non-owner / non-authority callers cannot enroll", async () => { + it("non-owner callers cannot enroll", async () => { const app = await makeApp(); const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); const uri = ((await create.json()) as any).space.uri; @@ -137,7 +137,7 @@ describe("auto-enrollment via createSpace", () => { authority: SERVICE_DID, }); expect(res.status).toBe(403); - expect((await res.json()).reason).toBe("not-owner-or-authority"); + expect((await res.json()).reason).toBe("not-owner"); }); }); @@ -369,10 +369,10 @@ describe("split deployment — authority and record host on separate apps", () = expect((await list.json()).reason).toBe("not-enrolled"); }); - it("the declared authority can enroll a space without the owner's involvement", async () => { - // Scenario: the authority service ('Contrail' in our naming) is a - // separate DID and acts on behalf of an owner. The authority can enroll - // because phase 5 accepts either owner-or-authority. + it("a third party cannot enroll a space by claiming to be the authority", async () => { + // Regression: the enroll handler used to accept either owner-signed + // OR authority-self-attested calls. That let any DID claim "I am the + // authority for ats://<victim>/..." and rebind the space. const authorityDb = createSqliteDatabase(":memory:"); const hostDb = createSqliteDatabase(":memory:"); const cfg: ContrailConfig = { @@ -392,11 +392,43 @@ describe("split deployment — authority and record host on separate apps", () = const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); const uri = ((await create.json()) as any).space.uri; - // Authority service identifies itself via the JWT issuer matching its DID. + // SERVICE_DID self-attests as the authority for Alice's space. + // Must be rejected — only the owner can enroll. const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", SERVICE_DID, { spaceUri: uri, authority: SERVICE_DID, }); + expect(enroll.status).toBe(403); + expect((await enroll.json()).reason).toBe("not-owner"); + }); + + it("the owner can enroll their space designating a separate authority", async () => { + // Positive: the owner-signed path lets Alice point her space at the + // configured authority service (the canonical split-deployment flow). + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); expect(enroll.status).toBe(200); + expect(((await enroll.json()) as any).ok).toBe(true); }); }); -- 2.51.2 From be62232403137eed344abf923b183c75bcd9b41b Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 09:44:26 -0400 Subject: [PATCH 16/25] fix(spaces): drop LocalBindingResolver from default composite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default in-process credential verifier composed [Enrollment, Local], so a never-enrolled space still resolved its authority via LocalBindingResolver — collapsing the host-side consent layer the enrollment table is meant to provide. Default to enrollment-only. createLocalBindingResolver remains exported for explicit single-tenant wiring via options.credentialVerifier. --- packages/contrail-appview/src/core/spaces/router.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/contrail-appview/src/core/spaces/router.ts b/packages/contrail-appview/src/core/spaces/router.ts index 943f6a9..a0557f5 100644 --- a/packages/contrail-appview/src/core/spaces/router.ts +++ b/packages/contrail-appview/src/core/spaces/router.ts @@ -9,9 +9,7 @@ import { HostedAdapter } from "./adapter"; import { buildVerifier, createBindingCredentialVerifier, - createCompositeBindingResolver, createEnrollmentBindingResolver, - createLocalBindingResolver, createLocalKeyResolver, createServiceAuthMiddleware, } from "@atmo-dev/contrail-base"; @@ -71,18 +69,11 @@ export function registerSpacesRoutes( ); if (spacesConfig.recordHost) { - // Default in-process verifier: enrollment is the canonical binding - // source; Local-binding is a fallback for spaces created but not yet - // enrolled. Caller overrides via `options.credentialVerifier` to - // accept external authorities. const credentialVerifier = options.credentialVerifier ?? (authorityConfig.signing ? createBindingCredentialVerifier({ - bindings: createCompositeBindingResolver([ - createEnrollmentBindingResolver({ recordHost: adapter }), - createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), - ]), + bindings: createEnrollmentBindingResolver({ recordHost: adapter }), keys: createLocalKeyResolver({ authorityDid: authorityConfig.serviceDid, publicKey: authorityConfig.signing.publicKey, -- 2.51.2 From 428feead5f637e03666e60f9092d123bfaa87263 Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 10:05:44 -0400 Subject: [PATCH 17/25] fix(spaces): schema-validate PDS-record declarations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PDS binding resolver trusted body.value.authority on any record returned for the space's URI, with only a startsWith('did:') sniff. A malformed or attacker-shaped record at the same NSID could rebind authority — record content alone, no signature verification. Validate before trusting: - $type must equal the URI's type (the space-type lexicon embeds the declaration fields per tools.atmo.space.declaration's contract) - createdAt must be a string - authority must match a strict did:plc | did:web regex Fail closed on any deviation. Adds three negative tests covering each deviation path. --- packages/contrail-base/src/spaces/binding.ts | 12 +++- .../contrail/tests/spaces-binding.test.ts | 66 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/packages/contrail-base/src/spaces/binding.ts b/packages/contrail-base/src/spaces/binding.ts index a81b95f..2457f4d 100644 --- a/packages/contrail-base/src/spaces/binding.ts +++ b/packages/contrail-base/src/spaces/binding.ts @@ -141,10 +141,16 @@ export function createPdsBindingResolver(args: { } if (!res.ok) return null; const body = (await res.json().catch(() => null)) as - | { value?: { authority?: unknown } } + | { value?: { $type?: unknown; authority?: unknown; createdAt?: unknown } } | null; - const authority = body?.value?.authority; - return typeof authority === "string" && authority.startsWith("did:") ? authority : null; + const value = body?.value; + if (!value) return null; + if (value.$type !== parts.type) return null; + if (typeof value.createdAt !== "string") return null; + const authority = value.authority; + if (typeof authority !== "string") return null; + if (!/^did:(plc|web):[a-zA-Z0-9._:%-]+(#[a-zA-Z0-9._-]+)?$/.test(authority)) return null; + return authority; }, }; } diff --git a/packages/contrail/tests/spaces-binding.test.ts b/packages/contrail/tests/spaces-binding.test.ts index 427b6f2..d121df4 100644 --- a/packages/contrail/tests/spaces-binding.test.ts +++ b/packages/contrail/tests/spaces-binding.test.ts @@ -158,6 +158,72 @@ describe("BindingResolver — PDS record", () => { }); expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); }); + + it("returns null when the record's $type doesn't match the URI's type", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + value: { + $type: "com.attacker.fake.type", + authority: "did:web:authority.example", + createdAt: "2026-04-30T00:00:00Z", + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when createdAt is missing or non-string", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + value: { + $type: "com.example.event.space", + authority: "did:web:authority.example", + // createdAt deliberately omitted + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when authority isn't a well-formed DID", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + value: { + $type: "com.example.event.space", + authority: "did:fake!!://garbage", + createdAt: "2026-04-30T00:00:00Z", + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); }); describe("BindingResolver — DID-doc service entry", () => { -- 2.51.2 From dffbd36cb8f8855d990cd14532cbefbe6b09c46f Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 10:43:54 -0400 Subject: [PATCH 18/25] docs(auth): document binding-layer trust assumptions Two trust assumptions in the binding layer cannot be closed at the Contrail layer alone: - DID-doc service-entry rebind: deployments using createDidDocBindingResolver inherit PLC's rotation-key authorization model. Any rotation key on the owner's account can rewrite the #atproto_space_authority service entry. - PDS app password held for managed communities: community.provision and community.adopt hold unscoped app passwords that can write any record on the user-owned PDS, including the binding declaration read by createPdsBindingResolver. Both are constraints of their respective binding sources and onboarding flows, not bugs. Document each so operators picking a binding strategy can size them against their own threat model, with pointers to the canonical issues for the upstream/protocol-level work that would close them: #38 (DID-doc), #39 (PDS app password). --- docs/05-auth.md | 22 ++++++++++++++++++++++ docs/10-deployment-shapes.md | 9 +++++++++ 2 files changed, 31 insertions(+) diff --git a/docs/05-auth.md b/docs/05-auth.md index 7cb7578..6237fb8 100644 --- a/docs/05-auth.md +++ b/docs/05-auth.md @@ -190,6 +190,28 @@ No auth needed for: Public requests skip all verification middleware — no JWT parsing, no DID-doc fetch. Fast path. +## Trust assumptions + +Two trust assumptions in the binding layer cannot be closed at the Contrail layer alone. They are listed here so operators picking a binding strategy or onboarding flow can size them against their own threat model. + +### DID-doc binding path + +The record host can resolve a space's authority from three sources (see [Spaces § Discovery](./06-spaces.md#discovery--binding-resolution)): + +- **Local enrollment** — the host's own `record_host_enrollments` table, written via `recordHost.enroll`. Owner-signed; the host has full control over what's stored. +- **PDS record** — read from the owner's PDS at the space URI. +- **DID-doc service entry** — read from `service[id="#atproto_space_authority"]` on the owner's DID doc. + +The DID-doc path inherits PLC's authorization model: any rotation key on the owner's account can submit an update op rewriting the service entry. There is no per-entry signature or "this entry can only be edited by key X" constraint at the PLC layer. + +If the integrity of the space-authority binding needs to exceed what any one of the owner's rotation keys can already do, configure the host to resolve via local enrollment instead, or wait for an upstream signed-binding mechanism. Tracked upstream as [flo-bit/contrail#38](https://github.com/flo-bit/contrail/issues/38). + +### Contrail-held PDS app password + +Deployments running `community.provision` or `community.adopt` necessarily hold an ATProto app password for the user-owned PDS account so Contrail can write on the community's behalf. ATProto app passwords are unscoped at the PDS layer — they can write any record to the repo, including the `tools.atmo.space.declaration` record (or its embedded equivalent on a space-type record) that drives binding decisions in `createPdsBindingResolver`. + +This is an intentional consequence of the managed-community model, not a bug. Operators should treat stored app passwords with the same care as rotation keys, and reach for scoped app passwords (when ATProto adds them) or signed binding records to tighten the model further. Tracked upstream as [flo-bit/contrail#39](https://github.com/flo-bit/contrail/issues/39). + ## How the pieces fit A typical flow for a third-party app acting as a user in a space: diff --git a/docs/10-deployment-shapes.md b/docs/10-deployment-shapes.md index 25464ed..7b8675b 100644 --- a/docs/10-deployment-shapes.md +++ b/docs/10-deployment-shapes.md @@ -192,6 +192,15 @@ A deployment can act as the authority for spaces it owns *and* a record host for When in doubt, all-in-one. Splitting is for when you have a real operational reason to separate the two — different teams running them, different latency profiles, different scaling targets, different governance. +## Known trust assumptions + +Two assumptions in the binding layer cannot be closed at the Contrail layer alone, and which one applies depends on the shape you pick: + +- Deployments wiring `createDidDocBindingResolver` inherit PLC's rotation-key authorization model for the `#atproto_space_authority` service entry. Tracked as [flo-bit/contrail#38](https://github.com/flo-bit/contrail/issues/38). +- Deployments running `community.provision` or `community.adopt` hold an unscoped ATProto app password for each provisioned PDS account. Tracked as [flo-bit/contrail#39](https://github.com/flo-bit/contrail/issues/39). + +See [Auth § Trust assumptions](./05-auth.md#trust-assumptions) for the constraints in detail and what binding source to prefer when those assumptions don't fit your threat model. + ## What's not here - **Authority migration** — moving a space's authority from DID A to DID B. The architecture supports it (re-enroll on the host with the new authority binding) but no helper API yet. -- 2.51.2 From 5a238661f2c8e73ac875be094c716e311ad670c5 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:27:57 +0200 Subject: [PATCH 19/25] bump packages --- .changeset/config.json | 12 ++- .changeset/spaces-host-authority-split.md | 101 ------------------ packages/contrail-appview/CHANGELOG.md | 9 ++ packages/contrail-appview/package.json | 2 +- packages/contrail-authority/CHANGELOG.md | 7 ++ packages/contrail-authority/package.json | 2 +- packages/contrail-base/CHANGELOG.md | 3 + packages/contrail-base/package.json | 2 +- packages/contrail-community/CHANGELOG.md | 109 +++++++++++++++++++ packages/contrail-community/package.json | 2 +- packages/contrail-record-host/CHANGELOG.md | 7 ++ packages/contrail-record-host/package.json | 2 +- packages/contrail/CHANGELOG.md | 115 +++++++++++++++++++-- packages/contrail/package.json | 2 +- packages/lexicons/CHANGELOG.md | 10 +- packages/lexicons/package.json | 2 +- packages/sync/CHANGELOG.md | 104 ++++++++++++++++++- packages/sync/package.json | 2 +- 18 files changed, 371 insertions(+), 122 deletions(-) delete mode 100644 .changeset/spaces-host-authority-split.md create mode 100644 packages/contrail-appview/CHANGELOG.md create mode 100644 packages/contrail-authority/CHANGELOG.md create mode 100644 packages/contrail-base/CHANGELOG.md create mode 100644 packages/contrail-community/CHANGELOG.md create mode 100644 packages/contrail-record-host/CHANGELOG.md diff --git a/.changeset/config.json b/.changeset/config.json index 745d87f..070c0d5 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -2,8 +2,16 @@ "$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json", "changelog": "@changesets/cli/changelog", "commit": false, - "fixed": [], - "linked": [["@atmo-dev/contrail", "@atmo-dev/contrail-sync", "@atmo-dev/contrail-community"]], + "fixed": [[ + "@atmo-dev/contrail", + "@atmo-dev/contrail-community", + "@atmo-dev/contrail-sync", + "@atmo-dev/contrail-base", + "@atmo-dev/contrail-authority", + "@atmo-dev/contrail-record-host", + "@atmo-dev/contrail-appview" + ]], + "linked": [], "access": "public", "baseBranch": "main", "updateInternalDependencies": "patch", diff --git a/.changeset/spaces-host-authority-split.md b/.changeset/spaces-host-authority-split.md deleted file mode 100644 index e14e46c..0000000 --- a/.changeset/spaces-host-authority-split.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -"@atmo-dev/contrail": minor -"@atmo-dev/contrail-community": minor -"@atmo-dev/contrail-sync": minor ---- - -Spaces refactor: split authority + record host into independently runnable -roles, add space credentials, extract community into its own package. - -**Breaking — config shape** - -`spaces` is no longer flat — split into `authority` and `recordHost`: - -```ts -// before -spaces: { - type: "com.example.event.space", - serviceDid: "did:web:example.com", - blobs: { adapter, maxSize }, -} - -// after -spaces: { - authority: { - type: "com.example.event.space", - serviceDid: "did:web:example.com", - signing: await generateAuthoritySigningKey(), - }, - recordHost: { - blobs: { adapter, maxSize }, - }, -} -``` - -**Breaking — community moved to its own package** - -Community has been extracted to `@atmo-dev/contrail-community`. Wire it via -`createCommunityIntegration`: - -```ts -import { Contrail, resolveConfig } from "@atmo-dev/contrail"; -import { createCommunityIntegration } from "@atmo-dev/contrail-community"; - -const resolved = resolveConfig(config); -const communityIntegration = createCommunityIntegration({ db, config: resolved }); -const contrail = new Contrail({ ...config, communityIntegration }); -``` - -The community config (`config.community`) stays the same; only the wiring -moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, -`reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of -`@atmo-dev/contrail`. - -**New — space credentials (`X-Space-Credential`)** - -The space authority issues short-lived ES256 JWTs (default 2h TTL) via -`<ns>.space.getCredential` and `refreshCredential`. The record host accepts -them on read/write paths in lieu of per-request service-auth JWTs. Skips -DID-doc fetches and member checks; the credential's signature is the proof. - -Generate a signing key once at deploy time: - -```ts -import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; -const signing = await generateAuthoritySigningKey(); -// Store the JWK; pass to spaces.authority.signing. -``` - -**New — binding resolution** - -Verifiers can resolve "which authority signs for this space?" from three -sources, in order: local enrollment table, PDS records at -`at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service -entry, owner-self fallback. Lets user-owned DIDs authorize a third-party -authority via a normal PDS write — no DID-doc surgery. - -**New — independent deployments + enrollment** - -The authority and record host can run as separate processes/operators. -A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) -register a space onto a host. In-process deployments auto-enroll on -`createSpace`; nothing changes for single-instance setups. - -See `docs/10-deployment-shapes.md` for all-in-one / authority-only / -host-only configurations and when to choose each. - -**Migration** - -For most deployments running spaces today, the migration is: - -1. Update the config: split `spaces.{type, serviceDid, blobs}` into - `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. -2. Generate and store an authority signing key - (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. -3. If using community: install `@atmo-dev/contrail-community`, build - `createCommunityIntegration({ db, config })`, pass via - `new Contrail({ communityIntegration })` (or `createApp({ community })`). - -Existing service-auth JWT clients keep working as a fallback path. -Migrate to space credentials when convenient — exchange a JWT for a -credential once via `getCredential`, then reuse it. diff --git a/packages/contrail-appview/CHANGELOG.md b/packages/contrail-appview/CHANGELOG.md new file mode 100644 index 0000000..cd18649 --- /dev/null +++ b/packages/contrail-appview/CHANGELOG.md @@ -0,0 +1,9 @@ +# @atmo-dev/contrail-appview + +## 0.7.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 +- @atmo-dev/contrail-authority@0.7.0 +- @atmo-dev/contrail-record-host@0.7.0 diff --git a/packages/contrail-appview/package.json b/packages/contrail-appview/package.json index c76172b..f2627dd 100644 --- a/packages/contrail-appview/package.json +++ b/packages/contrail-appview/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-appview", - "version": "0.6.0", + "version": "0.7.0", "description": "Public-records appview for contrail — jetstream ingestion, backfill, query layer, feeds, labels, profiles, per-collection XRPC routes.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-authority/CHANGELOG.md b/packages/contrail-authority/CHANGELOG.md new file mode 100644 index 0000000..de82f64 --- /dev/null +++ b/packages/contrail-authority/CHANGELOG.md @@ -0,0 +1,7 @@ +# @atmo-dev/contrail-authority + +## 0.7.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 diff --git a/packages/contrail-authority/package.json b/packages/contrail-authority/package.json index 07c0477..a088146 100644 --- a/packages/contrail-authority/package.json +++ b/packages/contrail-authority/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-authority", - "version": "0.6.0", + "version": "0.7.0", "description": "Default space-authority implementation for contrail — member list, invites, app policy, credential issuance. Contrail's binary-membership ACL flavor; for ladder-style access levels see @atmo-dev/contrail-community.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-base/CHANGELOG.md b/packages/contrail-base/CHANGELOG.md new file mode 100644 index 0000000..e6ed047 --- /dev/null +++ b/packages/contrail-base/CHANGELOG.md @@ -0,0 +1,3 @@ +# @atmo-dev/contrail-base + +## 0.7.0 diff --git a/packages/contrail-base/package.json b/packages/contrail-base/package.json index e72201a..5535411 100644 --- a/packages/contrail-base/package.json +++ b/packages/contrail-base/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-base", - "version": "0.6.0", + "version": "0.7.0", "description": "Shared infrastructure for the contrail family of packages — interfaces (SpaceAuthority, RecordHost, CommunityIntegration), credential primitives, binding resolvers, realtime infra, schema scaffolding. No routes, no tables of its own.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-community/CHANGELOG.md b/packages/contrail-community/CHANGELOG.md new file mode 100644 index 0000000..163f0e5 --- /dev/null +++ b/packages/contrail-community/CHANGELOG.md @@ -0,0 +1,109 @@ +# @atmo-dev/contrail-community + +## 0.7.0 + +### Minor Changes + +- 7e3145b: Spaces refactor: split authority + record host into independently runnable + roles, add space credentials, extract community into its own package. + + **Breaking — config shape** + + `spaces` is no longer flat — split into `authority` and `recordHost`: + + ```ts + // before + spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, + } + + // after + spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, + } + ``` + + **Breaking — community moved to its own package** + + Community has been extracted to `@atmo-dev/contrail-community`. Wire it via + `createCommunityIntegration`: + + ```ts + import { Contrail, resolveConfig } from "@atmo-dev/contrail"; + import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ + db, + config: resolved, + }); + const contrail = new Contrail({ ...config, communityIntegration }); + ``` + + The community config (`config.community`) stays the same; only the wiring + moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, + `reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of + `@atmo-dev/contrail`. + + **New — space credentials (`X-Space-Credential`)** + + The space authority issues short-lived ES256 JWTs (default 2h TTL) via + `<ns>.space.getCredential` and `refreshCredential`. The record host accepts + them on read/write paths in lieu of per-request service-auth JWTs. Skips + DID-doc fetches and member checks; the credential's signature is the proof. + + Generate a signing key once at deploy time: + + ```ts + import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + const signing = await generateAuthoritySigningKey(); + // Store the JWK; pass to spaces.authority.signing. + ``` + + **New — binding resolution** + + Verifiers can resolve "which authority signs for this space?" from three + sources, in order: local enrollment table, PDS records at + `at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service + entry, owner-self fallback. Lets user-owned DIDs authorize a third-party + authority via a normal PDS write — no DID-doc surgery. + + **New — independent deployments + enrollment** + + The authority and record host can run as separate processes/operators. + A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) + register a space onto a host. In-process deployments auto-enroll on + `createSpace`; nothing changes for single-instance setups. + + See `docs/10-deployment-shapes.md` for all-in-one / authority-only / + host-only configurations and when to choose each. + + **Migration** + + For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. + 2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. + 3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + + Existing service-auth JWT clients keep working as a fallback path. + Migrate to space credentials when convenient — exchange a JWT for a + credential once via `getCredential`, then reuse it. + +### Patch Changes + +- Updated dependencies [7e3145b] + - @atmo-dev/contrail@0.7.0 + - @atmo-dev/contrail-base@0.7.0 diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index c6abcb2..ac7bc8a 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-community", - "version": "0.4.2", + "version": "0.7.0", "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-record-host/CHANGELOG.md b/packages/contrail-record-host/CHANGELOG.md new file mode 100644 index 0000000..bbc13ac --- /dev/null +++ b/packages/contrail-record-host/CHANGELOG.md @@ -0,0 +1,7 @@ +# @atmo-dev/contrail-record-host + +## 0.7.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 diff --git a/packages/contrail-record-host/package.json b/packages/contrail-record-host/package.json index 2182f37..b0f50f9 100644 --- a/packages/contrail-record-host/package.json +++ b/packages/contrail-record-host/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-record-host", - "version": "0.6.0", + "version": "0.7.0", "description": "Default record-host implementation for contrail — stores records and blobs for permissioned spaces, enforces local enrollment as the host's consent layer.", "type": "module", "sideEffects": false, diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 63e68f6..d404678 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,11 +1,119 @@ # @atmo-dev/contrail +## 0.7.0 + +### Minor Changes + +- 7e3145b: Spaces refactor: split authority + record host into independently runnable + roles, add space credentials, extract community into its own package. + + **Breaking — config shape** + + `spaces` is no longer flat — split into `authority` and `recordHost`: + + ```ts + // before + spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, + } + + // after + spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, + } + ``` + + **Breaking — community moved to its own package** + + Community has been extracted to `@atmo-dev/contrail-community`. Wire it via + `createCommunityIntegration`: + + ```ts + import { Contrail, resolveConfig } from "@atmo-dev/contrail"; + import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ + db, + config: resolved, + }); + const contrail = new Contrail({ ...config, communityIntegration }); + ``` + + The community config (`config.community`) stays the same; only the wiring + moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, + `reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of + `@atmo-dev/contrail`. + + **New — space credentials (`X-Space-Credential`)** + + The space authority issues short-lived ES256 JWTs (default 2h TTL) via + `<ns>.space.getCredential` and `refreshCredential`. The record host accepts + them on read/write paths in lieu of per-request service-auth JWTs. Skips + DID-doc fetches and member checks; the credential's signature is the proof. + + Generate a signing key once at deploy time: + + ```ts + import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + const signing = await generateAuthoritySigningKey(); + // Store the JWK; pass to spaces.authority.signing. + ``` + + **New — binding resolution** + + Verifiers can resolve "which authority signs for this space?" from three + sources, in order: local enrollment table, PDS records at + `at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service + entry, owner-self fallback. Lets user-owned DIDs authorize a third-party + authority via a normal PDS write — no DID-doc surgery. + + **New — independent deployments + enrollment** + + The authority and record host can run as separate processes/operators. + A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) + register a space onto a host. In-process deployments auto-enroll on + `createSpace`; nothing changes for single-instance setups. + + See `docs/10-deployment-shapes.md` for all-in-one / authority-only / + host-only configurations and when to choose each. + + **Migration** + + For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. + 2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. + 3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + + Existing service-auth JWT clients keep working as a fallback path. + Migrate to space credentials when convenient — exchange a JWT for a + credential once via `getCredential`, then reuse it. + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 +- @atmo-dev/contrail-authority@0.7.0 +- @atmo-dev/contrail-record-host@0.7.0 +- @atmo-dev/contrail-appview@0.7.0 + ## 0.6.0 ### Minor Changes - af24714: Add per-collection `recordFilter` and apply Jetstream `#identity` handle changes during ingest. - - `CollectionConfig.recordFilter?: (record) => boolean` runs against each create/update during ingest; returning false drops the record before it reaches the DB. Useful for narrowing high-volume collections to just the records you care about (e.g. only `app.bsky.feed.post` records mentioning a particular URL). Deletes are not filtered, so they still tear down any record the filter previously let through. Throws are caught, logged, and treated as drops. - Jetstream `#identity` events (handle changes) now flow through to the `identities` table via a new `applyIdentityEvent` helper. UPDATE-only — unknown DIDs are no-ops so we don't materialize partial rows lacking PDS. @@ -71,7 +179,6 @@ ``` what changed: - - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -98,7 +205,6 @@ ``` changes: - - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. @@ -138,7 +244,6 @@ - ad3a61d: add `contrail dev` — local dev wrapper for cloudflare workers deployments. replaces `wrangler dev --test-scheduled` + a separate cron-trigger script with one command. on start it: - 1. connects to your local D1 via wrangler's `getPlatformProxy`, inspects state 2. prompts to run `backfillAll` if no completed backfills exist yet 3. prompts to run `refresh` if the ingest cursor is older than 60 minutes (configurable with `--stale-after`) @@ -166,7 +271,6 @@ options: `binding` (D1 binding name, default `"DB"`), `lexicons` (see below), `onInit` (one-shot app-specific setup). **`/xrpc/<ns>.lexicons` endpoint + `contrail-lex pull-service`** lets consumer apps typegen against a deployed contrail over HTTP, no PDS or DNS required: - - `contrail-lex generate` now emits a barrel `lexicons/generated/index.ts` that imports every lexicon the deployment speaks: generated + pulled + custom. The pulled lexicons are needed so consumer typegen can resolve `$ref`s out of the generated schemas. - Pass `{ lexicons }` to `createWorker` (or `createHandler(contrail, { lexicons })`) and the service exposes them at `GET /xrpc/<namespace>.lexicons`. - From a consumer app: @@ -184,7 +288,6 @@ unlike `backfillAll`, it ignores the `backfills` state table and sweeps fresh. useful after jetstream outages or after leaving a dev deployment idle for days. each record in each configured collection is classified as: - - **missing** — PDS has it, DB doesn't - **stale update** — DB has it with a different CID, _and_ the DB row was written before the ignore window (default 60s, configurable) - **in sync** — same CID, or DB row is within the ignore window diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 0eec411..0219561 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.6.0", + "version": "0.7.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, diff --git a/packages/lexicons/CHANGELOG.md b/packages/lexicons/CHANGELOG.md index d5f674f..ab5abd0 100644 --- a/packages/lexicons/CHANGELOG.md +++ b/packages/lexicons/CHANGELOG.md @@ -1,5 +1,12 @@ # @atmo-dev/contrail-lexicons +## 0.4.7 + +### Patch Changes + +- Updated dependencies [7e3145b] + - @atmo-dev/contrail@0.7.0 + ## 0.4.6 ### Patch Changes @@ -53,7 +60,6 @@ ``` what changed: - - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -86,7 +92,6 @@ ``` changes: - - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. @@ -110,7 +115,6 @@ options: `binding` (D1 binding name, default `"DB"`), `lexicons` (see below), `onInit` (one-shot app-specific setup). **`/xrpc/<ns>.lexicons` endpoint + `contrail-lex pull-service`** lets consumer apps typegen against a deployed contrail over HTTP, no PDS or DNS required: - - `contrail-lex generate` now emits a barrel `lexicons/generated/index.ts` that imports every lexicon the deployment speaks: generated + pulled + custom. The pulled lexicons are needed so consumer typegen can resolve `$ref`s out of the generated schemas. - Pass `{ lexicons }` to `createWorker` (or `createHandler(contrail, { lexicons })`) and the service exposes them at `GET /xrpc/<namespace>.lexicons`. - From a consumer app: diff --git a/packages/lexicons/package.json b/packages/lexicons/package.json index 9cc3ea4..066e06c 100644 --- a/packages/lexicons/package.json +++ b/packages/lexicons/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-lexicons", - "version": "0.4.6", + "version": "0.4.7", "description": "Generate atproto lexicon JSON (and optionally TypeScript types via @atcute/lex-cli) from a Contrail config.", "type": "module", "files": [ diff --git a/packages/sync/CHANGELOG.md b/packages/sync/CHANGELOG.md index f570219..aa4b041 100644 --- a/packages/sync/CHANGELOG.md +++ b/packages/sync/CHANGELOG.md @@ -1,5 +1,107 @@ # @atmo-dev/contrail-sync +## 0.7.0 + +### Minor Changes + +- 7e3145b: Spaces refactor: split authority + record host into independently runnable + roles, add space credentials, extract community into its own package. + + **Breaking — config shape** + + `spaces` is no longer flat — split into `authority` and `recordHost`: + + ```ts + // before + spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, + } + + // after + spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, + } + ``` + + **Breaking — community moved to its own package** + + Community has been extracted to `@atmo-dev/contrail-community`. Wire it via + `createCommunityIntegration`: + + ```ts + import { Contrail, resolveConfig } from "@atmo-dev/contrail"; + import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ + db, + config: resolved, + }); + const contrail = new Contrail({ ...config, communityIntegration }); + ``` + + The community config (`config.community`) stays the same; only the wiring + moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, + `reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of + `@atmo-dev/contrail`. + + **New — space credentials (`X-Space-Credential`)** + + The space authority issues short-lived ES256 JWTs (default 2h TTL) via + `<ns>.space.getCredential` and `refreshCredential`. The record host accepts + them on read/write paths in lieu of per-request service-auth JWTs. Skips + DID-doc fetches and member checks; the credential's signature is the proof. + + Generate a signing key once at deploy time: + + ```ts + import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + const signing = await generateAuthoritySigningKey(); + // Store the JWK; pass to spaces.authority.signing. + ``` + + **New — binding resolution** + + Verifiers can resolve "which authority signs for this space?" from three + sources, in order: local enrollment table, PDS records at + `at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service + entry, owner-self fallback. Lets user-owned DIDs authorize a third-party + authority via a normal PDS write — no DID-doc surgery. + + **New — independent deployments + enrollment** + + The authority and record host can run as separate processes/operators. + A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) + register a space onto a host. In-process deployments auto-enroll on + `createSpace`; nothing changes for single-instance setups. + + See `docs/10-deployment-shapes.md` for all-in-one / authority-only / + host-only configurations and when to choose each. + + **Migration** + + For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. + 2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. + 3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + + Existing service-auth JWT clients keep working as a fallback path. + Migrate to space credentials when convenient — exchange a JWT for a + credential once via `getCredential`, then reuse it. + ## 0.4.0 ### Minor Changes @@ -26,7 +128,6 @@ ``` what changed: - - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -53,7 +154,6 @@ ``` changes: - - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. diff --git a/packages/sync/package.json b/packages/sync/package.json index ef9e485..9bd77e4 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-sync", - "version": "0.4.0", + "version": "0.7.0", "description": "Client-side reactive watch-store over contrail's watchRecords endpoints. SSE + WebSocket transports, optimistic updates, optional IndexedDB cache.", "type": "module", "sideEffects": false, -- 2.51.2 From d7e093663afacbc5a868e07aa4a8a82b1342d594 Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 5 Jun 2026 10:21:47 -0400 Subject: [PATCH 20/25] feat(contrail): private-network deployment + concurrent-init hardening (#44) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(contrail-base): networkOverrides config for private-network deployments Add optional ContrailConfig.networkOverrides with three subfields: plcUrl, slingshotUrl, additionalAllowedHosts. Plumb config through resolvePDS, getPDSViaDidDoc (inlined; module-level didResolver removed), getPDS, resolvePDSCached, getClient, and all five identity.ts functions (fetchAndSave, resolveIdentity, resolveIdentities, resolveActor, refreshStaleIdentities). Pass config through backfill.ts call sites (getClient + getPDS) and the persistent.ts:160 refreshStaleIdentities call. additionalAllowedHosts is exact + case-insensitive + port-agnostic; the default SSRF validator still runs for non-listed hosts. All changes are additive — omitting networkOverrides preserves current public-internet defaults. validatePdsUrl remains internal (no new exports beyond the networkOverrides field itself). * test(contrail-base): network overrides + SSRF regression coverage Add three test files exercising the networkOverrides plumbing: - client.test.ts: ContrailConfig.networkOverrides type, validatePdsUrl regression baseline (public HTTPS accepted; non-HTTPS + private CIDRs + localhost + link-local rejected), additionalAllowedHosts allowlist (allowed host accepted; non-listed hosts still rejected; port-agnostic match), getClient + getPDS + resolvePDS plumb-through with mocked fetch. - identity-config.test.ts: resolveIdentity, resolveIdentities, resolveActor, refreshStaleIdentities all route to override slingshot URL. - network-overrides.test.ts: integration test against node:http stub PLC and Slingshot servers. Asserts resolvePDS, getClient, and refreshStaleIdentities actually hit the configured override URLs end to end, exercising the full chain. The SSRF regression cases give the existing validator implicit baseline explicit coverage in this PR. * feat(contrail-base): apply networkOverrides.resolver to spaces credential verifier * feat(contrail-appview): apply networkOverrides to label-endpoint resolution * fix(contrail-appview): use IF NOT EXISTS on schema migrations to remove init race Concurrent `initSchema` calls previously raced on `ALTER TABLE ADD COLUMN` because the SQL had no `IF NOT EXISTS` clause, and the two consumer-site `try { ... } catch { /* Column already exists — ignore */ }` blocks silently swallowed *every* DDL error — masking missing tables, syntax errors, and type mismatches alongside the intended duplicate-column case. L3 replaces both swallows with a dialect-aware `addColumnIfNotExists` helper: - Postgres: emits `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` (atomic in PG). - SQLite: pre-checks `PRAGMA table_info`, then narrowly absorbs only the literal "duplicate column name" error from the still-possible PRAGMA->ALTER race. Every other error surfaces. The `MIGRATIONS` array is now structured `{ table, column, columnDef, target }` data routed through the same helper. `target: "spaces"` routes to `spacesDb` when one is configured (split-DB deployments), `target: "feeds"` is gated on feeds being configured so the absent `feed_backfills` table no longer silently fails. Tests at `packages/contrail/tests/schema-idempotency.test.ts` cover: - sequential and concurrent (3-way) `initSchema` calls - repeated init does not leave duplicate count columns - helper is a no-op when column already exists - helper surfaces "no such table" errors (the L3-removed swallow case) - `initSchema` propagates errors thrown from extension schema modules OpenMeet's consumer-side Postgres advisory lock around `contrail.init()` exists only to work around this race; with L3 in place that lock can be deleted in a follow-up. * test: L3 concurrent CREATE race on Postgres * chore: add changeset for private-network deployment support * fix(contrail-appview): thread networkOverrides config through all resolver/identity call sites The networkOverrides config param was threaded into the resolver/identity functions but dropped at ~8 appview call sites (live-ingest refresh cycle, on-demand refresh, and router getProfile/getFeed/collection/profiles/notify), so private-network deploys silently fell back to the public resolver and the un-widened SSRF guard on those paths. Pass the in-scope config at each site. Add a regression test driving an override config through the live-ingest refresh path (runIngestCycle) and a router actor-resolution path (getProfile), asserting the override slingshot is hit instead of the default. * refactor(contrail-base): dedup SSRF validator into shared validateExternalUrl labels/resolve.ts validateEndpointUrl duplicated contrail-base validatePdsUrl, and this PR had edited the additionalAllowedHosts allowlist into both copies — a future SSRF-rule fix applied to one copy would leave the other exploitable. Export a single validateExternalUrl(url, allowedHosts?) from contrail-base and consume it from both the PDS client and labeler-endpoint resolution. validateEndpointUrl stays exported as a thin alias for backward compat. Behavior is identical (covered by existing SSRF tests). Also document the PDS resolve cache's process-wide-config assumption (keyed by DID only; safe now that all callers thread the same config). * docs(contrail-base): document validateExternalUrl threat-model scope Note inline that the SSRF guard is best-effort literal-match only: no DNS resolution, partial IPv6 / encoded-IP coverage, egress network policy expected for fully untrusted resolver inputs. No behavior change. --- .changeset/private-network-overrides.md | 19 ++ .../contrail-appview/src/core/backfill.ts | 8 +- .../contrail-appview/src/core/db/schema.ts | 236 ++++++++++++++--- .../contrail-appview/src/core/jetstream.ts | 2 +- .../src/core/labels/resolve.ts | 69 +++-- .../src/core/labels/subscribe.ts | 34 ++- .../contrail-appview/src/core/persistent.ts | 2 +- packages/contrail-appview/src/core/refresh.ts | 2 +- .../src/core/router/collection.ts | 2 +- .../contrail-appview/src/core/router/feed.ts | 2 +- .../contrail-appview/src/core/router/index.ts | 4 +- .../src/core/router/notify.ts | 2 +- .../src/core/router/profiles.ts | 4 +- .../src/core/spaces/router.ts | 2 +- .../src/core/spaces/schema.ts | 15 +- packages/contrail-base/src/client.ts | 111 +++++--- packages/contrail-base/src/identity.ts | 27 +- packages/contrail-base/src/spaces/auth.ts | 18 +- packages/contrail-base/src/types.ts | 31 +++ .../contrail-community/src/integration.ts | 2 +- packages/contrail/tests/client.test.ts | 239 ++++++++++++++++++ .../contrail/tests/identity-config.test.ts | 87 +++++++ .../contrail/tests/labels-resolve.test.ts | 81 ++++++ .../tests/network-overrides-appview.test.ts | 96 +++++++ .../contrail/tests/network-overrides.test.ts | 189 ++++++++++++++ .../tests/postgres-concurrent-init.test.ts | 92 +++++++ .../contrail/tests/schema-idempotency.test.ts | 139 ++++++++++ packages/contrail/tests/spaces-auth.test.ts | 65 +++++ 28 files changed, 1450 insertions(+), 130 deletions(-) create mode 100644 .changeset/private-network-overrides.md create mode 100644 packages/contrail/tests/client.test.ts create mode 100644 packages/contrail/tests/identity-config.test.ts create mode 100644 packages/contrail/tests/labels-resolve.test.ts create mode 100644 packages/contrail/tests/network-overrides-appview.test.ts create mode 100644 packages/contrail/tests/network-overrides.test.ts create mode 100644 packages/contrail/tests/postgres-concurrent-init.test.ts create mode 100644 packages/contrail/tests/schema-idempotency.test.ts create mode 100644 packages/contrail/tests/spaces-auth.test.ts diff --git a/.changeset/private-network-overrides.md b/.changeset/private-network-overrides.md new file mode 100644 index 0000000..874794e --- /dev/null +++ b/.changeset/private-network-overrides.md @@ -0,0 +1,19 @@ +--- +"@atmo-dev/contrail-base": minor +"@atmo-dev/contrail-appview": minor +"@atmo-dev/contrail-community": minor +--- + +Private-network deployment support via a new optional `ContrailConfig.networkOverrides` block. + +`networkOverrides` carries three optional subfields, all defaulting to the current public-internet behavior (omit the block entirely and nothing changes): + +- **`resolver`** — a custom `DidDocumentResolver` used during DID-doc PDS fallback, labeler-endpoint resolution, and spaces service-auth JWT verification. Lets a deployment point at a private PLC mirror or inject a custom fetch (mTLS, retry, instrumentation). Trusted; not SSRF-checked. +- **`slingshotUrl`** — override the slingshot identity-resolver endpoint. Trusted; not SSRF-checked. +- **`additionalAllowedHosts`** — hostnames that bypass the default SSRF guard when validating a resolved PDS or labeler endpoint. Match is exact, case-insensitive, port-agnostic (e.g. `["pds.dev.svc.cluster.local"]`). This is the only knob that widens the validator; there is no "disable SSRF" flag. + +The overrides are threaded through PDS/identity resolution (`resolvePDS`, `getPDS`, `getClient`, `resolveIdentity*`, `refreshStaleIdentities`), labeler endpoint resolution and ingest (`resolveLabelerEndpoint`, `getLabelerState`, label subscribe cycles), and service-auth verification (`buildVerifier` in both the appview router and the community integration). The in-scope `config` is now also passed at every appview call site that resolves identities or PDS endpoints — the live-ingest refresh cycle (`runIngestCycle` → `refreshStaleIdentities`), the on-demand `refresh` path, and the router actor/identity/PDS resolution paths (`getProfile`, `getFeed`, collection queries, profile hydration, notify) — so private-network deploys honor the override on those paths instead of silently falling back to the public resolver and un-widened SSRF guard. + +The SSRF guard is now a single shared validator: `validateExternalUrl(url, additionalAllowedHosts?)` is exported from `contrail-base` and consumed by both the PDS client and labeler-endpoint resolution. `validateEndpointUrl` remains exported as a thin alias for backward compatibility. This removes the previous duplicate validator (`validatePdsUrl` + `validateEndpointUrl`) where an allowlist or SSRF-rule edit could be applied to only one copy. + +Also hardens schema initialization for concurrent/Postgres deployments: a dialect-aware `addColumnIfNotExists` (Postgres `ADD COLUMN IF NOT EXISTS`; SQLite pre-check), narrow absorption of the Postgres concurrent-`CREATE` race (42P07 / 23505 on pg_type/pg_class/pg_namespace indexes), and per-statement (rather than batched) DDL during `initSchema` / `initSpacesSchema` / spaces schema. Genuine DDL errors (syntax, type mismatch, missing column/table) still propagate. diff --git a/packages/contrail-appview/src/core/backfill.ts b/packages/contrail-appview/src/core/backfill.ts index a0474a7..f7fdd8b 100644 --- a/packages/contrail-appview/src/core/backfill.ts +++ b/packages/contrail-appview/src/core/backfill.ts @@ -186,7 +186,7 @@ export async function backfillUser( if (!client) { try { client = await withRetry( - () => getClient(did as Did, db), + () => getClient(did as Did, db, config), `getClient(${did})`, Math.min(retries, 1), timeout @@ -356,7 +356,7 @@ export async function backfillPending( for (let i = 0; i < dids.length; i += 200) { await Promise.allSettled( dids.slice(i, i + 200).map((did) => - getPDS(did as Did, db).catch(() => {}) + getPDS(did as Did, db, config).catch(() => {}) ) ); } @@ -386,7 +386,7 @@ export async function backfillPending( let client: Client | undefined; try { client = await withRetry( - () => getClient(did as Did, db), + () => getClient(did as Did, db, config), `getClient(${did})`, 0, FAST_TIMEOUT @@ -436,7 +436,7 @@ export async function backfillPending( let client: Client | undefined; try { client = await withRetry( - () => getClient(did as Did, db), + () => getClient(did as Did, db, config), `getClient(${did})`, 2 ); diff --git a/packages/contrail-appview/src/core/db/schema.ts b/packages/contrail-appview/src/core/db/schema.ts index c1778f0..df93e72 100644 --- a/packages/contrail-appview/src/core/db/schema.ts +++ b/packages/contrail-appview/src/core/db/schema.ts @@ -1,6 +1,6 @@ import type { ContrailConfig, Database, ResolvedContrailConfig, ResolvedMaps } from "../types"; import type { SqlDialect } from "../dialect"; -import { buildFtsSchema, getDialect } from "../dialect"; +import { buildFtsSchema, getDialect, postgresDialect } from "../dialect"; import { getRelationField, countColumnName, @@ -197,6 +197,135 @@ export function buildCountColumns(config: ContrailConfig, opts: BuilderOpts = {} return stmts; } +/** + * Idempotently add a column to a table, surfacing real DDL errors. + * + * Postgres supports `ALTER TABLE ... ADD COLUMN IF NOT EXISTS` natively, so we + * issue that and let any non-duplicate error propagate. SQLite (including + * `node:sqlite`) does NOT support `IF NOT EXISTS` on `ADD COLUMN`, so we + * pre-check `PRAGMA table_info` and short-circuit if the column is already + * there. Because the PRAGMA-check + ALTER pair is not atomic, a concurrent + * second `initSchema` call can still hit a "duplicate column name" race; we + * narrowly absorb exactly that error message and re-throw everything else. + * + * Net effect: only the duplicate-column case is absorbed. Missing tables, + * syntax errors, type mismatches, and any other DDL failure will throw. + * + * Exported for direct testing of the idempotency contract; callers in + * `initSchema` use this internally. + */ +export async function addColumnIfNotExists( + db: Database, + table: string, + column: string, + columnDef: string, +): Promise<void> { + const dialect = getDialect(db); + if (dialect === postgresDialect) { + await db.prepare( + `ALTER TABLE ${table} ADD COLUMN IF NOT EXISTS ${column} ${columnDef}`, + ).run(); + return; + } + // SQLite path: check existence first, then ALTER without IF NOT EXISTS. + // PRAGMA table_info() does not accept parameter binding, so we rely on the + // caller to pass a sanitized identifier (all current callers do — table + // names come from `recordsTableName`/`spacesRecordsTableName` which + // sanitize, and column names come from `countColumnName` / + // `groupedCountColumnName` which also sanitize). + const info = await db + .prepare(`PRAGMA table_info(${table})`) + .all<{ name: string }>(); + if (info.results.some((c) => c.name === column)) return; + try { + await db.prepare( + `ALTER TABLE ${table} ADD COLUMN ${column} ${columnDef}`, + ).run(); + } catch (err) { + // Narrow swallow: only the "duplicate column" race between the PRAGMA + // read and the ALTER is acceptable. Everything else surfaces. + if (!isDuplicateColumnError(err)) throw err; + } +} + +function isDuplicateColumnError(err: unknown): boolean { + if (!err || typeof err !== "object") return false; + const msg = (err as { message?: unknown }).message; + if (typeof msg !== "string") return false; + // node:sqlite / better-sqlite3: "duplicate column name: <col>" + return /duplicate column name/i.test(msg); +} + +/** + * Postgres `CREATE TABLE IF NOT EXISTS` and `CREATE INDEX IF NOT EXISTS` are + * NOT atomic against concurrent creators: two transactions can both pass the + * existence check before either has inserted into `pg_class` / `pg_type`. The + * loser raises 23505 on `pg_type_typname_nsp_index` (the unique index on + * `(typname, typnamespace)`) or `pg_class_relname_nsp_index`. Pre-existing + * tables also surface as 42P07 (`duplicate_table`). + * + * SQLite serializes DDL globally, so this race never manifests there. + * + * The caller is expected to issue idempotent DDL (IF NOT EXISTS); this helper + * only absorbs the narrow concurrent-create race. + */ +function isConcurrentCreateError(err: unknown): boolean { + if (!err || typeof err !== "object") return false; + const code = (err as { code?: unknown }).code; + if (code === "42P07" || code === "42P06") return true; + if (code === "23505") { + const constraint = (err as { constraint?: unknown }).constraint; + return ( + constraint === "pg_type_typname_nsp_index" || + constraint === "pg_class_relname_nsp_index" || + constraint === "pg_namespace_nspname_index" + ); + } + return false; +} + +/** + * Run a single DDL statement, absorbing only the concurrent-create race that + * `CREATE TABLE IF NOT EXISTS` / `CREATE INDEX IF NOT EXISTS` can hit on + * Postgres when multiple processes init the same schema in parallel. Genuine + * DDL errors (syntax, type mismatch, missing column) surface unchanged. + */ +async function runIdempotentDdl(db: Database, stmt: string): Promise<void> { + try { + await db.prepare(stmt).run(); + } catch (err) { + if (!isConcurrentCreateError(err)) throw err; + } +} + +/** + * Apply the ALTER+INDEX statements emitted by `buildCountColumns` + * idempotently and without swallowing non-duplicate errors. + * + * `buildCountColumns` mixes two statement shapes: `ALTER TABLE ... ADD COLUMN + * ...` (not idempotent on SQLite without a pre-check; supports IF NOT EXISTS + * on Postgres) and `CREATE INDEX IF NOT EXISTS ...` (idempotent on both + * dialects). We route ALTERs through `addColumnIfNotExists` and run indexes + * directly. + */ +export async function applyCountColumns( + db: Database, + config: ContrailConfig, + opts: BuilderOpts = {}, +): Promise<void> { + for (const stmt of buildCountColumns(config, opts)) { + const match = stmt.match( + /^ALTER TABLE\s+(\S+)\s+ADD COLUMN\s+(\S+)\s+(.+)$/i, + ); + if (match) { + const [, table, column, columnDef] = match; + await addColumnIfNotExists(db, table, column, columnDef); + } else { + await db.prepare(stmt).run(); + } + } +} + function buildFeedTables(config: ContrailConfig, dialect: SqlDialect): string[] { if (!config.feeds || Object.keys(config.feeds).length === 0) return []; const stmts = [ @@ -250,22 +379,56 @@ export function buildFtsTables( return stmts; } -const MIGRATIONS = [ - "ALTER TABLE backfills ADD COLUMN retries INTEGER NOT NULL DEFAULT 0", - "ALTER TABLE backfills ADD COLUMN last_error TEXT", - "ALTER TABLE spaces_invites ADD COLUMN kind TEXT NOT NULL DEFAULT 'join'", - "ALTER TABLE feed_backfills ADD COLUMN retries INTEGER NOT NULL DEFAULT 0", - "ALTER TABLE feed_backfills ADD COLUMN last_error TEXT", - "ALTER TABLE feed_backfills ADD COLUMN started_at BIGINT", +/** + * Schema migrations expressed as structured ADD-COLUMN ops. Each entry is + * applied via `addColumnIfNotExists` so the operation is idempotent on both + * dialects without swallowing genuine DDL errors. + * + * `target: "spaces"` is routed to the spaces DB (which may differ from the + * main DB in split-DB deployments) and only applied when spaces is enabled. + * `target: "feeds"` is only applied when feeds are configured (the + * `feed_backfills` table doesn't exist otherwise). All other migrations + * target the main DB unconditionally. + */ +interface MigrationOp { + table: string; + column: string; + columnDef: string; + target?: "spaces" | "feeds"; +} + +const MIGRATIONS: MigrationOp[] = [ + { table: "backfills", column: "retries", columnDef: "INTEGER NOT NULL DEFAULT 0" }, + { table: "backfills", column: "last_error", columnDef: "TEXT" }, + { + table: "spaces_invites", + column: "kind", + columnDef: "TEXT NOT NULL DEFAULT 'join'", + target: "spaces", + }, + { table: "feed_backfills", column: "retries", columnDef: "INTEGER NOT NULL DEFAULT 0", target: "feeds" }, + { table: "feed_backfills", column: "last_error", columnDef: "TEXT", target: "feeds" }, + { table: "feed_backfills", column: "started_at", columnDef: "BIGINT", target: "feeds" }, ]; -async function runMigrations(db: Database): Promise<void> { - for (const sql of MIGRATIONS) { - try { - await db.prepare(sql).run(); - } catch { - // Column already exists — ignore +async function runMigrations( + db: Database, + spacesDb: Database | undefined, + hasSpaces: boolean, + hasFeeds: boolean, +): Promise<void> { + for (const op of MIGRATIONS) { + if (op.target === "spaces") { + if (!hasSpaces) continue; + await addColumnIfNotExists(spacesDb ?? db, op.table, op.column, op.columnDef); + continue; + } + if (op.target === "feeds") { + if (!hasFeeds) continue; + await addColumnIfNotExists(db, op.table, op.column, op.columnDef); + continue; } + await addColumnIfNotExists(db, op.table, op.column, op.columnDef); } } @@ -290,15 +453,20 @@ async function applySpacesSchema( const base = buildSpacesBaseSchema(dialect); const perCollection = buildCollectionTables(config, dialect, { forSpaces: true }); const indexes = buildDynamicIndexes(config, dialect, { forSpaces: true }); - await target.batch([...base, ...perCollection, ...indexes].map((s) => target.prepare(s))); + // Per-statement (not batched) so concurrent applySpacesSchema on Postgres + // races only on the individual CREATE statements; see initSchema for + // rationale. + for (const stmt of [...base, ...perCollection, ...indexes]) { + await runIdempotentDdl(target, stmt); + } const ftsStmts = buildFtsTables(config, dialect, { forSpaces: true }); for (const stmt of ftsStmts) { try { await target.prepare(stmt).run(); } catch { /* FTS5 unavailable */ } } - for (const stmt of buildCountColumns(config, { forSpaces: true })) { - try { await target.prepare(stmt).run(); } catch { /* already exists */ } - } + // Idempotent count-column ALTERs + their indexes. Non-duplicate-column + // errors propagate. + await applyCountColumns(target, config, { forSpaces: true }); } export async function initSchema( @@ -320,7 +488,13 @@ export async function initSchema( const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...feedStatements]; - await db.batch(all.map((s) => db.prepare(s))); + // Per-statement run (not a batched transaction) so concurrent initSchema + // callers on Postgres race only on individual CREATEs; the loser's + // duplicate-relation error is absorbed by runIdempotentDdl. Each statement + // is already idempotent (IF NOT EXISTS). + for (const stmt of all) { + await runIdempotentDdl(db, stmt); + } if (config.spaces?.authority || config.spaces?.recordHost) { await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); @@ -339,7 +513,9 @@ export async function initSchema( // Labels tables live on the main DB — they're keyed by at-URI / DID and // are read alongside public records during hydration. const labelsStmts = buildLabelsSchema(dialect); - await db.batch(labelsStmts.map((s) => db.prepare(s))); + for (const stmt of labelsStmts) { + await runIdempotentDdl(db, stmt); + } } // FTS5 may not be available (e.g. node:sqlite) — skip gracefully @@ -350,14 +526,14 @@ export async function initSchema( // FTS5 not supported in this environment } } - await runMigrations(db); - - // Add count columns (ALTER TABLE — may already exist) - for (const stmt of buildCountColumns(config)) { - try { - await db.prepare(stmt).run(); - } catch { - // Column/index already exists — ignore - } - } + const hasSpaces = !!(config.spaces?.authority || config.spaces?.recordHost); + const hasFeeds = !!(config.feeds && Object.keys(config.feeds).length > 0); + // Spaces-targeted migrations route to spacesDb when one is configured; + // otherwise they hit the main db (which is where the spaces tables live + // when no separate spacesDb is supplied). + await runMigrations(db, spacesSharesMainDb ? undefined : spacesDb, hasSpaces, hasFeeds); + + // Idempotent count-column ALTERs + their indexes. Routed through + // `applyCountColumns` so non-duplicate-column errors propagate. + await applyCountColumns(db, config); } diff --git a/packages/contrail-appview/src/core/jetstream.ts b/packages/contrail-appview/src/core/jetstream.ts index abc0287..486d398 100644 --- a/packages/contrail-appview/src/core/jetstream.ts +++ b/packages/contrail-appview/src/core/jetstream.ts @@ -307,7 +307,7 @@ export async function runIngestCycle( const uniqueDids = [...new Set(events.map((e) => e.did))]; if (uniqueDids.length > 0) { try { - await refreshStaleIdentities(db, uniqueDids); + await refreshStaleIdentities(db, uniqueDids, config); } catch (err) { log.warn(`Identity refresh failed: ${err}`); } diff --git a/packages/contrail-appview/src/core/labels/resolve.ts b/packages/contrail-appview/src/core/labels/resolve.ts index 694bfec..af673d9 100644 --- a/packages/contrail-appview/src/core/labels/resolve.ts +++ b/packages/contrail-appview/src/core/labels/resolve.ts @@ -1,31 +1,37 @@ import { CompositeDidDocumentResolver, + type DidDocumentResolver, PlcDidDocumentResolver, WebDidDocumentResolver, } from "@atcute/identity-resolver"; import type { Did } from "@atcute/lexicons"; import type { Database } from "../types"; +import { validateExternalUrl } from "../client"; -/** Reject endpoint URLs that point to private/internal addresses or non-HTTPS. - * Mirrors the validator in core/client.ts — labeler endpoints should be - * publicly reachable for the same reasons PDS endpoints should. */ -function validateEndpointUrl(url: string): boolean { - try { - const parsed = new URL(url); - if (parsed.protocol !== "https:") return false; - const host = parsed.hostname; - if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; - if (host.startsWith("10.")) return false; - if (host.startsWith("192.168.")) return false; - if (host.startsWith("169.254.")) return false; - if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; - return true; - } catch { - return false; - } +/** Optional network-override knobs accepted by labeler-endpoint resolution. + * Mirrors the `ContrailConfig.networkOverrides` shape — kept narrow here so + * callers can pass `config.networkOverrides` directly without re-shaping. + * Omitting the object preserves the previous public-internet behavior. */ +export interface LabelerResolveOverrides { + /** DID document resolver used when looking up the labeler service entry. + * When unset, falls back to a default composite (PLC + Web) pointing at the + * upstream PLC directory. Trusted; not SSRF-checked. + * Mirrors the resolver-injection pattern in `core/client.ts`. */ + resolver?: DidDocumentResolver; + /** Hostnames (DNS names or IP literals) to allow past the default SSRF + * guard when validating a resolved labeler endpoint. Match is exact, + * case-insensitive, port-agnostic. */ + additionalAllowedHosts?: string[]; } -const didResolver = new CompositeDidDocumentResolver({ +/** Reject endpoint URLs that point to private/internal addresses or non-HTTPS. + * Thin alias for the single shared SSRF guard {@link validateExternalUrl} in + * `contrail-base` — labeler endpoints are validated by the exact same rules as + * PDS endpoints, so the allowlist logic must live in one place. Kept exported + * under this name for existing callers/tests. */ +export const validateEndpointUrl = validateExternalUrl; + +const DEFAULT_DID_RESOLVER: DidDocumentResolver = new CompositeDidDocumentResolver({ methods: { plc: new PlcDidDocumentResolver(), web: new WebDidDocumentResolver(), @@ -33,16 +39,26 @@ const didResolver = new CompositeDidDocumentResolver({ }); /** Look up the labeler service endpoint from a DID. - * Reads the DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ -export async function resolveLabelerEndpoint(did: string): Promise<string | null> { + * Reads the DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. + * + * `networkOverrides` (optional): customize the DID resolver used during the + * lookup, and/or which hostnames bypass the default SSRF guard. Omitting it + * preserves the original public-internet behavior. */ +export async function resolveLabelerEndpoint( + did: string, + networkOverrides?: LabelerResolveOverrides, +): Promise<string | null> { if (!did.startsWith("did:plc:") && !did.startsWith("did:web:")) return null; + const resolver = networkOverrides?.resolver ?? DEFAULT_DID_RESOLVER; try { - const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); + const doc = await resolver.resolve(did as Did<"plc"> | Did<"web">); const endpoint = doc.service ?.find((s) => s.id === "#atproto_labeler") ?.serviceEndpoint?.toString(); if (!endpoint) return null; - if (!validateEndpointUrl(endpoint)) return null; + if (!validateEndpointUrl(endpoint, networkOverrides?.additionalAllowedHosts ?? [])) { + return null; + } return endpoint; } catch { return null; @@ -63,11 +79,16 @@ const ENDPOINT_TTL_MS = 6 * 60 * 60 * 1000; // 6h, matches the recommended clien /** Get cached `(endpoint, cursor)` for a labeler. Resolves endpoint on * cache miss or staleness; persists endpoint + resolved_at back to the DB - * so subsequent ingest cycles avoid the network round-trip. */ + * so subsequent ingest cycles avoid the network round-trip. + * + * `networkOverrides` (optional): forwarded to `resolveLabelerEndpoint` for + * the cache-miss/stale path. Has no effect when `endpointOverride` is set + * or when a fresh cached endpoint is used. */ export async function getLabelerState( db: Database, did: string, endpointOverride: string | undefined, + networkOverrides?: LabelerResolveOverrides, ): Promise<LabelerState | null> { const row = await db .prepare( @@ -81,7 +102,7 @@ export async function getLabelerState( !row?.resolved_at || Date.now() - row.resolved_at > ENDPOINT_TTL_MS; if (!endpoint || (!endpointOverride && stale)) { - endpoint = await resolveLabelerEndpoint(did); + endpoint = await resolveLabelerEndpoint(did, networkOverrides); if (!endpoint) return null; const now = Date.now(); await db diff --git a/packages/contrail-appview/src/core/labels/subscribe.ts b/packages/contrail-appview/src/core/labels/subscribe.ts index c80ed2b..1351d01 100644 --- a/packages/contrail-appview/src/core/labels/subscribe.ts +++ b/packages/contrail-appview/src/core/labels/subscribe.ts @@ -36,7 +36,15 @@ export async function runLabelIngestCycle( } const remaining = Math.max(2_000, deadline - Date.now()); try { - await pumpOneLabeler(db, source, log, remaining, /* persistent */ false); + await pumpOneLabeler( + db, + source, + log, + remaining, + /* persistent */ false, + {}, + config.networkOverrides, + ); } catch (err) { log.warn(`[labels] cycle for ${source.did} failed: ${err}`); } @@ -62,7 +70,7 @@ export async function runPersistentLabels( const signal = options.signal; const tasks = config.labels.sources.map((source) => - runOneLabelerForever(db, source, log, signal, options), + runOneLabelerForever(db, source, log, signal, options, config.networkOverrides), ); await Promise.all(tasks); } @@ -73,15 +81,24 @@ async function runOneLabelerForever( log: Logger, signal: AbortSignal | undefined, options: PersistentLabelsOptions, + networkOverrides: ContrailConfig["networkOverrides"], ): Promise<void> { let attempts = 0; while (!signal?.aborted) { try { - await pumpOneLabeler(db, source, log, /* timeoutMs */ Infinity, true, { - signal, - batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE, - flushIntervalMs: options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, - }); + await pumpOneLabeler( + db, + source, + log, + /* timeoutMs */ Infinity, + true, + { + signal, + batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE, + flushIntervalMs: options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, + }, + networkOverrides, + ); attempts = 0; } catch (err) { if (signal?.aborted) break; @@ -114,8 +131,9 @@ async function pumpOneLabeler( timeoutMs: number, persistent: boolean, pumpOpts: PumpOptions = {}, + networkOverrides?: ContrailConfig["networkOverrides"], ): Promise<void> { - const state = await getLabelerState(db, source.did, source.endpoint); + const state = await getLabelerState(db, source.did, source.endpoint, networkOverrides); if (!state) { log.warn(`[labels] could not resolve labeler endpoint for ${source.did}; skipping`); return; diff --git a/packages/contrail-appview/src/core/persistent.ts b/packages/contrail-appview/src/core/persistent.ts index 882c9b8..94ca666 100644 --- a/packages/contrail-appview/src/core/persistent.ts +++ b/packages/contrail-appview/src/core/persistent.ts @@ -157,7 +157,7 @@ async function streamAndFlush( const uniqueDids = [...new Set(batch.map((e) => e.did))]; if (uniqueDids.length > 0) { try { - await refreshStaleIdentities(db, uniqueDids); + await refreshStaleIdentities(db, uniqueDids, config); } catch (err) { log.warn(`Identity refresh failed: ${err}`); } diff --git a/packages/contrail-appview/src/core/refresh.ts b/packages/contrail-appview/src/core/refresh.ts index 7b2fd21..acbae5c 100644 --- a/packages/contrail-appview/src/core/refresh.ts +++ b/packages/contrail-appview/src/core/refresh.ts @@ -134,7 +134,7 @@ export async function refresh( let client: Client; try { client = await withTimeout( - () => getClient(did as Did, db), + () => getClient(did as Did, db, config), requestTimeout ); } catch { diff --git a/packages/contrail-appview/src/core/router/collection.ts b/packages/contrail-appview/src/core/router/collection.ts index 9482631..875a4b7 100644 --- a/packages/contrail-appview/src/core/router/collection.ts +++ b/packages/contrail-appview/src/core/router/collection.ts @@ -301,7 +301,7 @@ export async function runPipeline( let did: string | undefined; if (actor) { - const resolved = await resolveActor(db, actor); + const resolved = await resolveActor(db, actor, config); if (!resolved) throw new Error("Could not resolve actor"); did = resolved; // backfillUser expects the record NSID (for PDS calls), not the short name. diff --git a/packages/contrail-appview/src/core/router/feed.ts b/packages/contrail-appview/src/core/router/feed.ts index 0d6a21d..e817e40 100644 --- a/packages/contrail-appview/src/core/router/feed.ts +++ b/packages/contrail-appview/src/core/router/feed.ts @@ -243,7 +243,7 @@ export function registerFeedRoutes( return c.json({ error: "Unknown feed" }, 404); } - const did = await resolveActor(db, actor); + const did = await resolveActor(db, actor, config); if (!did) return c.json({ error: "Could not resolve actor" }, 400); await maybeBackfillFeed(c, db, config, did, feedName, feedConfig); diff --git a/packages/contrail-appview/src/core/router/index.ts b/packages/contrail-appview/src/core/router/index.ts index 10e4990..10291f4 100644 --- a/packages/contrail-appview/src/core/router/index.ts +++ b/packages/contrail-appview/src/core/router/index.ts @@ -88,7 +88,7 @@ export function createApp( const actor = c.req.query("actor"); if (!actor) return c.json({ error: "actor parameter required" }, 400); - const did = await resolveActor(db, actor); + const did = await resolveActor(db, actor, config); if (!did) return c.json({ error: "Could not resolve actor" }, 400); // Ensure profile records are backfilled @@ -138,7 +138,7 @@ export function createApp( : config.spaces?.authority ? { adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), - verifier: buildVerifier(config.spaces.authority), + verifier: buildVerifier(config.spaces.authority, config.networkOverrides), manifestVerifier: config.spaces.authority.signing ? createManifestVerifier({ resolveKey: async (iss) => diff --git a/packages/contrail-appview/src/core/router/notify.ts b/packages/contrail-appview/src/core/router/notify.ts index 5ff630b..bedd74e 100644 --- a/packages/contrail-appview/src/core/router/notify.ts +++ b/packages/contrail-appview/src/core/router/notify.ts @@ -78,7 +78,7 @@ export async function processNotifyUris( ); for (const { uri, parsed } of validUris) { - const pds = await getPDS(parsed.did as Did, db); + const pds = await getPDS(parsed.did as Did, db, config); if (!pds) { errors.push(`could not resolve PDS for ${parsed.did}`); continue; diff --git a/packages/contrail-appview/src/core/router/profiles.ts b/packages/contrail-appview/src/core/router/profiles.ts index 5552357..8a95c82 100644 --- a/packages/contrail-appview/src/core/router/profiles.ts +++ b/packages/contrail-appview/src/core/router/profiles.ts @@ -85,7 +85,7 @@ export async function resolveProfiles( } // Resolve identities for all DIDs - const identities = await resolveIdentities(db, dids); + const identities = await resolveIdentities(db, dids, config); // Fetch missing profile records from PDS on demand const missingDids = dids.filter((d) => !result[d]); @@ -134,7 +134,7 @@ async function fetchMissingProfiles( const rkey = configRkey ?? "self"; const table = recordsTableName(shortName ?? collection); try { - const pds = await getPDS(did as Did, db); + const pds = await getPDS(did as Did, db, config); if (!pds) return; const url = new URL("/xrpc/com.atproto.repo.getRecord", pds); diff --git a/packages/contrail-appview/src/core/spaces/router.ts b/packages/contrail-appview/src/core/spaces/router.ts index a0557f5..9d4ce90 100644 --- a/packages/contrail-appview/src/core/spaces/router.ts +++ b/packages/contrail-appview/src/core/spaces/router.ts @@ -54,7 +54,7 @@ export function registerSpacesRoutes( if (!authorityConfig) return; const adapter = options.adapter ?? ctx?.adapter ?? new HostedAdapter(db, config); - const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); + const verifier = ctx?.verifier ?? buildVerifier(authorityConfig, config.networkOverrides); const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); const localRecordHost = spacesConfig.recordHost ? adapter : null; diff --git a/packages/contrail-appview/src/core/spaces/schema.ts b/packages/contrail-appview/src/core/spaces/schema.ts index ae33a7c..fcdf0cd 100644 --- a/packages/contrail-appview/src/core/spaces/schema.ts +++ b/packages/contrail-appview/src/core/spaces/schema.ts @@ -5,7 +5,7 @@ import { buildCollectionTables, buildDynamicIndexes, buildFtsTables, - buildCountColumns, + applyCountColumns, } from "../db/schema"; /** Spaces metadata tables — spaces, members, invites. No per-collection tables. */ @@ -76,8 +76,9 @@ export function buildSpacesBaseSchema(dialect: SqlDialect): string[] { /** Full spaces schema (base + per-collection tables + indexes). For callers * that need a single array of statements. Note: this does NOT include FTS - * virtual tables or ALTER TABLE count columns — those must be applied with - * try/catch fallbacks and are handled by `initSchema`. */ + * virtual tables or ALTER TABLE count columns — FTS is best-effort (engine + * may not be present) and count columns require dialect-aware idempotent + * ALTER. Both are handled by `initSchema` / `initSpacesSchema`. */ export function buildSpacesSchema(db: Database, config?: ContrailConfig): string[] { const dialect = getDialect(db); const base = buildSpacesBaseSchema(dialect); @@ -94,10 +95,10 @@ export async function initSpacesSchema(db: Database, config?: ContrailConfig): P const stmts = buildSpacesSchema(db, config); await db.batch(stmts.map((s) => db.prepare(s))); if (!config) return; + // FTS virtual tables: best-effort; the runtime may not have FTS5 compiled + // in (e.g. node:sqlite). Other DDL failures (count columns) propagate. for (const stmt of buildFtsTables(config, dialect, { forSpaces: true })) { - try { await db.prepare(stmt).run(); } catch { /* ignore */ } - } - for (const stmt of buildCountColumns(config, { forSpaces: true })) { - try { await db.prepare(stmt).run(); } catch { /* ignore */ } + try { await db.prepare(stmt).run(); } catch { /* FTS5 unavailable */ } } + await applyCountColumns(db, config, { forSpaces: true }); } diff --git a/packages/contrail-base/src/client.ts b/packages/contrail-base/src/client.ts index f37bea3..cd154ef 100644 --- a/packages/contrail-base/src/client.ts +++ b/packages/contrail-base/src/client.ts @@ -2,11 +2,12 @@ import { CompositeDidDocumentResolver, PlcDidDocumentResolver, WebDidDocumentResolver, + type DidDocumentResolver, } from "@atcute/identity-resolver"; import { type Did } from "@atcute/lexicons"; import { Client, simpleFetchHandler } from "@atcute/client"; import type {} from "@atcute/atproto"; -import type { Database } from "./types"; +import type { ContrailConfig, Database } from "./types"; // Slingshot-first PDS resolution with fallback to DID document resolution const SLINGSHOT_URL = @@ -18,28 +19,47 @@ export interface ResolvedIdentity { pds: string | null; } -/** Reject PDS URLs that point to private/internal addresses or non-HTTPS */ -function validatePdsUrl(url: string): boolean { +/** Reject external URLs (PDS, labeler, …) that point to private/internal + * addresses or non-HTTPS. The single SSRF guard shared across packages — + * callers MUST route every externally-resolved endpoint through this so the + * allowlist rules live in exactly one place. + * + * Hostnames in `additionalAllowedHosts` skip both checks. Match is exact, + * case-insensitive (allowlist entries are lowercased on compare; `URL.hostname` + * is already lowercased), and port-agnostic. + * + * Scope: best-effort guard against the obvious internal-address classes + * (private/link-local IPv4 literals, localhost, non-HTTPS). It does NOT + * resolve DNS, so a public hostname that resolves to a private address is not + * caught here, and IPv6 / non-canonical IP encodings are only partially + * covered. Defense-in-depth (egress network policy) is expected when resolver + * inputs are fully untrusted. */ +export function validateExternalUrl(url: string, additionalAllowedHosts?: string[]): boolean { + let parsed: URL; try { - const parsed = new URL(url); - if (parsed.protocol !== "https:") return false; - const host = parsed.hostname; - // Block private/internal IP ranges - if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; - if (host.startsWith("10.")) return false; - if (host.startsWith("192.168.")) return false; - if (host.startsWith("169.254.")) return false; - if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; - return true; + parsed = new URL(url); } catch { return false; } + if (additionalAllowedHosts?.some((h) => h.toLowerCase() === parsed.hostname)) { + return true; + } + if (parsed.protocol !== "https:") return false; + const host = parsed.hostname; + // Block private/internal IP ranges + if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; + if (host.startsWith("10.")) return false; + if (host.startsWith("192.168.")) return false; + if (host.startsWith("169.254.")) return false; + if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; + return true; } async function resolveViaSlingshot( - identifier: string + identifier: string, + slingshotUrl: string, ): Promise<ResolvedIdentity | undefined> { - const url = new URL(SLINGSHOT_URL); + const url = new URL(slingshotUrl); url.searchParams.set("identifier", identifier); try { @@ -61,15 +81,19 @@ async function resolveViaSlingshot( } } -const didResolver = new CompositeDidDocumentResolver({ +const DEFAULT_DID_RESOLVER: DidDocumentResolver = new CompositeDidDocumentResolver({ methods: { plc: new PlcDidDocumentResolver(), web: new WebDidDocumentResolver(), }, }); -async function getPDSViaDidDoc(did: Did): Promise<string | undefined> { - const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); +async function getPDSViaDidDoc( + did: Did, + config?: ContrailConfig, +): Promise<string | undefined> { + const resolver = config?.networkOverrides?.resolver ?? DEFAULT_DID_RESOLVER; + const doc = await resolver.resolve(did as Did<"plc"> | Did<"web">); return doc.service ?.find((s) => s.id === "#atproto_pds") ?.serviceEndpoint.toString(); @@ -78,21 +102,28 @@ async function getPDSViaDidDoc(did: Did): Promise<string | undefined> { /** * Resolve identity info (did, handle, pds) for a DID or handle. * Uses slingshot first, falls back to DID doc for PDS. + * + * `config?.networkOverrides` (optional): customize the slingshot endpoint, + * the PLC URL used during DID-doc fallback, and/or which hostnames bypass + * the default SSRF guard. Omitting `config` preserves all defaults. */ export async function resolvePDS( - identifier: string + identifier: string, + config?: ContrailConfig, ): Promise<ResolvedIdentity | undefined> { - const result = await resolveViaSlingshot(identifier); + const slingshotUrl = config?.networkOverrides?.slingshotUrl ?? SLINGSHOT_URL; + const allowed = config?.networkOverrides?.additionalAllowedHosts; + const result = await resolveViaSlingshot(identifier, slingshotUrl); if (result?.pds) { - if (!validatePdsUrl(result.pds)) return { ...result, pds: null }; + if (!validateExternalUrl(result.pds, allowed)) return { ...result, pds: null }; return result; } // Fall back to DID doc resolution (only works for DIDs, not handles) if (identifier.startsWith("did:")) { try { - const pds = await getPDSViaDidDoc(identifier as Did); - if (pds && validatePdsUrl(pds)) { + const pds = await getPDSViaDidDoc(identifier as Did, config); + if (pds && validateExternalUrl(pds, allowed)) { return { did: identifier, handle: result?.handle ?? null, @@ -107,7 +138,14 @@ export async function resolvePDS( return result; } -// In-memory PDS cache with TTL + size limit, plus in-flight deduplication +// In-memory PDS cache with TTL + size limit, plus in-flight deduplication. +// +// Keyed by DID only — this assumes a single, process-wide `networkOverrides` +// config (the deployment model: one resolver + one SSRF allowlist per process). +// Every caller in this monorepo now threads the same in-scope `config`, so a +// config-less and an override-aware resolution can never race for the same DID. +// If a future deployment ever resolves the same DID under differing overrides +// in one process, key these caches by an override fingerprint instead. const PDS_CACHE_TTL = 60 * 60 * 1000; // 1 hour const PDS_CACHE_MAX = 10_000; const pdsCache = new Map<string, { pds: string; at: number }>(); @@ -134,7 +172,8 @@ function pdsCacheSet(did: string, pds: string): void { export async function getPDS( did: Did, - db?: Database + db?: Database, + config?: ContrailConfig, ): Promise<string | undefined> { const mem = pdsCacheGet(did); if (mem) return mem; @@ -143,7 +182,7 @@ export async function getPDS( const inflight = pdsInflight.get(did); if (inflight) return inflight; - const promise = resolvePDSCached(did, db); + const promise = resolvePDSCached(did, db, config); pdsInflight.set(did, promise); try { return await promise; @@ -154,7 +193,8 @@ export async function getPDS( async function resolvePDSCached( did: Did, - db?: Database + db?: Database, + config?: ContrailConfig, ): Promise<string | undefined> { if (db) { const cached = await db @@ -167,7 +207,7 @@ async function resolvePDSCached( } } - const resolved = await resolvePDS(did); + const resolved = await resolvePDS(did, config); if (!resolved?.pds) return undefined; pdsCacheSet(did, resolved.pds); @@ -185,10 +225,21 @@ async function resolvePDSCached( return resolved.pds; } -export async function getClient(did: Did, db?: Database): Promise<Client> { - const pds = await getPDS(did, db); +export async function getClient( + did: Did, + db?: Database, + config?: ContrailConfig, +): Promise<Client> { + const pds = await getPDS(did, db, config); if (!pds) throw new Error(`PDS not found for ${did}`); return new Client({ handler: simpleFetchHandler({ service: pds }), }); } + +/** Test-only: clear module-level PDS caches. Production code MUST NOT call this. + * Exported with a `__` prefix to signal it is not part of the public API. */ +export function __resetPdsCachesForTests(): void { + pdsCache.clear(); + pdsInflight.clear(); +} diff --git a/packages/contrail-base/src/identity.ts b/packages/contrail-base/src/identity.ts index fe207b3..26e84fd 100644 --- a/packages/contrail-base/src/identity.ts +++ b/packages/contrail-base/src/identity.ts @@ -1,5 +1,5 @@ import type { Did } from "@atcute/lexicons"; -import type { Database, Logger } from "./types"; +import type { ContrailConfig, Database, Logger } from "./types"; import { isDid, isHandle } from "@atcute/lexicons/syntax"; import { resolvePDS } from "./client"; @@ -28,9 +28,10 @@ function isStale(resolvedAt: number): boolean { async function fetchAndSave( db: Database, identifier: string, - cached?: Identity | null + cached?: Identity | null, + config?: ContrailConfig, ): Promise<Identity> { - const resolved = await resolvePDS(identifier); + const resolved = await resolvePDS(identifier, config); const identity: Identity = { did: resolved?.did ?? identifier, handle: resolved?.handle ?? cached?.handle ?? null, @@ -43,7 +44,8 @@ async function fetchAndSave( export async function resolveIdentity( db: Database, - did: Did + did: Did, + config?: ContrailConfig, ): Promise<Identity> { const cached = await db .prepare("SELECT did, handle, pds, resolved_at FROM identities WHERE did = ?") @@ -52,12 +54,13 @@ export async function resolveIdentity( if (cached && !isStale(cached.resolved_at)) return cached; - return fetchAndSave(db, did, cached); + return fetchAndSave(db, did, cached, config); } export async function resolveIdentities( db: Database, - dids: string[] + dids: string[], + config?: ContrailConfig, ): Promise<Map<string, Identity>> { const map = new Map<string, Identity>(); if (dids.length === 0) return map; @@ -80,7 +83,7 @@ export async function resolveIdentities( for (const did of dids) { if (map.has(did) || !isDid(did)) continue; try { - const identity = await fetchAndSave(db, did); + const identity = await fetchAndSave(db, did, undefined, config); map.set(did, identity); } catch { // Silently skip unresolvable identities @@ -92,7 +95,8 @@ export async function resolveIdentities( export async function resolveActor( db: Database, - actor: string + actor: string, + config?: ContrailConfig, ): Promise<string | null> { if (isDid(actor)) return actor; if (!isHandle(actor)) return null; @@ -106,7 +110,7 @@ export async function resolveActor( if (cached && !isStale(cached.resolved_at)) return cached.did; // Resolve via slingshot - const resolved = await resolvePDS(actor); + const resolved = await resolvePDS(actor, config); if (!resolved?.did || !isDid(resolved.did)) return null; await saveIdentity(db, { @@ -139,7 +143,8 @@ export async function applyIdentityEvent( export async function refreshStaleIdentities( db: Database, - dids: string[] + dids: string[], + config?: ContrailConfig, ): Promise<void> { if (dids.length === 0) return; @@ -169,7 +174,7 @@ export async function refreshStaleIdentities( for (const did of toRefresh) { try { - await fetchAndSave(db, did); + await fetchAndSave(db, did, undefined, config); } catch { // Silently skip unresolvable identities } diff --git a/packages/contrail-base/src/spaces/auth.ts b/packages/contrail-base/src/spaces/auth.ts index 4c1e817..4c255f6 100644 --- a/packages/contrail-base/src/spaces/auth.ts +++ b/packages/contrail-base/src/spaces/auth.ts @@ -12,12 +12,22 @@ import { readInProcess } from "./in-process"; export { ServiceJwtVerifier }; -/** Build a ServiceJwtVerifier from an AuthorityConfig, using the configured - * resolver or a default PLC+Web composite. The verifier checks that incoming - * JWTs target this authority's serviceDid (aud claim). */ -export function buildVerifier(authority: AuthorityConfig): ServiceJwtVerifier { +/** Build a ServiceJwtVerifier from an AuthorityConfig, using (in precedence + * order) the authority-specific resolver, then the deployment-wide + * `networkOverrides.resolver`, then a default PLC+Web composite. The verifier + * checks that incoming JWTs target this authority's serviceDid (aud claim). + * + * `networkOverrides` is optional and is the same shape carried on + * `ContrailConfig.networkOverrides` — callers with a ContrailConfig in scope + * should pass `config.networkOverrides` so private-network deployments share + * one resolver across both identity resolution and service-auth verification. */ +export function buildVerifier( + authority: AuthorityConfig, + networkOverrides?: { resolver?: DidDocumentResolver }, +): ServiceJwtVerifier { const resolver = authority.resolver ?? + networkOverrides?.resolver ?? new CompositeDidDocumentResolver({ methods: { plc: new PlcDidDocumentResolver(), diff --git a/packages/contrail-base/src/types.ts b/packages/contrail-base/src/types.ts index 9d63336..8a15a4a 100644 --- a/packages/contrail-base/src/types.ts +++ b/packages/contrail-base/src/types.ts @@ -241,6 +241,37 @@ export interface ContrailConfig { * ingests synthesized rows for any follower already in our identities * table. Lets newcomers immediately appear in existing users' feeds. */ constellation?: ConstellationConfig | false; + /** Network overrides for private-network or test deployments. + * All subfields default to current public-internet behavior; + * omitting `networkOverrides` entirely preserves current behavior. + * + * SECURITY: `resolver` and `slingshotUrl` are taken at face value and are + * NOT validated against the SSRF guard — the consumer is trusted to + * configure them. Only the PDS URL returned downstream is validated, + * and only `additionalAllowedHosts` widens that PDS validator. There is + * no "disable SSRF" flag. */ + networkOverrides?: { + /** DID document resolver used during the DID-doc PDS fallback. When + * unset, contrail constructs a default `CompositeDidDocumentResolver` + * with PLC + Web methods pointing at the upstream PLC directory. + * Pass a custom resolver to point at a private PLC mirror, inject a + * custom fetch (mTLS, retry, instrumentation), or swap in an + * alternative DID method composition. + * Mirrors the `AuthorityConfig.resolver` pattern in `spaces/types.ts`. */ + resolver?: import("@atcute/identity-resolver").DidDocumentResolver; + /** Slingshot identity resolver URL override. Trusted; not SSRF-checked. + * Default: https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc */ + slingshotUrl?: string; + /** Hostnames (DNS names or IP literals) to allow past the default SSRF + * guard when validating a resolved PDS URL. + * For listed hostnames, the non-HTTPS + private-CIDR checks are skipped. + * For all other hostnames, the default validator runs unchanged. + * Match semantics: exact hostname, case-insensitive (entries are + * lowercased on comparison; `URL.hostname` is already lowercased), + * port-agnostic. + * Example: ["pds.dev.svc.cluster.local"]. */ + additionalAllowedHosts?: string[]; + }; } export interface ConstellationConfig { diff --git a/packages/contrail-community/src/integration.ts b/packages/contrail-community/src/integration.ts index 9dca172..38b9435 100644 --- a/packages/contrail-community/src/integration.ts +++ b/packages/contrail-community/src/integration.ts @@ -60,7 +60,7 @@ export function createCommunityIntegration( registerRoutes(app, opts) { // Reuse the spaces JWT verifier — the auth model is identical. if (!config.spaces?.authority) return; - const verifier = buildVerifier(config.spaces.authority); + const verifier = buildVerifier(config.spaces.authority, config.networkOverrides); const authMiddleware = opts?.authMiddleware ?? createServiceAuthMiddleware(verifier); registerCommunityRoutes( diff --git a/packages/contrail/tests/client.test.ts b/packages/contrail/tests/client.test.ts new file mode 100644 index 0000000..7aa2143 --- /dev/null +++ b/packages/contrail/tests/client.test.ts @@ -0,0 +1,239 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { resolvePDS, getClient, getPDS, __resetPdsCachesForTests } from "../src/core/client"; +import { type DidDocumentResolver } from "@atcute/identity-resolver"; +import { createTestDbWithSchema } from "./helpers"; +import type { Did } from "@atcute/lexicons"; + +describe("validatePdsUrl via resolvePDS — regression baseline", () => { + let fetchSpy: ReturnType<typeof vi.spyOn>; + + beforeEach(() => { + fetchSpy = vi.spyOn(global, "fetch"); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + it("accepts a public HTTPS PDS", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:a", handle: "a.bsky.social", pds: "https://shimeji.us-east.host.bsky.network" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:a"); + expect(r?.pds).toBe("https://shimeji.us-east.host.bsky.network"); + }); + + it("rejects non-HTTPS PDS (returns pds: null)", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:b", handle: "b.test", pds: "http://malicious.example.com" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:b"); + expect(r?.pds).toBe(null); + }); + + it("rejects 10.x private CIDR", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:c", pds: "https://10.0.0.1" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:c"); + expect(r?.pds).toBe(null); + }); + + it("rejects 192.168.x private CIDR", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:d", pds: "https://192.168.1.1" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:d"); + expect(r?.pds).toBe(null); + }); + + it("rejects 172.16-31.x private CIDR", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:e", pds: "https://172.20.0.5" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:e"); + expect(r?.pds).toBe(null); + }); + + it("rejects localhost and 169.254 link-local", async () => { + fetchSpy.mockResolvedValueOnce( + new Response(JSON.stringify({ did: "did:plc:f", pds: "https://localhost" }), { status: 200 }) + ); + expect((await resolvePDS("did:plc:f"))?.pds).toBe(null); + + fetchSpy.mockResolvedValueOnce( + new Response(JSON.stringify({ did: "did:plc:g", pds: "https://169.254.169.254" }), { status: 200 }) + ); + expect((await resolvePDS("did:plc:g"))?.pds).toBe(null); + }); +}); + +describe("validatePdsUrl via resolvePDS — additionalAllowedHosts allowlist", () => { + let fetchSpy: ReturnType<typeof vi.spyOn>; + + beforeEach(() => { + fetchSpy = vi.spyOn(global, "fetch"); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + it("accepts http://pds.dev.svc.cluster.local when host is on allowlist", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:h", pds: "http://pds.dev.svc.cluster.local" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:h", { + namespace: "test", + collections: {}, + networkOverrides: { additionalAllowedHosts: ["pds.dev.svc.cluster.local"] }, + }); + expect(r?.pds).toBe("http://pds.dev.svc.cluster.local"); + }); + + it("still rejects http://other.private.host when not on allowlist", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:i", pds: "http://other.private.host" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:i", { + namespace: "test", + collections: {}, + networkOverrides: { additionalAllowedHosts: ["pds.dev.svc.cluster.local"] }, + }); + expect(r?.pds).toBe(null); + }); + + it("still rejects http://192.168.1.1 when not on allowlist", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:j", pds: "http://192.168.1.1" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:j", { + namespace: "test", + collections: {}, + networkOverrides: { additionalAllowedHosts: ["pds.dev.svc.cluster.local"] }, + }); + expect(r?.pds).toBe(null); + }); + + it("port-agnostic: allowlist matches hostname regardless of port", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:k", pds: "http://pds.dev.svc.cluster.local:8080" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:k", { + namespace: "test", + collections: {}, + networkOverrides: { additionalAllowedHosts: ["pds.dev.svc.cluster.local"] }, + }); + expect(r?.pds).toBe("http://pds.dev.svc.cluster.local:8080"); + }); + + it("case-insensitive: mixed-case allowlist entries match lowercase URL.hostname", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:l", pds: "http://pds.dev.svc.cluster.local" }), { status: 200 }) + ); + const r = await resolvePDS("did:plc:l", { + namespace: "test", + collections: {}, + networkOverrides: { additionalAllowedHosts: ["PDS.Dev.Svc.Cluster.Local"] }, + }); + expect(r?.pds).toBe("http://pds.dev.svc.cluster.local"); + }); +}); + +describe("getPDSViaDidDoc — resolver override", () => { + let fetchSpy: ReturnType<typeof vi.spyOn>; + + beforeEach(() => { + __resetPdsCachesForTests(); + fetchSpy = vi.spyOn(global, "fetch"); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + it("falls back to DID doc when slingshot returns no pds, and uses injected resolver", async () => { + fetchSpy.mockResolvedValueOnce( + new Response(JSON.stringify({ did: "did:plc:abc", handle: "alice.test" }), { status: 200 }) + ); + + const resolveCalls: string[] = []; + const injectedResolver: DidDocumentResolver = { + async resolve(did: string) { + resolveCalls.push(did); + return { + service: [ + { id: "#atproto_pds", type: "AtprotoPersonalDataServer", serviceEndpoint: "http://pds.dev.svc.cluster.local" }, + ], + } as any; + }, + } as any; + + const r = await resolvePDS("did:plc:abc", { + namespace: "test", + collections: {}, + networkOverrides: { + resolver: injectedResolver, + additionalAllowedHosts: ["pds.dev.svc.cluster.local"], + }, + }); + expect(r?.pds).toBe("http://pds.dev.svc.cluster.local"); + expect(resolveCalls).toContain("did:plc:abc"); + }); +}); + +describe("getClient + getPDS — config plumb-through", () => { + let fetchSpy: ReturnType<typeof vi.spyOn>; + + beforeEach(() => { + __resetPdsCachesForTests(); + fetchSpy = vi.spyOn(global, "fetch"); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + it("getClient with config?.networkOverrides.slingshotUrl uses the override", async () => { + fetchSpy.mockImplementation(async (input) => { + const url = String(input); + if (url.includes("my-slingshot")) { + return new Response( + JSON.stringify({ did: "did:plc:x", pds: "https://pds.allowed.test" }), + { status: 200 }, + ); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + const db = await createTestDbWithSchema(); + const client = await getClient("did:plc:x" as Did, db, { + namespace: "test", + collections: {}, + networkOverrides: { + slingshotUrl: "https://my-slingshot.test/xrpc/com.bad-example.identity.resolveMiniDoc", + additionalAllowedHosts: ["pds.allowed.test"], + }, + }); + expect(client).toBeDefined(); + const calls = fetchSpy.mock.calls.map((c) => String(c[0])); + expect(calls.some((u) => u.includes("my-slingshot.test"))).toBe(true); + }); + + it("getPDS with no config uses the default slingshot URL (backward-compat)", async () => { + fetchSpy.mockImplementation(async (input) => { + const url = String(input); + if (url.includes("slingshot.microcosm.blue")) { + return new Response( + JSON.stringify({ did: "did:plc:y", pds: "https://shimeji.us-east.host.bsky.network" }), + { status: 200 }, + ); + } + throw new Error(`Unexpected fetch: ${url}`); + }); + const db = await createTestDbWithSchema(); + const pds = await getPDS("did:plc:y" as Did, db); + expect(pds).toBe("https://shimeji.us-east.host.bsky.network"); + const calls = fetchSpy.mock.calls.map((c) => String(c[0])); + expect(calls.some((u) => u.includes("slingshot.microcosm.blue"))).toBe(true); + }); +}); diff --git a/packages/contrail/tests/identity-config.test.ts b/packages/contrail/tests/identity-config.test.ts new file mode 100644 index 0000000..aa5df42 --- /dev/null +++ b/packages/contrail/tests/identity-config.test.ts @@ -0,0 +1,87 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { + resolveIdentity, + resolveIdentities, + resolveActor, + refreshStaleIdentities, +} from "../src/core/identity"; +import { __resetPdsCachesForTests } from "../src/core/client"; +import { createTestDbWithSchema } from "./helpers"; +import type { Did } from "@atcute/lexicons"; + +describe("identity.ts — config plumb-through", () => { + let fetchSpy: ReturnType<typeof vi.spyOn>; + + beforeEach(() => { + __resetPdsCachesForTests(); + fetchSpy = vi.spyOn(global, "fetch"); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + const overrideConfig = { + namespace: "test", + collections: {}, + networkOverrides: { + slingshotUrl: "https://my-slingshot.test/xrpc/com.bad-example.identity.resolveMiniDoc", + additionalAllowedHosts: ["pds.allowed.test"], + }, + }; + + it("resolveIdentity routes slingshot to override URL", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:a", handle: "a.test", pds: "https://pds.allowed.test" }), { status: 200 }), + ); + const db = await createTestDbWithSchema(); + const id = await resolveIdentity(db, "did:plc:a" as Did, overrideConfig); + expect(id.pds).toBe("https://pds.allowed.test"); + expect(fetchSpy.mock.calls.some(([u]) => String(u).includes("my-slingshot.test"))).toBe(true); + }); + + it("resolveIdentities batch routes to override URL", async () => { + fetchSpy.mockImplementation(async (input) => { + const url = String(input); + const id = new URL(url).searchParams.get("identifier") ?? "did:plc:?"; + return new Response( + JSON.stringify({ did: id, handle: `${id}.test`, pds: "https://pds.allowed.test" }), + { status: 200 }, + ); + }); + const db = await createTestDbWithSchema(); + const m = await resolveIdentities(db, ["did:plc:b", "did:plc:c"], overrideConfig); + expect(m.get("did:plc:b")?.pds).toBe("https://pds.allowed.test"); + expect(m.get("did:plc:c")?.pds).toBe("https://pds.allowed.test"); + }); + + it("resolveActor routes a handle lookup to override URL", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:d", handle: "user.test", pds: "https://pds.allowed.test" }), { status: 200 }), + ); + const db = await createTestDbWithSchema(); + const did = await resolveActor(db, "user.test", overrideConfig); + expect(did).toBe("did:plc:d"); + expect(fetchSpy.mock.calls.some(([u]) => String(u).includes("my-slingshot.test"))).toBe(true); + }); + + it("refreshStaleIdentities routes to override URL", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:e", handle: "e.test", pds: "https://pds.allowed.test" }), { status: 200 }), + ); + const db = await createTestDbWithSchema(); + await refreshStaleIdentities(db, ["did:plc:e"], overrideConfig); + const row = await db.prepare("SELECT did, pds FROM identities WHERE did = ?").bind("did:plc:e").first<{ pds: string }>(); + expect(row?.pds).toBe("https://pds.allowed.test"); + }); + + it("backward-compat: no config preserves default slingshot URL", async () => { + fetchSpy.mockResolvedValue( + new Response(JSON.stringify({ did: "did:plc:f", handle: "f.bsky.social", pds: "https://shimeji.us-east.host.bsky.network" }), { status: 200 }), + ); + const db = await createTestDbWithSchema(); + const id = await resolveIdentity(db, "did:plc:f" as Did); + expect(id.pds).toBe("https://shimeji.us-east.host.bsky.network"); + expect(fetchSpy.mock.calls.some(([u]) => String(u).includes("slingshot.microcosm.blue"))).toBe(true); + }); +}); diff --git a/packages/contrail/tests/labels-resolve.test.ts b/packages/contrail/tests/labels-resolve.test.ts new file mode 100644 index 0000000..80ca73d --- /dev/null +++ b/packages/contrail/tests/labels-resolve.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect, vi } from "vitest"; +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, + type DidDocumentResolver, +} from "@atcute/identity-resolver"; +import { + resolveLabelerEndpoint, + validateEndpointUrl, +} from "../src/core/labels/resolve"; + +describe("validateEndpointUrl additionalAllowedHosts", () => { + it("rejects pds.dev.svc.cluster.local without override (HTTP + private hostname)", () => { + expect(validateEndpointUrl("http://pds.dev.svc.cluster.local:2583")).toBe(false); + }); + + it("accepts pds.dev.svc.cluster.local when listed in additionalAllowedHosts (case-insensitive)", () => { + expect( + validateEndpointUrl("http://PDS.dev.svc.cluster.local:2583", [ + "pds.dev.svc.cluster.local", + ]), + ).toBe(true); + }); + + it("does not relax HTTPS requirement for non-listed hosts when override is present", () => { + expect( + validateEndpointUrl("http://attacker.com", ["pds.dev.svc.cluster.local"]), + ).toBe(false); + }); + + it("ignores port differences (host-only match)", () => { + expect( + validateEndpointUrl("http://pds.dev.svc.cluster.local:9999", [ + "pds.dev.svc.cluster.local", + ]), + ).toBe(true); + }); +}); + +describe("resolveLabelerEndpoint resolver injection", () => { + it("uses networkOverrides.resolver when provided", async () => { + const mockResolver = { + resolve: vi.fn().mockResolvedValue({ + service: [ + { id: "#atproto_labeler", serviceEndpoint: "https://labeler.test" }, + ], + }), + }; + const endpoint = await resolveLabelerEndpoint("did:plc:abc123", { + resolver: mockResolver as unknown as DidDocumentResolver, + }); + expect(endpoint).toBe("https://labeler.test"); + expect(mockResolver.resolve).toHaveBeenCalledOnce(); + }); + + it("applies additionalAllowedHosts to resolved labeler endpoint", async () => { + const mockResolver = { + resolve: vi.fn().mockResolvedValue({ + service: [ + { + id: "#atproto_labeler", + serviceEndpoint: "http://labeler.dev.svc.cluster.local:2583", + }, + ], + }), + }; + // Without override the http endpoint should be rejected + const rejected = await resolveLabelerEndpoint("did:plc:abc123", { + resolver: mockResolver as unknown as DidDocumentResolver, + }); + expect(rejected).toBeNull(); + + // With override it should pass + const accepted = await resolveLabelerEndpoint("did:plc:abc123", { + resolver: mockResolver as unknown as DidDocumentResolver, + additionalAllowedHosts: ["labeler.dev.svc.cluster.local"], + }); + expect(accepted).toBe("http://labeler.dev.svc.cluster.local:2583"); + }); +}); diff --git a/packages/contrail/tests/network-overrides-appview.test.ts b/packages/contrail/tests/network-overrides-appview.test.ts new file mode 100644 index 0000000..1110a57 --- /dev/null +++ b/packages/contrail/tests/network-overrides-appview.test.ts @@ -0,0 +1,96 @@ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; +import { createApp } from "../src/core/router"; +import { runIngestCycle } from "../src/core/jetstream"; +import { __resetPdsCachesForTests } from "../src/core/client"; +import { createTestDbWithSchema, TEST_CONFIG } from "./helpers"; +import type { ContrailConfig } from "../src/core/types"; + +// Mock the Jetstream subscription so `runIngestCycle` ingests one synthetic +// commit without opening a real WebSocket. Everything else in the appview +// ingest path runs for real, so this exercises the live-ingest refresh path +// end-to-end (the smoking-gun call site `refreshStaleIdentities(db, dids)`). +vi.mock("@atcute/jetstream", async (importOriginal) => { + const actual = await importOriginal<typeof import("@atcute/jetstream")>(); + class MockJetstreamSubscription { + cursor: number | null = null; + constructor(_opts: unknown) {} + async *[Symbol.asyncIterator]() { + yield { + kind: "commit", + // A past time_us so the ingest loop doesn't treat it as "caught up". + time_us: 1_000_000, + did: "did:plc:ingest", + commit: { + collection: "community.lexicon.calendar.event", + operation: "create", + rkey: "abc", + cid: "bafyabc", + record: { name: "Test Event", startsAt: "2026-04-01T10:00:00Z", mode: "online" }, + }, + }; + } + } + return { ...actual, JetstreamSubscription: MockJetstreamSubscription }; +}); + +const silentLogger = { log() {}, warn() {}, error() {} }; + +const OVERRIDE = { + slingshotUrl: "https://my-slingshot.test/xrpc/com.bad-example.identity.resolveMiniDoc", + additionalAllowedHosts: ["pds.allowed.test"], +}; + +function overrideConfig(): ContrailConfig { + return { ...TEST_CONFIG, logger: silentLogger, networkOverrides: OVERRIDE }; +} + +describe("networkOverrides — appview entry points thread config", () => { + let fetchSpy: ReturnType<typeof vi.spyOn>; + + beforeEach(() => { + __resetPdsCachesForTests(); + fetchSpy = vi.spyOn(globalThis, "fetch").mockResolvedValue( + new Response( + JSON.stringify({ did: "did:plc:resolved", handle: "user.test", pds: "https://pds.allowed.test" }), + { status: 200, headers: { "content-type": "application/json" } }, + ), + ); + }); + + afterEach(() => { + fetchSpy.mockRestore(); + }); + + // router actor-resolution path: GET /xrpc/<ns>.getProfile -> resolveActor + it("router getProfile resolves a handle via the override slingshot", async () => { + const db = await createTestDbWithSchema(); + const app = createApp(db, overrideConfig()); + + await app.fetch( + new Request(`http://localhost/xrpc/${TEST_CONFIG.namespace}.getProfile?actor=user.test`), + ); + + // The handle lookup (identifier=user.test) must go to the override + // slingshot, not the default public one. Before the fix `resolveActor` + // was called without `config`, so this fetch hit the default URL. + const hitOverrideForHandle = fetchSpy.mock.calls.some(([u]) => { + const s = String(u); + return s.includes("my-slingshot.test") && s.includes("identifier=user.test"); + }); + expect(hitOverrideForHandle).toBe(true); + }); + + // live-ingest refresh path: runIngestCycle -> refreshStaleIdentities + it("jetstream ingest cycle refreshes identities via the override slingshot", async () => { + const db = await createTestDbWithSchema(); + + await runIngestCycle(db, overrideConfig(), 1_000); + + // refreshStaleIdentities resolved the ingested DID; before the fix it was + // called without `config`, so the resolve hit the default slingshot. + const hitOverride = fetchSpy.mock.calls.some(([u]) => + String(u).includes("my-slingshot.test"), + ); + expect(hitOverride).toBe(true); + }); +}); diff --git a/packages/contrail/tests/network-overrides.test.ts b/packages/contrail/tests/network-overrides.test.ts new file mode 100644 index 0000000..011ab76 --- /dev/null +++ b/packages/contrail/tests/network-overrides.test.ts @@ -0,0 +1,189 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; +import http from "node:http"; +import type { AddressInfo } from "node:net"; +import { resolvePDS, getClient, __resetPdsCachesForTests } from "../src/core/client"; +import { refreshStaleIdentities } from "../src/core/identity"; +import { createTestDbWithSchema } from "./helpers"; +import type { ContrailConfig } from "../src/core/types"; +import type { Did } from "@atcute/lexicons"; +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, +} from "@atcute/identity-resolver"; + +type Hit = { method: string; url: string }; + +interface Stub { + url: string; + hits: Hit[]; + setHandler: (h: (req: http.IncomingMessage, res: http.ServerResponse) => void) => void; + close: () => Promise<void>; +} + +async function startStub(): Promise<Stub> { + const hits: Hit[] = []; + let handler: (req: http.IncomingMessage, res: http.ServerResponse) => void = (_req, res) => { + res.writeHead(404); + res.end(); + }; + const server = http.createServer((req, res) => { + hits.push({ method: req.method ?? "?", url: req.url ?? "" }); + handler(req, res); + }); + await new Promise<void>((r) => server.listen(0, "127.0.0.1", r)); + const { port } = server.address() as AddressInfo; + return { + url: `http://127.0.0.1:${port}`, + hits, + setHandler: (h) => { + handler = h; + }, + close: () => new Promise<void>((r) => server.close(() => r())), + }; +} + +let plc: Stub; +let slingshot: Stub; + +const baseConfig: ContrailConfig = { + namespace: "test", + collections: {}, +}; + +beforeAll(async () => { + plc = await startStub(); + slingshot = await startStub(); +}); + +afterAll(async () => { + await plc.close(); + await slingshot.close(); +}); + +beforeEach(() => { + plc.hits.length = 0; + slingshot.hits.length = 0; + __resetPdsCachesForTests(); +}); + +describe("networkOverrides — full chain integration", () => { + it("resolvePDS routes slingshot fetch to slingshotUrl override", async () => { + slingshot.setHandler((_req, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ did: "did:plc:abc", handle: "alice.test", pds: "http://pds.private.test" })); + }); + + const config: ContrailConfig = { + ...baseConfig, + networkOverrides: { + slingshotUrl: `${slingshot.url}/xrpc/com.bad-example.identity.resolveMiniDoc`, + additionalAllowedHosts: ["pds.private.test"], + }, + }; + const result = await resolvePDS("did:plc:abc", config); + expect(result?.pds).toBe("http://pds.private.test"); + expect(slingshot.hits.length).toBeGreaterThan(0); + expect(slingshot.hits[0].url).toContain("identifier=did%3Aplc%3Aabc"); + }); + + it("rejects pds when not in allowlist (default validator still applies)", async () => { + slingshot.setHandler((_req, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ did: "did:plc:bcd", pds: "http://pds.private.test" })); + }); + + const config: ContrailConfig = { + ...baseConfig, + networkOverrides: { + slingshotUrl: `${slingshot.url}/xrpc/com.bad-example.identity.resolveMiniDoc`, + }, + }; + const result = await resolvePDS("did:plc:bcd", config); + expect(result?.pds).toBe(null); + }); + + it("falls back to plcUrl when slingshot returns no pds", async () => { + slingshot.setHandler((_req, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ did: "did:plc:cde", handle: "carol.test" })); + }); + plc.setHandler((req, res) => { + const decoded = req.url ? decodeURIComponent(req.url) : ""; + if (decoded.includes("did:plc:cde")) { + res.writeHead(200, { "content-type": "application/did+ld+json" }); + res.end(JSON.stringify({ + "@context": ["https://www.w3.org/ns/did/v1"], + id: "did:plc:cde", + alsoKnownAs: ["at://carol.test"], + verificationMethod: [], + service: [ + { id: "#atproto_pds", type: "AtprotoPersonalDataServer", serviceEndpoint: "http://pds.private.test" }, + ], + })); + } else { + res.writeHead(404); + res.end(); + } + }); + + const resolver = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver({ apiUrl: plc.url }), + web: new WebDidDocumentResolver(), + }, + }); + const config: ContrailConfig = { + ...baseConfig, + networkOverrides: { + slingshotUrl: `${slingshot.url}/xrpc/com.bad-example.identity.resolveMiniDoc`, + resolver, + additionalAllowedHosts: ["pds.private.test"], + }, + }; + const result = await resolvePDS("did:plc:cde", config); + expect(result?.pds).toBe("http://pds.private.test"); + expect(plc.hits.length).toBeGreaterThan(0); + }); + + it("refreshStaleIdentities persists pds via override path", async () => { + slingshot.setHandler((_req, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ did: "did:plc:def", handle: "dave.test", pds: "http://pds.private.test" })); + }); + const config: ContrailConfig = { + ...baseConfig, + networkOverrides: { + slingshotUrl: `${slingshot.url}/xrpc/com.bad-example.identity.resolveMiniDoc`, + additionalAllowedHosts: ["pds.private.test"], + }, + }; + const db = await createTestDbWithSchema(); + await refreshStaleIdentities(db, ["did:plc:def"], config); + + const row = await db + .prepare("SELECT did, pds FROM identities WHERE did = ?") + .bind("did:plc:def") + .first<{ did: string; pds: string }>(); + expect(row?.pds).toBe("http://pds.private.test"); + expect(slingshot.hits.length).toBeGreaterThan(0); + }); + + it("getClient with override config resolves PDS via stubbed slingshot", async () => { + slingshot.setHandler((_req, res) => { + res.writeHead(200, { "content-type": "application/json" }); + res.end(JSON.stringify({ did: "did:plc:efg", handle: "eve.test", pds: "http://pds.private.test" })); + }); + const config: ContrailConfig = { + ...baseConfig, + networkOverrides: { + slingshotUrl: `${slingshot.url}/xrpc/com.bad-example.identity.resolveMiniDoc`, + additionalAllowedHosts: ["pds.private.test"], + }, + }; + const db = await createTestDbWithSchema(); + const client = await getClient("did:plc:efg" as Did, db, config); + expect(client).toBeDefined(); + expect(slingshot.hits.some((h) => h.url.includes("identifier=did%3Aplc%3Aefg"))).toBe(true); + }); +}); diff --git a/packages/contrail/tests/postgres-concurrent-init.test.ts b/packages/contrail/tests/postgres-concurrent-init.test.ts new file mode 100644 index 0000000..4545a0d --- /dev/null +++ b/packages/contrail/tests/postgres-concurrent-init.test.ts @@ -0,0 +1,92 @@ +import { describe, it, expect, beforeAll, afterAll, beforeEach } from "vitest"; +import pg from "pg"; +import { createPostgresDatabase } from "../src/adapters/postgres"; +import { initSchema } from "../src/core/db/schema"; +import { resolveConfig } from "../src/core/types"; + +/** + * Postgres-dialect concurrent-init race. + * + * SQLite serializes DDL globally, so the existing `schema-idempotency.test.ts` + * (which uses `createSqliteDatabase`) can't surface the Postgres-specific race + * where two concurrent `CREATE TABLE IF NOT EXISTS` statements both pass the + * existence check and then both try to insert into pg_class/pg_type, with the + * loser raising 23505 on `pg_type_typname_nsp_index` (the unique index on + * (typname, typnamespace)). + * + * Real-world hit: discovered during PR44 Phase C local validation when the OM + * API consumer's `contrail-init-idempotency.spec.ts` ran three parallel + * `contrail.init(db)` calls against a fresh Postgres schema. + */ + +const TEST_CONFIG = resolveConfig({ + namespace: "com.example", + collections: { + event: { + collection: "community.lexicon.calendar.event", + queryable: { mode: {}, name: {}, startsAt: { type: "range" } }, + searchable: ["name", "description"], + relations: { + rsvps: { + collection: "rsvp", + groupBy: "status", + groups: { + going: "community.lexicon.calendar.rsvp#going", + }, + }, + }, + }, + rsvp: { + collection: "community.lexicon.calendar.rsvp", + references: { + event: { + collection: "event", + field: "subject.uri", + }, + }, + }, + }, +}); + +const PG_URL = process.env.TEST_DATABASE_URL; +if (!PG_URL) { + describe.skip("PostgreSQL concurrent init (TEST_DATABASE_URL not set)", () => { + it("skipped", () => {}); + }); +} else { + let pool: pg.Pool; + let db: ReturnType<typeof createPostgresDatabase>; + + beforeAll(async () => { + pool = new pg.Pool({ connectionString: PG_URL }); + await pool.query("SELECT 1"); + db = createPostgresDatabase(pool); + }); + + afterAll(async () => { + await pool?.end(); + }); + + beforeEach(async () => { + const tables = await pool.query( + `SELECT tablename FROM pg_tables WHERE schemaname = 'public' + AND (tablename LIKE 'records_%' OR tablename LIKE 'fts_%' + OR tablename IN ('backfills', 'discovery', 'cursor', 'identities', 'feed_items', 'feed_backfills'))` + ); + for (const { tablename } of tables.rows) { + await pool.query(`DROP TABLE IF EXISTS ${tablename} CASCADE`); + } + }); + + describe("PostgreSQL initSchema under concurrency", () => { + it("is safe to call three times concurrently against a fresh schema", async () => { + await expect( + Promise.all([ + initSchema(db, TEST_CONFIG), + initSchema(db, TEST_CONFIG), + initSchema(db, TEST_CONFIG), + ]) + ).resolves.not.toThrow(); + }); + }); +} diff --git a/packages/contrail/tests/schema-idempotency.test.ts b/packages/contrail/tests/schema-idempotency.test.ts new file mode 100644 index 0000000..9151132 --- /dev/null +++ b/packages/contrail/tests/schema-idempotency.test.ts @@ -0,0 +1,139 @@ +import { describe, it, expect } from "vitest"; +import { initSchema, addColumnIfNotExists } from "../src/core/db/schema"; +import type { Database } from "../src/core/types"; +import { resolveConfig } from "../src/core/types"; +import { createTestDb, TEST_CONFIG } from "./helpers"; + +/** + * L3 — schema idempotency. + * + * These tests pin the contract that `initSchema` is safe to call repeatedly + * (sequentially or concurrently) and that real DDL errors are NOT silently + * swallowed. The previous implementation wrapped ALTER TABLE statements in + * `try { ... } catch { /* ignore *\/ }` blocks that masked syntax errors, + * missing tables, and type mismatches alongside the intended duplicate-column + * case. After L3, only duplicate-column races are absorbed (via dialect-aware + * IF-NOT-EXISTS / PRAGMA pre-check); other DDL failures surface. + */ + +describe("initSchema idempotency", () => { + it("is safe to call twice sequentially", async () => { + const db = createTestDb(); + await initSchema(db, TEST_CONFIG); + await expect(initSchema(db, TEST_CONFIG)).resolves.not.toThrow(); + }); + + it("is safe to call concurrently", async () => { + const db = createTestDb(); + await Promise.all([ + initSchema(db, TEST_CONFIG), + initSchema(db, TEST_CONFIG), + initSchema(db, TEST_CONFIG), + ]); + + // Verify count column on records_event exists exactly once. The count + // columns are added via ALTER TABLE; duplicate adds would have failed + // without idempotent ALTER. + const cols = await db + .prepare("PRAGMA table_info(records_event)") + .all<{ name: string }>(); + const names = cols.results.map((c) => c.name); + + // There should be exactly one of each grouped-count column (sanity check + // that no parallel run got further than the first). + const countCols = names.filter((n) => n.startsWith("count_")); + const dedup = new Set(countCols); + expect(countCols.length).toBe(dedup.size); + // And we should have at least one count column (config defines rsvp groups). + expect(countCols.length).toBeGreaterThan(0); + }); + + it("does not leave duplicate count columns after repeated init", async () => { + const db = createTestDb(); + for (let i = 0; i < 5; i++) { + await initSchema(db, TEST_CONFIG); + } + const cols = await db + .prepare("PRAGMA table_info(records_event)") + .all<{ name: string }>(); + const names = cols.results.map((c) => c.name); + expect(new Set(names).size).toBe(names.length); + }); +}); + +describe("addColumnIfNotExists", () => { + // The helper is the seam where dialect-aware idempotent ALTER lives. We + // verify it both adds a column when absent and is a no-op when present. + it("adds the column when absent", async () => { + const db = createTestDb(); + await db.prepare("CREATE TABLE t (id INTEGER PRIMARY KEY)").run(); + + await addColumnIfNotExists(db, "t", "extra", "INTEGER NOT NULL DEFAULT 0"); + + const cols = await db.prepare("PRAGMA table_info(t)").all<{ name: string }>(); + expect(cols.results.map((c) => c.name)).toContain("extra"); + }); + + it("is a no-op when the column already exists", async () => { + const db = createTestDb(); + await db.prepare("CREATE TABLE t (id INTEGER PRIMARY KEY, extra INTEGER)").run(); + + // First call: column already present — should not throw. + await expect( + addColumnIfNotExists(db, "t", "extra", "INTEGER NOT NULL DEFAULT 0") + ).resolves.not.toThrow(); + + // Calling it again should still be a no-op. + await expect( + addColumnIfNotExists(db, "t", "extra", "INTEGER NOT NULL DEFAULT 0") + ).resolves.not.toThrow(); + + // And we still have exactly one `extra` column. + const cols = await db.prepare("PRAGMA table_info(t)").all<{ name: string }>(); + const extras = cols.results.filter((c) => c.name === "extra"); + expect(extras.length).toBe(1); + }); + + it("surfaces real DDL errors (target table does not exist)", async () => { + // The previous swallow-all `try { ... } catch { /* ignore */ }` masked + // *any* DDL failure, including target-table-missing. The new helper must + // only absorb the duplicate-column case and surface everything else. + const db = createTestDb(); + + await expect( + addColumnIfNotExists(db, "no_such_table", "x", "INTEGER") + ).rejects.toThrow(); + }); +}); + +describe("initSchema with extra schemas — real DDL errors surface", () => { + // Belt-and-suspenders: confirm that a genuine failure inside an extension + // schema module propagates rather than being absorbed. This is the + // user-visible behavior change from L3. + it("propagates errors thrown from an extra schema", async () => { + const db = createTestDb(); + const broken = async (_db: Database) => { + throw new Error("synthetic DDL failure"); + }; + + await expect( + initSchema(db, TEST_CONFIG, { extraSchemas: [broken] }) + ).rejects.toThrow("synthetic DDL failure"); + }); + + it("treats a config without spaces/labels/feeds the same way (sanity)", async () => { + // Minimal config with no relations — should still init cleanly twice. + const minimal = resolveConfig({ + namespace: "com.example", + collections: { + foo: { + collection: "com.example.foo", + queryable: {}, + }, + }, + }); + const db = createTestDb(); + await initSchema(db, minimal); + await expect(initSchema(db, minimal)).resolves.not.toThrow(); + }); +}); diff --git a/packages/contrail/tests/spaces-auth.test.ts b/packages/contrail/tests/spaces-auth.test.ts new file mode 100644 index 0000000..805c844 --- /dev/null +++ b/packages/contrail/tests/spaces-auth.test.ts @@ -0,0 +1,65 @@ +import { describe, it, expect } from "vitest"; +import { + PlcDidDocumentResolver, + CompositeDidDocumentResolver, + WebDidDocumentResolver, +} from "@atcute/identity-resolver"; +import { buildVerifier } from "../src/core/spaces/auth"; +import type { AuthorityConfig } from "../src/core/spaces/types"; + +const CUSTOM_PLC = "http://custom-plc.test"; + +function makeAuthority(overrides: Partial<AuthorityConfig> = {}): AuthorityConfig { + return { + type: "tools.atmo.event.space", + serviceDid: "did:web:authority.test", + ...overrides, + } as AuthorityConfig; +} + +// ServiceJwtVerifier from @atcute/xrpc-server@0.1.12 exposes the resolver as +// the public instance field `didDocResolver` (verified against +// node_modules/.../auth/jwt-verifier.d.ts). The plan's hint at `.resolver` was +// a guess — we use the real field here so the test verifies the resolver +// actually wired into the verifier instance. +describe("buildVerifier resolver precedence", () => { + it("uses AuthorityConfig.resolver when provided (most-specific wins)", () => { + const specific = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver({ apiUrl: "http://specific.test" }), + web: new WebDidDocumentResolver(), + }, + }); + const network = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver({ apiUrl: CUSTOM_PLC }), + web: new WebDidDocumentResolver(), + }, + }); + const verifier = buildVerifier(makeAuthority({ resolver: specific }), { + resolver: network, + }); + expect(verifier.didDocResolver).toBe(specific); + }); + + it("falls back to networkOverrides.resolver when authority resolver is absent", () => { + const network = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver({ apiUrl: CUSTOM_PLC }), + web: new WebDidDocumentResolver(), + }, + }); + const verifier = buildVerifier(makeAuthority(), { resolver: network }); + expect(verifier.didDocResolver).toBe(network); + }); + + it("falls back to default composite when both are absent", () => { + const verifier = buildVerifier(makeAuthority(), {}); + expect(verifier.didDocResolver).toBeInstanceOf(CompositeDidDocumentResolver); + }); + + it("treats omitted second arg the same as empty networkOverrides (backward-compat)", () => { + const verifier = buildVerifier(makeAuthority()); + expect(verifier.didDocResolver).toBeInstanceOf(CompositeDidDocumentResolver); + }); +}); -- 2.51.2 From bea0dd2c8a0bb5d3ca7da1a91caf66dbcd34355f Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 5 Jun 2026 10:56:04 -0400 Subject: [PATCH 21/25] feat(contrail): community DID provisioning on stock PDS (#31) * feat(contrail-community): provision schema, types, adapter CRUD * feat(contrail-community): PDS + PLC + service-auth helpers * feat(contrail-community): provision orchestrator + XRPC route + session cache * feat(contrail-community): reap CLI for stuck provisioning rows * test+docs(contrail): provision e2e, docs, changeset, deps * fix(contrail-community): harden provision allowlist + reap (PR #31 review) Security/review follow-ups on the community DID provisioning feature: - Provisioning fails closed: with allowProvisioning=true, an empty/undefined allowlist no longer accepts any caller pdsEndpoint (fail-open). Require a non-empty allowlist, or the explicit, loud allowAnyProvisionPdsEndpoint. Renamed allowedPdsEndpoints -> allowedProvisionPdsEndpoints to scope it to provisioning (Contrail still reads/indexes from every PDS). - reap --all-stuck gains a mandatory age floor (listStuckAttempts(olderThanMs) + --older-than <minutes>, default 30) so a bulk run can't tombstone an in-flight provision. --attempt-id still targets a known row regardless. - reap is reachable: ship it as a contrail-community bin (under pnpm the core contrail CLI can't resolve the community package; the dynamic import there now logs instead of an empty catch). Adds Postgres support via --db/DATABASE_URL alongside the D1 binding. - archiveStuckAttempt is now atomic (db.batch) + idempotent (ON CONFLICT DO NOTHING), so a retry after a partial failure can't PK-conflict. - Minor: export bytesToB64url from plc and reuse in service-auth; drop the duplicate reap mutual-exclusion validator. mint-route allowlist gap (pre-existing on main) tracked separately in om-vyok. * test(contrail-community): drop tautological builder-shape assertions Remove the 'buildTombstoneOp produces the expected shape' case and the two pass-through type/prev assertions in the update-op test: both assert literals the builder just set or args just passed in, exercising object-literal semantics rather than runtime behavior. The real signing behavior (signTombstoneOp / signUpdateOp base64url sig) and the live-PLC CID round-trip remain covered. * docs(contrail-community): document SSRF exposure of allowAnyProvisionPdsEndpoint The accept-any escape hatch bypasses endpoint validation: the caller-supplied pdsEndpoint reaches describeServer + createAccount fetches with no private-IP/metadata guard. Note inline that it must be paired with trusted auth + egress network policy + IMDSv2; the default allowlist path contains this by construction. No behavior change. --- .changeset/itchy-rice-kick.md | 22 + .gitignore | 1 + .../community-provision-walkthrough.test.ts | 402 ++++++++++++++ apps/contrail-e2e/tests/provision.test.ts | 490 ++++++++++++++++++ .../contrail-e2e/tests/reap-tombstone.test.ts | 178 +++++++ docs/07-communities.md | 25 +- packages/contrail-community/package.json | 19 + packages/contrail-community/src/adapter.ts | 254 +++++++++ packages/contrail-community/src/cli/index.ts | 49 ++ packages/contrail-community/src/cli/reap.ts | 397 ++++++++++++++ packages/contrail-community/src/index.ts | 50 +- packages/contrail-community/src/pds.ts | 207 ++++++++ packages/contrail-community/src/plc.ts | 152 +++++- packages/contrail-community/src/provision.ts | 403 ++++++++++++++ packages/contrail-community/src/router.ts | 337 +++++++++++- packages/contrail-community/src/schema.ts | 51 ++ .../contrail-community/src/service-auth.ts | 51 ++ packages/contrail-community/src/types.ts | 73 ++- .../contrail-community/tests/cli-reap.test.ts | 388 ++++++++++++++ .../community-provision-attempts.test.ts | 247 +++++++++ .../community-provision-pds-allowlist.test.ts | 301 +++++++++++ .../tests/community-provision-router.test.ts | 336 ++++++++++++ ...mmunity-publish-401-clears-session.test.ts | 150 ++++++ .../tests/community-publishing.test.ts | 375 +++++++++++++- .../tests/community-sessions-cache.test.ts | 85 +++ .../tests/pds-account-ops.test.ts | 81 +++ .../tests/pds-create-account.test.ts | 74 +++ .../tests/plc-log-last.test.ts | 81 +++ .../tests/plc-update-op.test.ts | 50 ++ .../tests/provision-orchestrator.test.ts | 259 +++++++++ .../tests/provision-self-sovereign.test.ts | 353 +++++++++++++ .../contrail-community/tests/schema.test.ts | 18 + .../tests/service-auth.test.ts | 135 +++++ packages/contrail-community/tsup.config.ts | 4 +- packages/contrail/src/cli.ts | 37 ++ packages/contrail/src/index.ts | 2 +- packages/contrail/tests/schema.test.ts | 1 + pnpm-lock.yaml | 15 + 38 files changed, 6127 insertions(+), 26 deletions(-) create mode 100644 .changeset/itchy-rice-kick.md create mode 100644 apps/contrail-e2e/tests/community-provision-walkthrough.test.ts create mode 100644 apps/contrail-e2e/tests/provision.test.ts create mode 100644 apps/contrail-e2e/tests/reap-tombstone.test.ts create mode 100644 packages/contrail-community/src/cli/index.ts create mode 100644 packages/contrail-community/src/cli/reap.ts create mode 100644 packages/contrail-community/src/provision.ts create mode 100644 packages/contrail-community/src/service-auth.ts create mode 100644 packages/contrail-community/tests/cli-reap.test.ts create mode 100644 packages/contrail-community/tests/community-provision-attempts.test.ts create mode 100644 packages/contrail-community/tests/community-provision-pds-allowlist.test.ts create mode 100644 packages/contrail-community/tests/community-provision-router.test.ts create mode 100644 packages/contrail-community/tests/community-publish-401-clears-session.test.ts create mode 100644 packages/contrail-community/tests/community-sessions-cache.test.ts create mode 100644 packages/contrail-community/tests/pds-account-ops.test.ts create mode 100644 packages/contrail-community/tests/pds-create-account.test.ts create mode 100644 packages/contrail-community/tests/plc-log-last.test.ts create mode 100644 packages/contrail-community/tests/plc-update-op.test.ts create mode 100644 packages/contrail-community/tests/provision-orchestrator.test.ts create mode 100644 packages/contrail-community/tests/provision-self-sovereign.test.ts create mode 100644 packages/contrail-community/tests/schema.test.ts create mode 100644 packages/contrail-community/tests/service-auth.test.ts diff --git a/.changeset/itchy-rice-kick.md b/.changeset/itchy-rice-kick.md new file mode 100644 index 0000000..84f0750 --- /dev/null +++ b/.changeset/itchy-rice-kick.md @@ -0,0 +1,22 @@ +--- +"@atmo-dev/contrail-community": minor +"@atmo-dev/contrail": minor +--- + +A third community-creation mode: **provision**. alongside the existing `adopt` (caller already has a `did:plc`) and `mint` (caller wants a DID but brings their own PDS) modes, contrail can now provision a community on a stock `@atproto/pds` end-to-end — minting the `did:plc`, creating and activating the PDS account, generating an app password, and persisting credentials so the existing `community.putRecord` / `.deleteRecord` publish path keeps working. contrail never holds PDS admin credentials. + +**`xrpc/{ns}.community.provision`** runs the five-step PLC + PDS dance (key generation → PLC genesis → `createAccount` → `getRecommendedDidCredentials` + signed PLC update op → `activateAccount`), persists each step in a new `provision_attempts` table so a partially-failed attempt can be resumed, mints an app password, and seeds the session cache. + +**`contrail-community reap [--all-stuck] [--older-than <minutes>] [--db <url>] [--dry-run]`** new CLI (a bin shipped by `@atmo-dev/contrail-community`) that cleans up provision attempts which didn't reach `status='activated'` by tombstoning their PLC entries. `--dry-run` is the default; per-row confirmation is required for live reaping unless `--all-stuck` is given. `--all-stuck` only acts on rows idle at least `--older-than` minutes (default 30) so a bulk run can't tombstone an in-flight provision. Runs against the Cloudflare D1 binding by default, or against the decoupled Postgres index when `--db`/`DATABASE_URL` is set. It ships as a contrail-community bin because the PR #30 package split removed contrail's edge into community code: under pnpm's isolated `node_modules` the core `contrail` CLI can't resolve `@atmo-dev/contrail-community`, so `contrail reap` only registers in hoisted installs where both packages sit together. + +custody model: the caller supplies a `rotationKey` and that key sits at `rotationKeys[0]` — the highest-priority rotation slot on the resulting DID. contrail generates a subordinate keypair and persists it (AES-GCM-encrypted under `masterKey`) at `rotationKeys[1]`, so it can submit later PLC ops on the community's behalf — most importantly the post-activation PLC update during provision, and the tombstone op that `reap` issues to clean up stuck DIDs. + +the caller's key dominates: PLC's 72-hour nullification window means any op contrail signs with its subordinate key can be overridden within 72h by an op signed with the caller's key. with this caveat: a tombstone is irrevocable. a malicious or compromised contrail instance could tombstone any DID it provisioned. there is no managed code path, no shared rotation, and `rootCredentials` are returned to the caller in the response so they can also be persisted out-of-band. + +what you need to configure / know: + +- new `community` config block: `masterKey` (32-byte AES-GCM envelope key for the encrypted credential columns), `allowedProvisionPdsEndpoints` (URL-origin matching, collapses scheme case / default ports / trailing slash / IDN), optional `plcDirectory` override. + +- **provisioning fails closed.** When `allowProvisioning` is true, `allowedProvisionPdsEndpoints` MUST be non-empty — a missing/empty allowlist no longer means "accept any PDS" (that was a fail-open hole: any caller could have a PLC genesis op signed by Contrail's rotation key against an attacker-chosen PDS). To deliberately accept any endpoint, set the separate, loud `allowAnyProvisionPdsEndpoint: true`. The field was renamed from `allowedPdsEndpoints` to make clear it gates *provisioning* only, not which PDSes Contrail reads/indexes. + +- new tables `provision_attempts` and `community_credentials`. credentials are stored AES-GCM-encrypted under that key; lose the key, lose the ability to mint sessions for previously-provisioned communities. diff --git a/.gitignore b/.gitignore index 09e8139..44f8311 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ node_modules .wrangler dist +*.tsbuildinfo # Turbo .turbo diff --git a/apps/contrail-e2e/tests/community-provision-walkthrough.test.ts b/apps/contrail-e2e/tests/community-provision-walkthrough.test.ts new file mode 100644 index 0000000..e9d847c --- /dev/null +++ b/apps/contrail-e2e/tests/community-provision-walkthrough.test.ts @@ -0,0 +1,402 @@ +/** + * Provisioned-community lifecycle walkthrough — the "happy day" the PR was + * built for, end-to-end on the same handler. + * + * Each previous test pins one slice (provision-only, publishing-only, + * ACL-only, ingest-only). This one chains them so a regression on any seam + * between modules — provision → ACL → proxy publish → ingest of a RSVP from + * a separate PDS repo — surfaces here even when each unit test still passes. + * + * Flow (each step is also asserted, so the test reads top-to-bottom as docs): + * + * 1. PROVISION — Alice calls `community.provision` with a caller-held + * P-256 rotation key (sovereign mode). Asserts: status=activated, + * DID well-formed, PLC log shows the caller's did:key at + * rotationKeys[0] (the sovereignty invariant). + * + * 2. GRANT — Alice grants Bob `member` on the community's `$publishers` + * space via `community.space.grant`. Asserts: listMembers shows Bob + * with accessLevel=member. + * + * 3. PUBLISH — Bob calls `community.putRecord` to write a public + * `community.lexicon.calendar.event` against the community DID's repo + * (proxied through Contrail's credential vault — Bob never holds the + * community's app password). Asserts: returned URI is rooted at the + * community DID; the record is visible via `com.atproto.repo.listRecords` + * against the community's PDS (proves the proxy actually wrote to the + * community repo, not a local index). + * + * 4. RSVP — Carol, a totally separate PDS account with no relationship + * to the community, writes a `community.lexicon.calendar.rsvp` to her + * own repo with `subject.uri = at://<communityDid>/.../<rkey>`. + * Anyone can RSVP — no grant required, that's the lexicon contract. + * + * 5. INDEX — The in-process ingester (Jetstream → Postgres) picks up both + * Bob's event and Carol's RSVP. Asserts: querying the event by URI + * shows rsvpsGoingCount=1, with the indexed event `did` being the + * community's DID (not Bob's, not Alice's). + * + * Prereqs: `pnpm stack:up` (devnet PDS+PLC + postgres reachable). + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import pg from "pg"; +import type { Client } from "@atcute/client"; +import "@atcute/atproto"; +import { + Contrail, + generateKeyPair, + runPersistent, +} from "@atmo-dev/contrail"; +import { createHandler } from "@atmo-dev/contrail/server"; +import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; +import { config as baseConfig } from "../config"; +import { + CONTRAIL_SERVICE_DID, + HANDLE_DOMAIN, + PDS_ADMIN_PASSWORD, + PDS_URL, + PLC_URL, + createCaller, + createDevnetResolver, + createIsolatedSchema, + createTestAccount, + devnetRewriteFetch, + getRecordFromPds, + jsonOr, + login, + waitFor, + type CallAs, + type TestAccount, +} from "./helpers"; + +const NS = `${baseConfig.namespace}.community`; +const SPACE_TYPE = "rsvp.atmo.event.space"; +const EVENT_NSID = "community.lexicon.calendar.event"; +const RSVP_NSID = "community.lexicon.calendar.rsvp"; +const TEST_MASTER_KEY = new Uint8Array(32).fill(7); + +describe("community provision → grant → publish → RSVP walkthrough", () => { + // Alice provisions the community (becomes owner of $admin and $publishers). + // Bob is granted member on $publishers and publishes the event on behalf + // of the community via the proxy. Carol is an arm's-length user on the + // same PDS who RSVPs from her own repo — she has no grants on the + // community, which is exactly the open-RSVP contract we want to pin. + let alice: TestAccount; + let bob: TestAccount; + let carol: TestAccount; + + let aliceClient: Client; + let bobClient: Client; + let carolClient: Client; + + let pool: pg.Pool; + let cleanupSchema: () => Promise<void>; + let pdsDid: string; + let handle: (req: Request) => Promise<Response>; + let callAs: CallAs; + + let ingestController: AbortController; + let ingestPromise: Promise<void>; + + // Keypair held only by this test process; the public did:key is what we + // pass to provision. The private JWK never leaves the test — that's the + // sovereignty invariant we assert against the PLC log in step 1. + let callerRotation: Awaited<ReturnType<typeof generateKeyPair>>; + + // Carried between tests in declaration order. + let communityDid: string; + let publishersUri: string; + let eventUri: string; + let eventCid: string; + let eventRkey: string; + + beforeAll(async () => { + // Discover the live PDS's DID — the orchestrator uses this as the `aud` + // claim of the service-auth JWT it mints for createAccount, and the + // devnet PDS validates `aud` against its own DID. + const dres = await fetch(`${PDS_URL}/xrpc/com.atproto.server.describeServer`); + if (!dres.ok) { + throw new Error( + `devnet PDS unreachable at ${PDS_URL}: ${dres.status} ${await dres.text()}`, + ); + } + pdsDid = ((await dres.json()) as { did?: string }).did!; + + [alice, bob, carol] = await Promise.all([ + createTestAccount(), + createTestAccount(), + createTestAccount(), + ]); + + aliceClient = await login(alice); + bobClient = await login(bob); + carolClient = await login(carol); + + callerRotation = await generateKeyPair(); + + const iso = await createIsolatedSchema("test_provision_walkthrough"); + pool = iso.pool; + cleanupSchema = iso.cleanup; + const db = createPostgresDatabase(pool); + + const contrail = new Contrail({ + ...baseConfig, + db, + spaces: { + type: SPACE_TYPE, + serviceDid: CONTRAIL_SERVICE_DID, + resolver: createDevnetResolver(), + }, + community: { + // Provision uses serviceDid as the `aud` of its createAccount + // service-auth JWT — must be the live PDS's DID, not the Contrail + // service DID we use for inbound auth verification. + serviceDid: pdsDid, + masterKey: TEST_MASTER_KEY, + plcDirectory: PLC_URL, + resolver: createDevnetResolver(), + // Devnet PDSes publish https://devnet.test in their DID document's + // atproto_pds entry. Rewrite outgoing requests so the proxied + // publish lands on the host-mapped port. + fetch: devnetRewriteFetch, + allowProvisioning: true, + }, + }); + await contrail.init(); + handle = createHandler(contrail); + callAs = createCaller(handle); + + // Run the ingester in-process so step 5 can see Carol's RSVP land in + // the local index after she writes it directly to her PDS repo. + ingestController = new AbortController(); + ingestPromise = runPersistent(db, baseConfig, { + batchSize: 50, + flushIntervalMs: 500, + signal: ingestController.signal, + }); + }, 30_000); + + afterAll(async () => { + ingestController?.abort(); + await ingestPromise?.catch(() => {}); + await cleanupSchema?.(); + }); + + // ----- helpers -------------------------------------------------------------- + + async function getIndexedRecord(uri: string): Promise<any | undefined> { + const url = + `http://test/xrpc/${baseConfig.namespace}.event.getRecord?uri=${encodeURIComponent(uri)}`; + const res = await handle(new Request(url)); + if (res.status === 404) return undefined; + if (!res.ok) throw new Error(`getRecord ${uri} → ${res.status}: ${await res.text()}`); + return await res.json(); + } + + async function mintPdsInvite(): Promise<string> { + const res = await fetch(`${PDS_URL}/xrpc/com.atproto.server.createInviteCode`, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Basic ${Buffer.from( + `admin:${PDS_ADMIN_PASSWORD}`, + ).toString("base64")}`, + }, + body: JSON.stringify({ useCount: 1 }), + }); + if (!res.ok) { + throw new Error(`createInviteCode → ${res.status}: ${await res.text()}`); + } + return ((await res.json()) as { code: string }).code; + } + + // ----- step 1: PROVISION --------------------------------------------------- + // Sovereign provision: the caller passes the public did:key of a rotation + // key they hold. Contrail mints a subordinate rotation key, lays down a + // genesis op with [callerKey, contrailKey] in that order, then runs an + // update op to install the PDS-recommended verification methods. The PLC + // log must end with the caller's key still at rotationKeys[0] — without + // that, recovery authority silently moved to Contrail. + + it("step 1 — provisions a sovereign community via XRPC and the PLC log shows the caller's rotation key first", async () => { + const inviteCode = await mintPdsInvite(); + + // Devnet PDS caps the local handle label at 18 chars. `pw-` prefix + + // 8-char suffix keeps the full label well under that. + const suffix = `${Date.now().toString(36).slice(-6)}${Math.random() + .toString(36) + .slice(2, 4)}`; + const newHandle = `pw-${suffix}${HANDLE_DOMAIN}`; + const email = `${suffix}@devnet.test`; + const password = `pw-${suffix}`; + + const res = await callAs(aliceClient, "POST", `${NS}.provision`, { + body: { + handle: newHandle, + email, + password, + inviteCode, + pdsEndpoint: PDS_URL, + rotationKey: callerRotation.publicDidKey, + }, + }); + expect(res.status, await res.clone().text()).toBe(200); + const body = (await res.json()) as { communityDid: string; status: string }; + expect(body.status).toBe("activated"); + expect(body.communityDid).toMatch(/^did:plc:[a-z2-7]{24}$/); + communityDid = body.communityDid; + + // PLC sovereignty check: latest op (the post-activation update op) keeps + // the caller's did:key at rotationKeys[0]. If this regresses, Contrail's + // subordinate key would silently take rotation priority. + const logRes = await fetch(`${PLC_URL}/${communityDid}/log`); + expect(logRes.ok).toBe(true); + const log = (await logRes.json()) as Array<{ rotationKeys: string[] }>; + expect(log.length).toBeGreaterThanOrEqual(2); + expect(log[log.length - 1]!.rotationKeys[0]).toBe(callerRotation.publicDidKey); + + publishersUri = `ats://${communityDid}/${SPACE_TYPE}/$publishers`; + }, 30_000); + + // ----- step 2: GRANT ------------------------------------------------------- + // bootstrapReservedSpaces seeded $publishers with Alice as owner. To let + // Bob publish on behalf of the community, Alice grants him `member` on + // $publishers — the minimum level the putRecord guard accepts. + + it("step 2 — Alice grants Bob `member` on $publishers and listMembers reflects it", async () => { + expect(communityDid, "step 1 must have provisioned the community").toBeTruthy(); + + const res = await callAs(aliceClient, "POST", `${NS}.space.grant`, { + body: { + spaceUri: publishersUri, + subject: { did: bob.did }, + accessLevel: "member", + }, + }); + expect(res.status, await res.clone().text()).toBe(200); + + const list = await callAs(aliceClient, "GET", `${NS}.space.listMembers`, { + query: { spaceUri: publishersUri }, + }); + expect(list.status).toBe(200); + const { rows } = (await jsonOr(list)) as { + rows: Array<{ subject: { did?: string }; accessLevel: string }>; + }; + const byDid = Object.fromEntries( + rows.filter((r) => r.subject.did).map((r) => [r.subject.did, r.accessLevel]), + ); + expect(byDid[alice.did]).toBe("owner"); + expect(byDid[bob.did]).toBe("member"); + }); + + // ----- step 3: PUBLISH ----------------------------------------------------- + // Bob calls community.putRecord. The router checks his level on + // $publishers (member ≥ member, OK), pulls the community's encrypted + // app password from the credential vault, opens a PDS session as the + // community DID, and proxies a com.atproto.repo.createRecord. The + // returned URI is rooted at the community DID — Bob never holds those + // credentials, and his own DID doesn't appear anywhere in the record. + + it("step 3 — Bob (a $publishers member) publishes a public event as the community via proxy", async () => { + expect(publishersUri, "step 2 must have granted bob").toBeTruthy(); + + const eventName = `walkthrough-event ${Date.now()}`; + const startsAt = new Date(Date.now() + 60 * 60_000).toISOString(); + + const res = await callAs(bobClient, "POST", `${NS}.putRecord`, { + body: { + communityDid, + collection: EVENT_NSID, + record: { + $type: EVENT_NSID, + name: eventName, + createdAt: new Date().toISOString(), + startsAt, + mode: `${EVENT_NSID}#inperson`, + status: `${EVENT_NSID}#scheduled`, + }, + }, + }); + expect(res.status, await res.clone().text()).toBe(200); + const out = (await res.json()) as { uri: string; cid: string }; + eventUri = out.uri; + eventCid = out.cid; + eventRkey = out.uri.split("/").pop()!; + + // URI is rooted at the community DID, not Bob's. + expect(eventUri).toMatch(new RegExp(`^at://${communityDid}/${EVENT_NSID}/`)); + + // The record is visible via the community PDS's listRecords — proves + // the proxy actually wrote to the community repo, not just Contrail's + // local index. listRecords is the lexicon endpoint the prompt named; + // we round it out with a getRecord on the same rkey to confirm payload. + const listUrl = + `${PDS_URL}/xrpc/com.atproto.repo.listRecords` + + `?repo=${encodeURIComponent(communityDid)}` + + `&collection=${encodeURIComponent(EVENT_NSID)}` + + `&limit=10`; + const listRes = await fetch(listUrl); + expect(listRes.ok, `listRecords ${listRes.status}`).toBe(true); + const listed = (await listRes.json()) as { + records: Array<{ uri: string; cid: string; value: { name?: string } }>; + }; + const found = listed.records.find((r) => r.uri === eventUri); + expect(found, `event ${eventUri} not in PDS listRecords`).toBeDefined(); + expect(found!.value.name).toBe(eventName); + + const onPds = await getRecordFromPds(communityDid, EVENT_NSID, eventRkey); + expect(onPds.status).toBe(200); + expect(onPds.record.name).toBe(eventName); + }, 30_000); + + // ----- step 4: RSVP -------------------------------------------------------- + // Carol writes a community.lexicon.calendar.rsvp to her OWN repo on the + // shared devnet PDS, with subject.uri pointing at the community's event. + // She has zero relationship to the community — that's the open-RSVP + // contract: anyone can RSVP, the record lives in the responder's repo. + + it("step 4 — Carol RSVPs from a separate PDS account by writing to her own repo", async () => { + expect(eventUri, "step 3 must have published the event").toBeTruthy(); + + const rsvpRes = await carolClient.post("com.atproto.repo.createRecord", { + input: { + repo: carol.did, + collection: RSVP_NSID, + record: { + $type: RSVP_NSID, + subject: { uri: eventUri, cid: eventCid }, + status: `${RSVP_NSID}#going`, + createdAt: new Date().toISOString(), + }, + }, + }); + expect(rsvpRes.ok, `RSVP createRecord: ${JSON.stringify(rsvpRes.data)}`).toBe(true); + if (!rsvpRes.ok) throw new Error("unreachable"); + expect(rsvpRes.data.uri).toMatch(new RegExp(`^at://${carol.did}/${RSVP_NSID}/`)); + }); + + // ----- step 5: INDEX ------------------------------------------------------- + // Both records hit Jetstream and the in-process ingester. Querying the + // event by URI from the local handler should hydrate rsvpsGoingCount=1 + // (Carol's RSVP referencing it), and the indexed `did` must be the + // community's, not Bob's — proves end-to-end attribution works. + + it("step 5 — the indexer surfaces the event under the community DID with Carol's RSVP counted", async () => { + expect(eventUri, "step 3 must have published the event").toBeTruthy(); + + const indexed = await waitFor( + async () => { + const r = await getIndexedRecord(eventUri); + return r && r.rsvpsGoingCount >= 1 ? r : undefined; + }, + { label: `indexed ${eventUri} with rsvpsGoingCount>=1`, timeoutMs: 20_000 }, + ); + + expect(indexed.uri).toBe(eventUri); + // Attribution: the event belongs to the community, not the publisher. + expect(indexed.did).toBe(communityDid); + expect(indexed.did).not.toBe(bob.did); + expect(indexed.did).not.toBe(alice.did); + expect(indexed.rsvpsGoingCount).toBe(1); + }, 30_000); +}); diff --git a/apps/contrail-e2e/tests/provision.test.ts b/apps/contrail-e2e/tests/provision.test.ts new file mode 100644 index 0000000..aefed96 --- /dev/null +++ b/apps/contrail-e2e/tests/provision.test.ts @@ -0,0 +1,490 @@ +/** + * End-to-end test exercising the full ProvisionOrchestrator flow against the + * live devnet stack (PDS on :4000, PLC on :2582). Validates the 5-RPC sequence + * genesis op → createAccount → getRecommendedDidCredentials + * → PLC update op → activateAccount + * lands an activated account. + * + * Catches integration bugs that mocks can't: + * - hand-rolled ES256 service-auth JWT vs real atproto verifier + * - hand-rolled DAG-CBOR encoder output vs real PLC parser + * - genesis-op DID computation matches what PLC expects + * - cidForOp output accepted by PLC as `prev` for the update op + * - low-S signature normalization + * + * Prereqs: `pnpm stack:up` (devnet PDS+PLC + postgres reachable). + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { randomUUID } from "node:crypto"; +import pg from "pg"; +import type { Client } from "@atcute/client"; +import { + CommunityAdapter, + CredentialCipher, + Contrail, + ProvisionOrchestrator, + initCommunitySchema, + pdsCreateAccount, + pdsGetRecommendedDidCredentials, + pdsActivateAccount, + pdsCreateAppPassword, + generateKeyPair, + createPdsSession, + submitGenesisOp, + type PdsClient, + type PlcClient, +} from "@atmo-dev/contrail"; +import { createHandler } from "@atmo-dev/contrail/server"; +import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; +import { + PDS_URL, + PLC_URL, + HANDLE_DOMAIN, + PDS_ADMIN_PASSWORD, + CONTRAIL_SERVICE_DID, + createCaller, + createDevnetResolver, + createIsolatedSchema, + createTestAccount, + login, + type CallAs, + type TestAccount, +} from "./helpers"; + +describe("ProvisionOrchestrator devnet e2e", () => { + let pool: pg.Pool; + let cleanupSchema: () => Promise<void>; + let adapter: CommunityAdapter; + let cipher: CredentialCipher; + let pdsDid: string; + + beforeAll(async () => { + // Discover the live PDS's DID via describeServer — used as the `aud` + // claim in the service-auth JWT we mint for createAccount. + const res = await fetch(`${PDS_URL}/xrpc/com.atproto.server.describeServer`); + if (!res.ok) { + throw new Error( + `devnet PDS unreachable at ${PDS_URL}: ${res.status} ${await res.text()}`, + ); + } + const body = (await res.json()) as { did?: string }; + if (!body.did) { + throw new Error(`describeServer response missing did: ${JSON.stringify(body)}`); + } + pdsDid = body.did; + + const iso = await createIsolatedSchema("test_provision_e2e"); + pool = iso.pool; + cleanupSchema = iso.cleanup; + const db = createPostgresDatabase(pool); + await initCommunitySchema(db); + adapter = new CommunityAdapter(db); + cipher = new CredentialCipher(new Uint8Array(32).fill(7)); + }, 15_000); + + afterAll(async () => { + await cleanupSchema?.(); + }); + + // Adapt our bare module-level functions to the orchestrator's wrapper + // interfaces. Shared by the managed and self-sovereign tests so they hit + // the same live PDS surface (Task 16 added createAppPassword to PdsClient). + const pdsClient: PdsClient = { + createAccount: ({ pdsUrl, serviceAuthJwt, body }) => + pdsCreateAccount(pdsUrl, serviceAuthJwt, body), + getRecommendedDidCredentials: ({ pdsUrl, accessJwt }) => + pdsGetRecommendedDidCredentials(pdsUrl, accessJwt), + activateAccount: ({ pdsUrl, accessJwt }) => + pdsActivateAccount(pdsUrl, accessJwt), + createAppPassword: ({ pdsUrl, accessJwt, name }) => + pdsCreateAppPassword(pdsUrl, accessJwt, name), + }; + + const plcClient: PlcClient = { + submit: (did, op) => submitGenesisOp(PLC_URL, did, op as any), + }; + + /** Mint a single-use invite via the PDS admin API. Shared helper for both + * the managed and self-sovereign tests. */ + async function mintInvite(): Promise<string> { + const inviteRes = await fetch( + `${PDS_URL}/xrpc/com.atproto.server.createInviteCode`, + { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Basic ${Buffer.from( + `admin:${PDS_ADMIN_PASSWORD}`, + ).toString("base64")}`, + }, + body: JSON.stringify({ useCount: 1 }), + }, + ); + if (!inviteRes.ok) { + throw new Error( + `createInviteCode failed (${inviteRes.status}): ${await inviteRes.text()}`, + ); + } + return ((await inviteRes.json()) as { code: string }).code; + } + + it( + "provisions a self-sovereign community: caller holds rotation key, contrail mints app password", + async () => { + // Caller-held rotation keypair. The private JWK never leaves this test — + // only callerRotation.publicDidKey is passed to the orchestrator. That's + // the negative invariant we assert below: no encrypted_* column on the + // persisted row contains the caller's did:key after decrypt. + const callerRotation = await generateKeyPair(); + + const inviteCode = await mintInvite(); + + // Keep handle short — devnet caps the local label at 18 chars. + // `ss-` (3) + 8-char suffix = 11 chars on the local label. + const suffix = `${Date.now().toString(36).slice(-5)}${Math.random() + .toString(36) + .slice(2, 5)}`; + const handle = `ss-${suffix}${HANDLE_DOMAIN}`; + const email = `${suffix}@devnet.test`; + const password = `pw-${suffix}`; + const attemptId = randomUUID(); + + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: plcClient, + pds: pdsClient, + pdsDid, + }); + + const result = await orch.provision({ + attemptId, + pdsEndpoint: PDS_URL, + handle, + email, + password, + inviteCode, + rotationKey: callerRotation.publicDidKey, + }); + + // Result-shape assertions: status activated; rootCredentials returned + // with the user's *root* password, not the minted app password. + expect(result.attemptId).toBe(attemptId); + expect(result.did).toMatch(/^did:plc:[a-z2-7]{24}$/); + expect(result.status).toBe("activated"); + expect(result.rootCredentials).toBeDefined(); + expect(result.rootCredentials!.handle).toBe(handle); + expect(result.rootCredentials!.password).toBe(password); + expect(typeof result.rootCredentials!.recoveryHint).toBe("string"); + expect(result.rootCredentials!.recoveryHint.length).toBeGreaterThan(0); + + // Persisted-row assertions: self-sovereign mode persists an *encrypted + // app password* — never the user's root password. Contrail's rotation + // key is the SUBORDINATE (rotationKeys[1]); the caller's did:key is + // rotationKeys[0] in the genesis op and lives only in PLC, never in + // any encrypted_* column. + const row = await adapter.getProvisionAttempt(attemptId); + expect(row).not.toBeNull(); + expect(row!.status).toBe("activated"); + expect(row!.did).toBe(result.did); + expect(row!.handle).toBe(handle); + expect(row!.encryptedSigningKey).toBeTruthy(); + expect(row!.encryptedRotationKey).toBeTruthy(); + expect(row!.encryptedPassword).toBeTruthy(); + expect(row!.activatedAt).toBeTruthy(); + expect(row!.lastError).toBeNull(); + + // Decrypt the persisted password — it must be the *minted app password*, + // distinct from the user's root password we supplied. + const decryptedAppPassword = await cipher.decryptString( + row!.encryptedPassword!, + ); + expect(decryptedAppPassword).not.toBe(password); + expect(decryptedAppPassword.length).toBeGreaterThan(0); + + // Decrypt the persisted rotation JWK — it must be Contrail's subordinate + // key (a fresh P-256 keypair), NOT the caller's. We assert NOT-equal on + // the JWK shape, including the `d` (private) coordinate which the caller + // never sent. + const decryptedRotationJwk = JSON.parse( + await cipher.decryptString(row!.encryptedRotationKey!), + ) as { kty?: string; crv?: string; x?: string; y?: string; d?: string }; + expect(decryptedRotationJwk.kty).toBe("EC"); + expect(decryptedRotationJwk.crv).toBe("P-256"); + // The caller's private `d` coordinate must never appear in Contrail's + // persistence — the strongest single-bit invariant of self-sovereign mode. + expect(decryptedRotationJwk.d).not.toBe(callerRotation.privateJwk.d); + // The public x/y must also differ — the persisted rotation key is a + // subordinate Contrail-generated key, not a re-derivation of the caller's. + expect(decryptedRotationJwk.x).not.toBe(callerRotation.privateJwk.x); + expect(decryptedRotationJwk.y).not.toBe(callerRotation.privateJwk.y); + + // Negative invariant: the caller's public did:key string must NOT appear + // inside ANY encrypted column after decryption. Encrypted_signing_key + // is a JWK; encrypted_rotation_key is the subordinate JWK; the password + // is opaque — none of them should contain the caller's did:key. + const decryptedSigningKey = await cipher.decryptString( + row!.encryptedSigningKey!, + ); + const callerDidKey = callerRotation.publicDidKey; + expect(decryptedSigningKey.indexOf(callerDidKey)).toBe(-1); + expect( + await cipher + .decryptString(row!.encryptedRotationKey!) + .then((s) => s.indexOf(callerDidKey)), + ).toBe(-1); + expect(decryptedAppPassword.indexOf(callerDidKey)).toBe(-1); + + // Prove the *minted* app password works against the live PDS. + // createPdsSession throws if the PDS rejects. + const appSession = await createPdsSession( + PDS_URL, + handle, + decryptedAppPassword, + ); + expect(appSession.did).toBe(result.did); + expect(appSession.accessJwt).toBeTruthy(); + + // And the user's root password also still works — PDS supports multiple + // credentials per account, so the caller's root creds remain valid. + const rootSession = await createPdsSession(PDS_URL, handle, password); + expect(rootSession.did).toBe(result.did); + expect(rootSession.accessJwt).toBeTruthy(); + + // PLC-log assertion (H2): the post-activation update op must keep the + // caller's did:key at rotationKeys[0]. Without this, contrail's + // subordinate would silently take rotation priority and the caller + // would lose self-sovereign recovery authority. + const logRes = await fetch(`${PLC_URL}/${result.did}/log`); + expect(logRes.ok).toBe(true); + const log = (await logRes.json()) as Array<{ + rotationKeys: string[]; + }>; + expect(log.length).toBeGreaterThanOrEqual(2); + const lastOp = log[log.length - 1]!; + expect(lastOp.rotationKeys[0]).toBe(callerRotation.publicDidKey); + }, + 30_000, + ); +}); + +/** + * Routed end-to-end coverage for the XRPC surface of the provision flow: + * `${NS}.community.provision` then `${NS}.community.putRecord` against the + * same provisioned community. Differs from the orchestrator-only test above + * by exercising the full Hono app — auth middleware, DB persistence, + * bootstrapReservedSpaces, and the credential-proxy publish path that + * Tasks 13/14 added. + * + * Also pins the Task 14 session-cache behavior: two sequential putRecords + * should perform exactly **one** `com.atproto.server.createSession` call to + * the PDS — the second hits the cached session. + */ +describe("community.provision + putRecord via XRPC route (devnet)", () => { + const NS = "rsvp.atmo.community"; + const SPACE_TYPE = "rsvp.atmo.event.space"; + const POST_NSID = "app.bsky.feed.post"; + const TEST_MASTER_KEY = new Uint8Array(32).fill(7); + + let pool: pg.Pool; + let cleanupSchema: () => Promise<void>; + let pdsDid: string; + let alice: TestAccount; + let aliceClient: Client; + let handle: (req: Request) => Promise<Response>; + let callAs: CallAs; + + // Counts createSession calls to the live PDS so we can assert that the + // session cache is reused across publishes (Task 14). + let createSessionCount = 0; + + beforeAll(async () => { + // Discover the live PDS's DID — needed as `aud` in the orchestrator's + // service-auth JWT for createAccount. + const res = await fetch(`${PDS_URL}/xrpc/com.atproto.server.describeServer`); + if (!res.ok) { + throw new Error( + `devnet PDS unreachable at ${PDS_URL}: ${res.status} ${await res.text()}`, + ); + } + pdsDid = ((await res.json()) as { did?: string }).did!; + + const iso = await createIsolatedSchema("test_provision_router_e2e"); + pool = iso.pool; + cleanupSchema = iso.cleanup; + const db = createPostgresDatabase(pool); + + // Wrap fetch to count createSession calls. Everything else passes + // through unchanged so the orchestrator + publish paths hit real devnet. + const countingFetch: typeof fetch = (input, init) => { + const url = typeof input === "string" ? input : input.toString(); + if (url.includes("/xrpc/com.atproto.server.createSession")) { + createSessionCount++; + } + return fetch(input as any, init); + }; + + const contrail = new Contrail({ + ...{ + namespace: "rsvp.atmo", + collections: { + // Minimal collection set — we only need community routes registered; + // no Jetstream ingestion is required for this test. + post: { collection: POST_NSID }, + }, + }, + db, + spaces: { + type: SPACE_TYPE, + serviceDid: CONTRAIL_SERVICE_DID, + resolver: createDevnetResolver(), + }, + community: { + // The orchestrator uses cfg.serviceDid as the `aud` of the + // createAccount service-auth JWT. The live devnet PDS validates + // `aud` against its own DID, so this must be the PDS's DID — not + // the Contrail service DID used for inbound JWT verification. + serviceDid: pdsDid, + masterKey: TEST_MASTER_KEY, + plcDirectory: PLC_URL, + resolver: createDevnetResolver(), + fetch: countingFetch, + allowProvisioning: true, + }, + }); + await contrail.init(); + handle = createHandler(contrail); + callAs = createCaller(handle); + + // Alice acts as the provisioning caller — she becomes owner of the new + // community's $admin and $publishers spaces, which lets her publish. + alice = await createTestAccount(); + aliceClient = await login(alice); + }, 30_000); + + afterAll(async () => { + await cleanupSchema?.(); + }); + + it( + "provisions via the XRPC route and publishes a record (one cached session across two putRecords)", + async () => { + // Mint an invite code via the PDS admin API — same pattern as the + // orchestrator-only test above. + const inviteRes = await fetch( + `${PDS_URL}/xrpc/com.atproto.server.createInviteCode`, + { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Basic ${Buffer.from( + `admin:${PDS_ADMIN_PASSWORD}`, + ).toString("base64")}`, + }, + body: JSON.stringify({ useCount: 1 }), + }, + ); + if (!inviteRes.ok) { + throw new Error( + `createInviteCode failed (${inviteRes.status}): ${await inviteRes.text()}`, + ); + } + const { code: inviteCode } = (await inviteRes.json()) as { code: string }; + + // Keep total label under PDS's 18-char limit: `r-` (2) + 8-char suffix + // = 10 chars, well under the cap. + const suffix = `${Date.now().toString(36).slice(-6)}${Math.random() + .toString(36) + .slice(2, 4)}`; + const newHandle = `r-${suffix}${HANDLE_DOMAIN}`; + const email = `${suffix}@devnet.test`; + const password = `pw-${suffix}`; + + const callerRotation = await generateKeyPair(); + + const baselineCreateSessionCount = createSessionCount; + + // ---- POST /xrpc/${NS}.provision ----------------------------------- + const provRes = await callAs(aliceClient, "POST", `${NS}.provision`, { + body: { + handle: newHandle, + email, + password, + inviteCode, + pdsEndpoint: PDS_URL, + rotationKey: callerRotation.publicDidKey, + }, + }); + const provText = await provRes.clone().text(); + expect(provRes.status, provText).toBe(200); + const provBody = (await provRes.json()) as { + communityDid: string; + status: string; + }; + expect(provBody.status).toBe("activated"); + expect(provBody.communityDid).toMatch(/^did:plc:[a-z2-7]{24}$/); + const communityDid = provBody.communityDid; + + // The orchestrator never calls createSession during provision — it + // gets accessJwt + refreshJwt directly from the createAccount response + // and seeds the community_sessions cache before returning, so the first + // publish hits a warm cache. + const provisionCreateSessionCount = + createSessionCount - baselineCreateSessionCount; + expect(provisionCreateSessionCount).toBe(0); + + // ---- First putRecord: cache miss → createSession ------------------ + const firstRecord = { + $type: POST_NSID, + text: `routed-e2e first ${suffix}`, + createdAt: new Date().toISOString(), + }; + const put1 = await callAs(aliceClient, "POST", `${NS}.putRecord`, { + body: { + communityDid, + collection: POST_NSID, + record: firstRecord, + }, + }); + const put1Text = await put1.clone().text(); + expect(put1.status, put1Text).toBe(200); + const put1Body = (await put1.json()) as { uri: string; cid: string }; + expect(put1Body.uri).toMatch( + new RegExp(`^at://${communityDid}/${POST_NSID}/`), + ); + expect(put1Body.cid).toBeTruthy(); + + // ---- Second putRecord: cache hit → no createSession --------------- + const secondRecord = { + $type: POST_NSID, + text: `routed-e2e second ${suffix}`, + createdAt: new Date().toISOString(), + }; + const put2 = await callAs(aliceClient, "POST", `${NS}.putRecord`, { + body: { + communityDid, + collection: POST_NSID, + record: secondRecord, + }, + }); + const put2Text = await put2.clone().text(); + expect(put2.status, put2Text).toBe(200); + const put2Body = (await put2.json()) as { uri: string; cid: string }; + expect(put2Body.uri).toMatch( + new RegExp(`^at://${communityDid}/${POST_NSID}/`), + ); + + // Zero createSession across the whole flow: provision pre-warms the + // cache, then both publishes hit the 30s-skew cache. + const publishesCreateSessionCount = + createSessionCount - baselineCreateSessionCount; + expect(publishesCreateSessionCount).toBe(0); + }, + 60_000, + ); + + // TODO: cover stale-session auto-recovery (provision Task 14, ensureSession + // refresh path). Hard to exercise deterministically against live devnet + // without aging out a real `accessExp` past the 30s skew; covered by unit + // tests in packages/contrail/tests/community-publishing.test.ts. +}); diff --git a/apps/contrail-e2e/tests/reap-tombstone.test.ts b/apps/contrail-e2e/tests/reap-tombstone.test.ts new file mode 100644 index 0000000..dfd40af --- /dev/null +++ b/apps/contrail-e2e/tests/reap-tombstone.test.ts @@ -0,0 +1,178 @@ +/** + * Devnet e2e for the tombstone CID derivation used by `contrail reap` (M6). + * + * `cli/commands/reap.ts:146` calls `cidForOp(signed as never)` because the + * helper's declared signed-op union covers genesis/update — not tombstone. + * The DAG-CBOR encoder accepts the smaller tombstone shape, but no other + * test submits a *real* tombstone to live PLC and verifies the CID we + * computed locally matches the one PLC returns from `log/last`. This test + * closes that gap. + * + * The test uses managed-mode provisioning to land a real DID on devnet PLC + * with the rotation key encrypted on the persisted row, then drives the + * tombstone flow (build → sign → cidForOp → submit) with the same helpers + * runReap uses, and asserts the post-submit `log/last` matches. + * + * Tombstones are irrevocable on PLC. This test always operates on a freshly- + * provisioned devnet DID never seen by any other test or user. + * + * Prereqs: `pnpm stack:up` (devnet PDS+PLC + postgres reachable). + */ +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { randomUUID } from "node:crypto"; +import pg from "pg"; +import { + CommunityAdapter, + CredentialCipher, + ProvisionOrchestrator, + generateKeyPair, + initCommunitySchema, + pdsCreateAccount, + pdsGetRecommendedDidCredentials, + pdsActivateAccount, + pdsCreateAppPassword, + submitGenesisOp, + getLastOpCid, + buildTombstoneOp, + signTombstoneOp, + submitTombstoneOp, + cidForOp, + type PdsClient, + type PlcClient, +} from "@atmo-dev/contrail"; +import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; +import { + PDS_URL, + PLC_URL, + HANDLE_DOMAIN, + PDS_ADMIN_PASSWORD, + createIsolatedSchema, +} from "./helpers"; + +describe("reap tombstone CID matches live PLC log/last (M6)", () => { + let pool: pg.Pool; + let cleanupSchema: () => Promise<void>; + let adapter: CommunityAdapter; + let cipher: CredentialCipher; + let pdsDid: string; + + beforeAll(async () => { + const res = await fetch(`${PDS_URL}/xrpc/com.atproto.server.describeServer`); + if (!res.ok) { + throw new Error( + `devnet PDS unreachable at ${PDS_URL}: ${res.status} ${await res.text()}`, + ); + } + pdsDid = ((await res.json()) as { did: string }).did; + + const iso = await createIsolatedSchema("test_reap_tombstone_e2e"); + pool = iso.pool; + cleanupSchema = iso.cleanup; + const db = createPostgresDatabase(pool); + await initCommunitySchema(db); + adapter = new CommunityAdapter(db); + cipher = new CredentialCipher(new Uint8Array(32).fill(7)); + }, 15_000); + + afterAll(async () => { + await cleanupSchema?.(); + }); + + const pdsClient: PdsClient = { + createAccount: ({ pdsUrl, serviceAuthJwt, body }) => + pdsCreateAccount(pdsUrl, serviceAuthJwt, body), + getRecommendedDidCredentials: ({ pdsUrl, accessJwt }) => + pdsGetRecommendedDidCredentials(pdsUrl, accessJwt), + activateAccount: ({ pdsUrl, accessJwt }) => pdsActivateAccount(pdsUrl, accessJwt), + createAppPassword: ({ pdsUrl, accessJwt, name }) => + pdsCreateAppPassword(pdsUrl, accessJwt, name), + }; + + const plcClient: PlcClient = { + submit: (did, op) => submitGenesisOp(PLC_URL, did, op as any), + }; + + async function mintInvite(): Promise<string> { + const inviteRes = await fetch( + `${PDS_URL}/xrpc/com.atproto.server.createInviteCode`, + { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Basic ${Buffer.from( + `admin:${PDS_ADMIN_PASSWORD}`, + ).toString("base64")}`, + }, + body: JSON.stringify({ useCount: 1 }), + }, + ); + if (!inviteRes.ok) { + throw new Error( + `createInviteCode failed (${inviteRes.status}): ${await inviteRes.text()}`, + ); + } + return ((await inviteRes.json()) as { code: string }).code; + } + + it( + "tombstone op submitted to PLC has the CID we computed locally via cidForOp", + async () => { + // Provision a fresh community to land a real DID on devnet PLC. + const inviteCode = await mintInvite(); + const suffix = `${Date.now().toString(36)}${Math.random() + .toString(36) + .slice(2, 6)}`; + const handle = `tomb-${suffix}${HANDLE_DOMAIN}`; + const email = `${suffix}@devnet.test`; + const password = `pw-${suffix}`; + const attemptId = randomUUID(); + + const callerRotation = await generateKeyPair(); + + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: plcClient, + pds: pdsClient, + pdsDid, + }); + const result = await orch.provision({ + attemptId, + pdsEndpoint: PDS_URL, + handle, + email, + password, + inviteCode, + rotationKey: callerRotation.publicDidKey, + }); + expect(result.status).toBe("activated"); + const did = result.did; + + // Pull the encrypted rotation key off the persisted row — same path + // runReap takes. + const row = await adapter.getProvisionAttempt(attemptId); + expect(row).not.toBeNull(); + expect(row!.encryptedRotationKey).toBeTruthy(); + const rotationJwk = JSON.parse( + await cipher.decryptString(row!.encryptedRotationKey!), + ) as { kty: string; crv: string; x: string; y: string; d: string }; + + // Drive the tombstone path the same way reap.ts does, but inline so + // the test pins the cidForOp CID independent of runReap's archival + // bookkeeping. + const prev = await getLastOpCid(PLC_URL, did); + const unsigned = buildTombstoneOp(prev); + const signed = await signTombstoneOp(unsigned, rotationJwk); + const expectedCid = await cidForOp(signed as never); + + await submitTombstoneOp(PLC_URL, did, signed); + + // PLC's log/last for the DID must now report the same CID we computed. + // If cidForOp's tombstone encoding ever drifts from PLC's, this is the + // failure mode that catches it. + const lastCid = await getLastOpCid(PLC_URL, did); + expect(lastCid).toBe(expectedCid); + }, + 45_000, + ); +}); diff --git a/docs/07-communities.md b/docs/07-communities.md index 0268c42..53b3d7b 100644 --- a/docs/07-communities.md +++ b/docs/07-communities.md @@ -56,12 +56,22 @@ Stored credentials (app passwords for adopted communities, signing keys for mint When you want atproto records published under a *shared* identity — a team, a project, a channel — not a single user. Think: a group's published calendar events, a community's published posts. -## Two modes +## Three modes - **Minted** — contrail creates a fresh `did:plc` for the community, holds the signing key plus one rotation key (a second rotation key is returned to the creator once for recovery), and publishes from it. - **Adopted** — contrail takes over an existing account by holding an **app password** issued from its PDS. The owner's identity, signing key, and rotation keys are unchanged; contrail just gets PDS write access via the app password. +- **Provisioned** — contrail creates a fresh `did:plc` and a new PDS account. The caller supplies the rotation key; that key is set as the DID's rotation key in PLC. Contrail receives an app password from the new account and publishes through it. -Either way, the result is the same: a DID that multiple members can act through, gated by access levels. +Whichever the mode, the result is the same shape: a DID that multiple members can act through, gated by access levels. + +## Choosing a mode + +Two questions typically determine the mode: + +1. **Does the community already have a DID?** Yes → **adopt**. No → continue. +2. **How should records be published?** Contrail signs them directly → **mint**. Contrail uses an app password against a PDS account → **provision**. + +The rotation key holder follows from the choice: contrail in mint mode, caller in adopt and provision modes. ## Access levels @@ -84,16 +94,23 @@ The integration plugs in to two contrail extension points: ## XRPCs -- `<ns>.community.mint | adopt | list | delete` +- `<ns>.community.mint | adopt | provision | list | delete` - `<ns>.community.invite.create | redeem | revoke | list` - `<ns>.community.setAccessLevel | revoke | listMembers` - `<ns>.community.space.create | grant | revoke | ...` — community-owned spaces - `<ns>.community.putRecord | deleteRecord` — publish records as the community DID +The `contrail-community reap` CLI tombstones provisioned DIDs in PLC when provisioning fails partway and orphan rows accumulate. It ships as a bin in the `@atmo-dev/contrail-community` package (`npx contrail-community reap`); under pnpm's isolated `node_modules` the core `contrail` CLI cannot resolve the community package, so `contrail reap` only works in hoisted installs where both packages sit together. + +- Default is dry-run; pass `--no-dry-run` to actually submit (irrevocable) tombstones. +- `--all-stuck` only reaps rows idle at least `--older-than <minutes>` (default 30), so a bulk run can't tombstone an in-flight provision that is mid-state-machine. `--attempt-id <uuid>` targets a single known row and ignores the age floor. +- **D1 and Postgres:** by default reap acquires the Cloudflare D1 binding via `wrangler getPlatformProxy()`. For the decoupled external (Postgres) index, pass `--db <connection-string>` or set `DATABASE_URL` and reap runs against Postgres instead (`--db` wins over the env var). The orchestrator and adapter SQL are dialect-agnostic, so both paths share the same reap logic. + ## What's not here - No per-record per-level ACLs. Model as spaces. - No auto-rotation on key compromise yet. -- Adoption can be revoked unilaterally by the owner — they revoke the app password on their PDS and contrail loses write access. (Mint mode is the irreversible one: the creator's recovery rotation key, returned once at mint time, is the only path back if contrail's signing/rotation key is compromised.) +- Adopted and provisioned modes: contrail's write access depends on an app password issued from the PDS, which the rotation-key holder can revoke at any time. +- Minted mode: contrail holds the signing key and one rotation key. The creator's recovery rotation key, returned once at mint time, is the only key not held by contrail. The design follows zicklag's [Arbiter design sketch](https://zicklag.leaflet.pub/3mjrvb5pul224) for group management on atproto. The post is an early design note; our implementation will track it as the spec firms up. diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index ac7bc8a..b0f2c47 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -16,6 +16,9 @@ "import": "./dist/index.js" } }, + "bin": { + "contrail-community": "./dist/cli/index.js" + }, "repository": { "type": "git", "url": "https://github.com/flo-bit/contrail.git", @@ -42,9 +45,25 @@ "@atcute/xrpc-server": "^0.1.12", "@atmo-dev/contrail": "workspace:*", "@atmo-dev/contrail-base": "workspace:*", + "cac": "^7.0.0", "hono": "^4.12.8" }, + "peerDependencies": { + "pg": "^8.0.0", + "wrangler": "^4.0.0" + }, + "peerDependenciesMeta": { + "pg": { + "optional": true + }, + "wrangler": { + "optional": true + } + }, "devDependencies": { + "@types/node": "^25.5.0", + "@types/pg": "^8.20.0", + "pg": "^8.20.0", "tsup": "^8.5.0", "typescript": "^5.7.3", "vitest": "^4.1.0" diff --git a/packages/contrail-community/src/adapter.ts b/packages/contrail-community/src/adapter.ts index 586fa88..39641f0 100644 --- a/packages/contrail-community/src/adapter.ts +++ b/packages/contrail-community/src/adapter.ts @@ -6,6 +6,9 @@ import type { CommunityMode, CommunityRow, CreateCommunityInviteInput, + CreateProvisionAttemptInput, + ProvisionAttemptRow, + ProvisionStatus, } from "./types"; function toNum(v: unknown): number { @@ -51,6 +54,19 @@ export interface CreateMintedCommunityInput { createdBy: string; } +export interface CreateProvisionedCommunityInput { + did: string; + pdsEndpoint: string; + handle: string; + /** Encrypted PDS app password — already-encrypted base64 envelope. The + * orchestrator persisted this on the provision_attempts row after a + * post-activation `createAppPassword` call; the route handler hands it + * through so we keep one source of truth for the credential and avoid + * round-tripping the plaintext password through the adapter. */ + appPasswordEncrypted: string; + createdBy: string; +} + export interface GrantInput { spaceUri: string; subjectDid?: string; @@ -91,6 +107,35 @@ export class CommunityAdapter { }; } + async createFromProvisioned( + input: CreateProvisionedCommunityInput + ): Promise<CommunityRow> { + const now = Date.now(); + await this.db + .prepare( + `INSERT INTO communities (did, mode, pds_endpoint, app_password_encrypted, identifier, created_by, created_at) + VALUES (?, 'provision', ?, ?, ?, ?, ?)` + ) + .bind( + input.did, + input.pdsEndpoint, + input.appPasswordEncrypted, + input.handle, + input.createdBy, + now + ) + .run(); + return { + did: input.did, + mode: "provision", + pdsEndpoint: input.pdsEndpoint, + identifier: input.handle, + createdBy: input.createdBy, + createdAt: now, + deletedAt: null, + }; + } + async createMintedCommunity(input: CreateMintedCommunityInput): Promise<CommunityRow> { const now = Date.now(); await this.db @@ -384,6 +429,215 @@ export class CommunityAdapter { .first<any>(); return row ? mapCommunityInviteRow(row) : null; } + + // ---- Provision attempts ------------------------------------------------ + + async createProvisionAttempt(input: CreateProvisionAttemptInput): Promise<void> { + const now = Date.now(); + await this.db + .prepare( + `INSERT INTO provision_attempts ( + attempt_id, did, status, pds_endpoint, handle, email, invite_code, + encrypted_signing_key, encrypted_rotation_key, + created_at, updated_at + ) VALUES (?, ?, 'keys_generated', ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .bind( + input.attemptId, + input.did, + input.pdsEndpoint, + input.handle, + input.email, + input.inviteCode ?? null, + input.encryptedSigningKey, + input.encryptedRotationKey, + now, + now + ) + .run(); + } + + async getProvisionAttempt(attemptId: string): Promise<ProvisionAttemptRow | null> { + const row = await this.db + .prepare(`SELECT * FROM provision_attempts WHERE attempt_id = ?`) + .bind(attemptId) + .first<Record<string, any>>(); + return row ? rowToProvisionAttempt(row) : null; + } + + async updateProvisionStatus( + attemptId: string, + status: ProvisionStatus, + opts: { lastError?: string; encryptedPassword?: string } = {} + ): Promise<void> { + const now = Date.now(); + const stampCol = ({ + genesis_submitted: "genesis_submitted_at", + account_created: "account_created_at", + did_doc_updated: "did_doc_updated_at", + activated: "activated_at", + } as Record<string, string | undefined>)[status]; + + const sets: string[] = [`status = ?`, `updated_at = ?`]; + const args: any[] = [status, now]; + if (stampCol) { + sets.push(`${stampCol} = ?`); + args.push(now); + } + if (opts.lastError !== undefined) { + sets.push(`last_error = ?`); + args.push(opts.lastError); + } + if (opts.encryptedPassword !== undefined) { + sets.push(`encrypted_password = ?`); + args.push(opts.encryptedPassword); + } + args.push(attemptId); + + await this.db + .prepare(`UPDATE provision_attempts SET ${sets.join(", ")} WHERE attempt_id = ?`) + .bind(...args) + .run(); + } + + /** List provision attempts that did NOT reach `activated` AND have been + * idle for at least `olderThanMs` (no update within that window). These are + * the rows reap can act on: a non-terminal status means the flow stopped + * partway, leaving (typically) a dangling DID in PLC that needs + * tombstoning. The age floor is mandatory and exists so a bulk reap can + * never select an in-flight attempt that is mid-state-machine (seconds old) + * and tombstone a DID that was about to activate. Pass `0` to disable the + * floor (e.g. a test, or an operator who has confirmed nothing is running). */ + async listStuckAttempts(olderThanMs: number): Promise<ProvisionAttemptRow[]> { + const cutoff = Date.now() - olderThanMs; + const rows = await this.db + .prepare( + `SELECT * FROM provision_attempts + WHERE status != 'activated' AND updated_at <= ? + ORDER BY updated_at ASC` + ) + .bind(cutoff) + .all<Record<string, any>>(); + return rows.results.map(rowToProvisionAttempt); + } + + /** Move a stuck provision_attempts row into the archive table after reap + * has tombstoned its DID in PLC. Insert + delete run as a single + * `db.batch()`, which is atomic on Postgres (BEGIN/COMMIT) and D1; on the + * plain sqlite adapter it is not, so the archive INSERT is also idempotent + * (`ON CONFLICT (attempt_id) DO NOTHING`). Together that makes a + * retry-after-partial-failure safe: if a prior run's INSERT landed but its + * DELETE did not, the row is stranded in both tables; re-running archives + * cleanly (the INSERT is a no-op, the DELETE removes the live row) instead + * of dying on a PRIMARY KEY conflict. */ + async archiveStuckAttempt( + attemptId: string, + opts: { tombstoneOpCid?: string | null; notes?: string | null } = {} + ): Promise<void> { + const row = await this.db + .prepare(`SELECT * FROM provision_attempts WHERE attempt_id = ?`) + .bind(attemptId) + .first<Record<string, any>>(); + if (!row) { + throw new Error(`provision_attempt not found: ${attemptId}`); + } + const now = Date.now(); + const insert = this.db + .prepare( + `INSERT INTO provision_attempts_archive ( + attempt_id, did, pds_endpoint, handle, email, invite_code, + last_status, last_error, + archived_at, tombstone_op_cid, notes + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (attempt_id) DO NOTHING` + ) + .bind( + row.attempt_id, + row.did, + row.pds_endpoint, + row.handle, + row.email, + row.invite_code ?? null, + row.status, + row.last_error ?? null, + now, + opts.tombstoneOpCid ?? null, + opts.notes ?? null + ); + const del = this.db + .prepare(`DELETE FROM provision_attempts WHERE attempt_id = ?`) + .bind(attemptId); + await this.db.batch([insert, del]); + } + + // ---- Community sessions cache ----------------------------------------- + + async getSession(communityDid: string): Promise<{ + accessJwt: string; + refreshJwt: string; + accessExp: number; + } | null> { + const r = await this.db + .prepare( + `SELECT access_jwt, refresh_jwt, access_exp FROM community_sessions WHERE community_did = ?` + ) + .bind(communityDid) + .first<{ access_jwt: string; refresh_jwt: string; access_exp: number }>(); + if (!r) return null; + return { + accessJwt: r.access_jwt, + refreshJwt: r.refresh_jwt, + accessExp: Number(r.access_exp), + }; + } + + async upsertSession( + communityDid: string, + s: { accessJwt: string; refreshJwt: string; accessExp: number } + ): Promise<void> { + const now = Date.now(); + await this.db + .prepare( + `INSERT INTO community_sessions (community_did, access_jwt, refresh_jwt, access_exp, updated_at) + VALUES (?, ?, ?, ?, ?) + ON CONFLICT (community_did) DO UPDATE SET + access_jwt = excluded.access_jwt, + refresh_jwt = excluded.refresh_jwt, + access_exp = excluded.access_exp, + updated_at = excluded.updated_at` + ) + .bind(communityDid, s.accessJwt, s.refreshJwt, s.accessExp, now) + .run(); + } + + async clearSession(communityDid: string): Promise<void> { + await this.db + .prepare(`DELETE FROM community_sessions WHERE community_did = ?`) + .bind(communityDid) + .run(); + } +} + +function rowToProvisionAttempt(r: Record<string, any>): ProvisionAttemptRow { + return { + attemptId: r.attempt_id, + did: r.did, + status: r.status as ProvisionStatus, + pdsEndpoint: r.pds_endpoint, + handle: r.handle, + email: r.email, + inviteCode: r.invite_code ?? null, + encryptedSigningKey: r.encrypted_signing_key ?? null, + encryptedRotationKey: r.encrypted_rotation_key ?? null, + encryptedPassword: r.encrypted_password ?? null, + genesisSubmittedAt: r.genesis_submitted_at == null ? null : Number(r.genesis_submitted_at), + accountCreatedAt: r.account_created_at == null ? null : Number(r.account_created_at), + didDocUpdatedAt: r.did_doc_updated_at == null ? null : Number(r.did_doc_updated_at), + activatedAt: r.activated_at == null ? null : Number(r.activated_at), + lastError: r.last_error ?? null, + createdAt: Number(r.created_at), + updatedAt: Number(r.updated_at), + }; } function mapCommunityInviteRow(row: any): CommunityInviteRow { diff --git a/packages/contrail-community/src/cli/index.ts b/packages/contrail-community/src/cli/index.ts new file mode 100644 index 0000000..15fe32b --- /dev/null +++ b/packages/contrail-community/src/cli/index.ts @@ -0,0 +1,49 @@ +#!/usr/bin/env node +/** + * contrail-community — CLI entrypoint. + * + * Exposes `reap` (tombstone stuck provision_attempts in PLC). This lives here, + * as contrail-community's own bin, rather than only as a dynamically-imported + * subcommand of `contrail`: the PR #30 package split removed contrail's + * dependency edge into community code, so under pnpm's isolated node_modules + * `contrail` cannot resolve `@atmo-dev/contrail-community` and the dynamic + * import there silently no-ops. Shipping the bin here guarantees `reap` is + * always reachable wherever contrail-community is installed. + * + * Config loading is reconstructed from contrail's public `./cli-config` + * helpers so contrail-community needs no copy of contrail's CLI plumbing and + * no new dependency edge (it already depends on `@atmo-dev/contrail`). + */ +import { cac } from "cac"; +import { + findConfigFile, + loadConfig, + CONFIG_CANDIDATES_MESSAGE, +} from "@atmo-dev/contrail/cli-config"; +import { registerReap } from "./reap.js"; + +const cli = cac("contrail-community"); + +registerReap(cli, { + resolveAndLoadConfig: async (opts) => { + const root = opts.root ?? process.cwd(); + const path = findConfigFile(root, opts.config); + if (!path) { + console.error( + "Could not find a Contrail config. Pass --config <path> or place one at\n" + + ` ${CONFIG_CANDIDATES_MESSAGE}` + ); + process.exit(1); + } + return loadConfig(path); + }, +}); + +cli.help(); + +try { + cli.parse(); +} catch (err) { + console.error(err); + process.exit(1); +} diff --git a/packages/contrail-community/src/cli/reap.ts b/packages/contrail-community/src/cli/reap.ts new file mode 100644 index 0000000..b053f99 --- /dev/null +++ b/packages/contrail-community/src/cli/reap.ts @@ -0,0 +1,397 @@ +import type { CAC } from "cac"; +import { createInterface } from "node:readline/promises"; +import { stdin as input, stdout as output } from "node:process"; +import type { Database } from "@atmo-dev/contrail-base"; +import { CommunityAdapter } from "../adapter.js"; +import { CredentialCipher } from "../credentials.js"; +import { + buildTombstoneOp, + cidForOp, + getLastOpCid, + signTombstoneOp, + submitTombstoneOp, +} from "../plc.js"; + +interface ReapOpts { + config?: string; + root?: string; + remote?: boolean; + binding: string; + attemptId?: string; + allStuck?: boolean; + dryRun?: boolean; + yes?: boolean; + olderThan?: number; + db?: string; +} + +/** Minimal logger interface so `runReap` can be invoked from tests with + * silent stubs and from the CLI shim with `console`. */ +export interface ReapLogger { + log(...args: unknown[]): void; + error(...args: unknown[]): void; +} + +/** Default idle window for `--all-stuck`: a row must have gone untouched this + * long before bulk reap will consider tombstoning it. Provisioning is a + * fast (seconds) state machine, so 30 minutes is a wide safety margin that + * still reaps genuinely-abandoned rows. Operators can override via + * `--older-than <minutes>`. */ +export const DEFAULT_REAP_AGE_FLOOR_MS = 30 * 60 * 1000; + +/** Where reap should read its provision_attempts rows from. Postgres (the + * decoupled external index) is selected by an explicit `--db <url>` or, as a + * convenience matching the apps/postgres deployment, a `DATABASE_URL` in the + * environment. Otherwise reap uses the Cloudflare D1 binding via wrangler. + * An explicit `--db` always wins over the env var. */ +export type ReapDbSource = + | { kind: "postgres"; url: string } + | { kind: "d1" }; + +export function chooseReapDbSource(opts: { + db?: string; + databaseUrl?: string; +}): ReapDbSource { + const url = opts.db ?? opts.databaseUrl; + return url ? { kind: "postgres", url } : { kind: "d1" }; +} + +export interface RunReapOptions { + adapter: CommunityAdapter; + cipher: CredentialCipher; + plcDirectory: string; + fetch?: typeof fetch; + logger: ReapLogger; + /** Skip confirmation prompts. The CLI passes this when the user supplied + * --yes; tests always pass true since they don't have a TTY. */ + yes: boolean; + attemptId?: string; + allStuck?: boolean; + dryRun?: boolean; + /** Idle-age floor for `--all-stuck` (milliseconds). Rows updated more + * recently than this are left alone so a bulk reap can't tombstone an + * in-flight provision. Ignored for the single `--attempt-id` path, which + * is an explicit operator action on a known row. Defaults to + * DEFAULT_REAP_AGE_FLOOR_MS. */ + olderThanMs?: number; +} + +export interface RunReapResult { + ok: boolean; + /** Validation/precondition error, present when ok=false. */ + error?: string; + /** Number of rows successfully reaped (PLC tombstone submitted + archived). */ + reaped: number; + /** Number of rows skipped because of --dry-run. */ + dryRunSkipped: number; + /** Number of rows that errored out during reap (activated row, PLC error, etc.). */ + errors: number; +} + +/** Yes/no prompt that respects --yes (auto-accept) and falls back to the + * provided default in non-TTY environments — the user isn't there to answer. */ +async function promptYesNo( + question: string, + defaultYes: boolean, + autoYes: boolean +): Promise<boolean> { + if (autoYes) return true; + if (!input.isTTY) return false; + const rl = createInterface({ input, output }); + try { + const hint = defaultYes ? "[Y/n]" : "[y/N]"; + const ans = (await rl.question(`${question} ${hint} `)).trim().toLowerCase(); + if (ans === "") return defaultYes; + return ans === "y" || ans === "yes"; + } finally { + rl.close(); + } +} + +/** Core reap logic, decoupled from the CAC plumbing so tests can drive it + * without going through `getPlatformProxy()`. */ +export async function runReap(opts: RunReapOptions): Promise<RunReapResult> { + const result: RunReapResult = { + ok: true, + reaped: 0, + dryRunSkipped: 0, + errors: 0, + }; + + // Safety default: an operator who omits the flag must NOT trigger + // irrevocable PLC tombstones. Real action requires an explicit + // `dryRun: false` (CLI: `--no-dry-run`). + const dryRun = opts.dryRun ?? true; + + const hasAttemptId = !!opts.attemptId; + const hasAllStuck = !!opts.allStuck; + if (!hasAttemptId && !hasAllStuck) { + return { + ...result, + ok: false, + error: "Specify exactly one of --attempt-id <uuid> or --all-stuck.", + }; + } + if (hasAttemptId && hasAllStuck) { + return { + ...result, + ok: false, + error: + "--attempt-id and --all-stuck are mutually exclusive; pass exactly one.", + }; + } + + const olderThanMs = opts.olderThanMs ?? DEFAULT_REAP_AGE_FLOOR_MS; + const rows = hasAttemptId + ? await loadSingle(opts.adapter, opts.attemptId!) + : await opts.adapter.listStuckAttempts(olderThanMs); + + if (rows.length === 0) { + opts.logger.log("No stuck provision_attempts to reap."); + return result; + } + + for (const row of rows) { + if (row.status === "activated") { + opts.logger.error( + `Refusing to reap ${row.attemptId}: status is "activated"; reap will not tombstone live communities.` + ); + result.errors += 1; + continue; + } + + opts.logger.log( + `Reaping ${row.attemptId} (did=${row.did}, status=${row.status})` + ); + + if (!row.encryptedRotationKey) { + opts.logger.error( + ` no encrypted_rotation_key on ${row.attemptId}; cannot sign tombstone` + ); + result.errors += 1; + continue; + } + + let rotationJwk: JsonWebKey; + try { + const decoded = await opts.cipher.decryptString(row.encryptedRotationKey); + rotationJwk = JSON.parse(decoded) as JsonWebKey; + } catch (err) { + opts.logger.error( + ` failed to decrypt rotation key for ${row.attemptId}: ${err instanceof Error ? err.message : err}` + ); + result.errors += 1; + continue; + } + + let prev: string; + try { + prev = await getLastOpCid(opts.plcDirectory, row.did, { + fetch: opts.fetch, + }); + } catch (err) { + opts.logger.error( + ` failed to fetch last PLC op cid for ${row.did}: ${err instanceof Error ? err.message : err}` + ); + result.errors += 1; + continue; + } + + const unsigned = buildTombstoneOp(prev); + const signed = await signTombstoneOp(unsigned, rotationJwk); + const opCid = await cidForOp(signed); + + if (dryRun) { + opts.logger.log( + ` [dry-run] would submit tombstone (op cid=${opCid}, prev=${prev})` + ); + result.dryRunSkipped += 1; + continue; + } + + if (!opts.yes) { + const confirmed = await promptYesNo( + `submit PLC tombstone for ${row.did}?`, + false, + false + ); + if (!confirmed) { + opts.logger.log(` skipped ${row.attemptId} (not confirmed)`); + continue; + } + } + + try { + await submitTombstoneOp(opts.plcDirectory, row.did, signed, { + fetch: opts.fetch, + }); + } catch (err) { + opts.logger.error( + ` PLC tombstone submit failed for ${row.did}: ${err instanceof Error ? err.message : err}` + ); + result.errors += 1; + continue; + } + + try { + await opts.adapter.archiveStuckAttempt(row.attemptId, { + tombstoneOpCid: opCid, + }); + } catch (err) { + opts.logger.error( + ` archive failed for ${row.attemptId} after tombstone submit: ${err instanceof Error ? err.message : err}` + ); + result.errors += 1; + continue; + } + + result.reaped += 1; + } + + opts.logger.log( + `Reaped ${result.reaped} attempts (${result.dryRunSkipped} dry-run skipped, ${result.errors} errors).` + ); + return result; +} + +async function loadSingle( + adapter: CommunityAdapter, + attemptId: string +): Promise<Awaited<ReturnType<CommunityAdapter["listStuckAttempts"]>>> { + const row = await adapter.getProvisionAttempt(attemptId); + return row ? [row] : []; +} + +/** Dependency-injection shape: host CLI (contrail) loads its own config and + * passes the result, so contrail-community doesn't depend on contrail's + * cli-config infrastructure. */ +export interface ReapHostDeps { + /** Returns a contrail config; host is expected to exit(1) on its own if + * no config is found. The shape is opaque to reap. */ + resolveAndLoadConfig: (opts: { config?: string; root?: string }) => Promise<{ + community?: { + plcDirectory?: string; + masterKey: Uint8Array; + }; + }>; +} + +export function registerReap(cli: CAC, host: ReapHostDeps): void { + cli + .command( + "reap", + "Tombstone stuck provision_attempts rows in PLC and archive them" + ) + .option("--config <path>", "Path to Contrail config file") + .option("--root <path>", "Project root for auto-detection (default: CWD)") + .option("--remote", "Use production D1 bindings") + .option("--binding <name>", "D1 binding name in wrangler.jsonc", { + default: "DB", + }) + .option( + "--db <url>", + "Postgres connection string for the decoupled external index. When set (or DATABASE_URL is in the env), reap runs against Postgres instead of the D1 binding." + ) + .option("--attempt-id <uuid>", "Reap a single attempt by ID") + .option( + "--all-stuck", + "Reap every provision_attempts row that did not reach status=activated and has been idle past --older-than" + ) + .option( + "--older-than <minutes>", + "With --all-stuck, only reap rows idle at least this many minutes (guards in-flight provisions)", + { default: DEFAULT_REAP_AGE_FLOOR_MS / 60_000 } + ) + .option( + "--dry-run", + "Print what would be tombstoned without submitting to PLC (DEFAULT)" + ) + .option( + "--no-dry-run", + "Actually submit tombstones to PLC. Irrevocable. Required for real runs." + ) + .option("--yes", "Auto-confirm prompts") + .action(async (options: ReapOpts) => { + // Flag validation (exactly one of --attempt-id / --all-stuck) lives in + // runReap, the testable core; the `!result.ok` branch below surfaces its + // error and exits non-zero, so there's no second copy of it here. + const config = await host.resolveAndLoadConfig(options); + const community = config.community; + if (!community) { + console.error( + "config.community is not set; reap requires a configured community module." + ); + process.exit(1); + } + if (!community.plcDirectory) { + console.error( + "config.community.plcDirectory is required for `reap`." + ); + process.exit(1); + } + + // Run reap against the acquired db, then exit non-zero on any failure. + const reapAndExit = async (db: Database): Promise<void> => { + const result = await runReap({ + adapter: new CommunityAdapter(db), + cipher: new CredentialCipher(community.masterKey), + plcDirectory: community.plcDirectory!, + logger: console, + yes: !!options.yes, + attemptId: options.attemptId, + allStuck: options.allStuck, + dryRun: options.dryRun, + olderThanMs: + options.olderThan !== undefined + ? Number(options.olderThan) * 60_000 + : undefined, + }); + if (!result.ok) { + console.error(result.error); + process.exit(1); + } + if (result.errors > 0) { + process.exit(1); + } + }; + + const source = chooseReapDbSource({ + db: options.db, + databaseUrl: process.env.DATABASE_URL, + }); + + if (source.kind === "postgres") { + // Decoupled external index. Build a pool, run, then close it. + const pg = (await import("pg")).default; + const { createPostgresDatabase } = await import( + "@atmo-dev/contrail/postgres" as string + ); + console.log("reap: using Postgres index"); + const pool = new pg.Pool({ connectionString: source.url }); + try { + await reapAndExit(createPostgresDatabase(pool)); + } finally { + await pool.end(); + } + return; + } + + // D1 via wrangler. + const { getPlatformProxy } = await import("wrangler"); + const { env, dispose } = await getPlatformProxy(); + try { + const db = (env as Record<string, unknown>)[options.binding] as + | Database + | undefined; + if (!db) { + console.error( + `D1 binding "${options.binding}" not found in wrangler env.` + ); + process.exit(1); + } + await reapAndExit(db); + } finally { + await dispose(); + } + }); +} diff --git a/packages/contrail-community/src/index.ts b/packages/contrail-community/src/index.ts index d349351..80373aa 100644 --- a/packages/contrail-community/src/index.ts +++ b/packages/contrail-community/src/index.ts @@ -24,7 +24,19 @@ export { reconcile } from "./reconcile"; export { createCommunityInviteHandler } from "./invite-handler"; export { createCommunityWhoamiExtension } from "./whoami"; export { initCommunitySchema, buildCommunitySchema } from "./schema"; -export { resolveIdentity, createPdsSession } from "./pds"; +export { + resolveIdentity, + createPdsSession, + pdsCreateAccount, + pdsGetRecommendedDidCredentials, + pdsActivateAccount, + pdsCreateAppPassword, +} from "./pds"; +export type { + PdsCreateAccountBody, + PdsCreateAccountResult, + RecommendedDidCredentials, +} from "./pds"; export { generateKeyPair, buildGenesisOp, @@ -33,11 +45,45 @@ export { submitGenesisOp, encodeDagCbor, jwkToDidKey, + buildUpdateOp, + signUpdateOp, + cidForOp, + getLastOpCid, + buildTombstoneOp, + signTombstoneOp, + submitTombstoneOp, +} from "./plc"; +export type { + KeyPair, + GenesisOpInput, + UnsignedGenesisOp, + SignedGenesisOp, + UpdateOpInput, + UnsignedUpdateOp, + SignedUpdateOp, + UnsignedTombstoneOp, + SignedTombstoneOp, } from "./plc"; -export type { KeyPair, GenesisOpInput, UnsignedGenesisOp, SignedGenesisOp } from "./plc"; // The headline export — wire community into a contrail app via: // const community = createCommunityIntegration({ db, config }); // const app = createApp(db, config, { community }); export { createCommunityIntegration } from "./integration"; export type { CommunityIntegrationOptions } from "./integration"; + +export { ProvisionOrchestrator } from "./provision"; +export type { + PdsClient, + PlcClient, + ProvisionInput, + ProvisionResult, + ProvisionOrchestratorDeps, +} from "./provision"; + +export { registerReap, runReap } from "./cli/reap"; +export type { + ReapLogger, + ReapHostDeps, + RunReapOptions, + RunReapResult, +} from "./cli/reap"; diff --git a/packages/contrail-community/src/pds.ts b/packages/contrail-community/src/pds.ts index 2f73181..b8c63cc 100644 --- a/packages/contrail-community/src/pds.ts +++ b/packages/contrail-community/src/pds.ts @@ -8,6 +8,16 @@ import { type DidDocumentResolver, } from "@atcute/identity-resolver"; +/** Canonicalize a PDS endpoint URL so allowlist comparisons aren't bypassed by + * trailing slash, default port, scheme case, or IDN encoding differences. + * Returns the URL's `origin` — scheme + host + (non-default) port — which + * collapses every variant of a single PDS to one string. Throws if the input + * is not a parseable URL; callers in request paths should catch and respond + * with a 400. */ +export function normalizePdsEndpoint(url: string): string { + return new URL(url).origin; +} + export interface ResolvedIdentity { did: string; handle: string | null; @@ -102,6 +112,50 @@ async function resolveHandleToDid(handle: string, f: typeof fetch): Promise<stri throw new Error(`could not resolve handle ${handle}`); } +/** Decode the `exp` claim from a JWT's payload (in seconds since epoch). Used + * by the session cache to decide if a cached access token is still usable. + * Returns 0 if the claim is missing or the token is malformed — callers should + * treat 0 as "expired, refresh now". Avoids `Buffer` so it works in Workers. */ +export function decodeJwtExp(jwt: string): number { + const parts = jwt.split("."); + if (parts.length < 2) return 0; + const payload = parts[1]!; + const padded = payload.replace(/-/g, "+").replace(/_/g, "/"); + const padding = "=".repeat((4 - (padded.length % 4)) % 4); + try { + const json = atob(padded + padding); + const claims = JSON.parse(json) as { exp?: number }; + return Number(claims.exp ?? 0); + } catch { + return 0; + } +} + +/** POST com.atproto.server.refreshSession with the refresh JWT in Authorization. + * Returns null on any non-200 — callers fall back to `createPdsSession`. */ +export async function tryRefreshSession(input: { + pdsUrl: string; + refreshJwt: string; + fetch?: typeof fetch; +}): Promise<{ accessJwt: string; refreshJwt: string; accessExp: number } | null> { + const f = input.fetch ?? fetch; + const url = `${input.pdsUrl.replace(/\/$/, "")}/xrpc/com.atproto.server.refreshSession`; + const res = await f(url, { + method: "POST", + headers: { authorization: `Bearer ${input.refreshJwt}` }, + }); + if (res.status !== 200) return null; + const body = (await res.json().catch(() => null)) as + | { accessJwt?: string; refreshJwt?: string } + | null; + if (!body?.accessJwt || !body.refreshJwt) return null; + return { + accessJwt: body.accessJwt, + refreshJwt: body.refreshJwt, + accessExp: decodeJwtExp(body.accessJwt), + }; +} + /** Create an atproto session on the given PDS using identifier + app password. * Returns the access/refresh JWTs and the session's DID. */ export async function createPdsSession( @@ -135,3 +189,156 @@ export async function createPdsSession( did: body.did, }; } + +export interface PdsDescribeServerResult { + did: string; + /** Other fields (availableUserDomains, contact, links, inviteCodeRequired) + * are returned by the PDS but unused by Contrail's provisioning flow. */ + [key: string]: unknown; +} + +/** Calls `com.atproto.server.describeServer` on the target PDS to discover + * the DID it publishes for itself. Used as `aud` in the service-auth JWT for + * `createAccount`; the PDS verifies the audience matches its own DID and + * rejects with `BadJwtAudience` otherwise. Resolving dynamically (instead of + * hardcoding to a config value) is what allows a single Contrail instance + * to mint communities on multiple PDSes. */ +export async function pdsDescribeServer( + pdsEndpoint: string, + opts: { fetch?: typeof fetch } = {} +): Promise<PdsDescribeServerResult> { + const f = opts.fetch ?? fetch; + const url = `${pdsEndpoint.replace(/\/$/, "")}/xrpc/com.atproto.server.describeServer`; + const res = await f(url); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`describeServer failed (${res.status}): ${text}`); + } + const body = (await res.json()) as PdsDescribeServerResult; + if (!body?.did || typeof body.did !== "string") { + throw new Error("describeServer response missing required `did` field"); + } + return body; +} + +export interface PdsCreateAccountBody { + handle: string; + did: string; + email: string; + password: string; + inviteCode?: string; +} + +export interface PdsCreateAccountResult { + did: string; + handle: string; + accessJwt: string; + refreshJwt: string; +} + +/** Calls `com.atproto.server.createAccount` on the target PDS using a + * service-auth JWT (signed by the iss DID's verificationMethod). The PDS + * verifies `requester === did` against the published DID-doc, validates the + * invite, and creates the account in deactivated state. */ +export async function pdsCreateAccount( + pdsEndpoint: string, + serviceAuthJwt: string, + body: PdsCreateAccountBody, + opts: { fetch?: typeof fetch } = {} +): Promise<PdsCreateAccountResult> { + const f = opts.fetch ?? fetch; + const url = `${pdsEndpoint.replace(/\/$/, "")}/xrpc/com.atproto.server.createAccount`; + const res = await f(url, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${serviceAuthJwt}`, + }, + body: JSON.stringify(body), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`createAccount failed (${res.status}): ${text}`); + } + return (await res.json()) as PdsCreateAccountResult; +} + +export interface RecommendedDidCredentials { + rotationKeys: string[]; + verificationMethods: { atproto: string }; + alsoKnownAs: string[]; + services: Record<string, { type: string; endpoint: string }>; +} + +/** Calls `com.atproto.identity.getRecommendedDidCredentials` on the target PDS + * using the session's accessJwt (returned by `pdsCreateAccount`, NOT a + * service-auth JWT). Returns the DID-doc fields the PDS would self-publish. */ +export async function pdsGetRecommendedDidCredentials( + pdsEndpoint: string, + accessJwt: string, + opts: { fetch?: typeof fetch } = {} +): Promise<RecommendedDidCredentials> { + const f = opts.fetch ?? fetch; + const url = `${pdsEndpoint.replace(/\/$/, "")}/xrpc/com.atproto.identity.getRecommendedDidCredentials`; + const res = await f(url, { + headers: { authorization: `Bearer ${accessJwt}` }, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`getRecommendedDidCredentials failed (${res.status}): ${text}`); + } + return (await res.json()) as RecommendedDidCredentials; +} + +/** Calls `com.atproto.server.activateAccount` on the target PDS using the + * session's accessJwt (returned by `pdsCreateAccount`, NOT a service-auth + * JWT). Resolves on success; throws otherwise. */ +export async function pdsActivateAccount( + pdsEndpoint: string, + accessJwt: string, + opts: { fetch?: typeof fetch } = {} +): Promise<void> { + const f = opts.fetch ?? fetch; + const url = `${pdsEndpoint.replace(/\/$/, "")}/xrpc/com.atproto.server.activateAccount`; + const res = await f(url, { + method: "POST", + headers: { authorization: `Bearer ${accessJwt}` }, + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`activateAccount failed (${res.status}): ${text}`); + } +} + +export interface PdsCreateAppPasswordResult { + name: string; + password: string; + createdAt: string; +} + +/** Calls `com.atproto.server.createAppPassword` on the target PDS using the + * session's accessJwt. Returns the freshly minted app password. We always + * send `privileged: false` so the credential can be revoked without + * affecting the root account. */ +export async function pdsCreateAppPassword( + pdsEndpoint: string, + accessJwt: string, + name: string, + opts: { fetch?: typeof fetch } = {} +): Promise<PdsCreateAppPasswordResult> { + const f = opts.fetch ?? fetch; + const url = `${pdsEndpoint.replace(/\/$/, "")}/xrpc/com.atproto.server.createAppPassword`; + const res = await f(url, { + method: "POST", + headers: { + "content-type": "application/json", + authorization: `Bearer ${accessJwt}`, + }, + body: JSON.stringify({ name, privileged: false }), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`createAppPassword failed (${res.status}): ${text}`); + } + return (await res.json()) as PdsCreateAppPasswordResult; +} diff --git a/packages/contrail-community/src/plc.ts b/packages/contrail-community/src/plc.ts index b4d1c5b..a66a883 100644 --- a/packages/contrail-community/src/plc.ts +++ b/packages/contrail-community/src/plc.ts @@ -54,7 +54,7 @@ const P256_N = BigInt( ); const P256_N_HALF = P256_N >> 1n; -async function signBytes(privateJwk: JsonWebKey, bytes: Uint8Array): Promise<Uint8Array> { +export async function signBytes(privateJwk: JsonWebKey, bytes: Uint8Array): Promise<Uint8Array> { const key = await crypto.subtle.importKey( "jwk", privateJwk, @@ -155,6 +155,154 @@ export async function computeDidPlc(signedOp: SignedGenesisOp): Promise<string> return "did:plc:" + base32Lower(hash).slice(0, 24); } +// ============================================================================ +// Update op construction (subsequent ops chain via `prev`) +// ============================================================================ + +export interface UpdateOpInput { + prev: string; // CID string of the previous op in the chain + rotationKeys: string[]; + verificationMethodAtproto: string; + alsoKnownAs: string[]; + services: Record<string, { type: string; endpoint: string }>; +} + +export interface UnsignedUpdateOp { + type: "plc_operation"; + prev: string; + rotationKeys: string[]; + verificationMethods: { atproto: string }; + alsoKnownAs: string[]; + services: Record<string, { type: string; endpoint: string }>; +} + +export interface SignedUpdateOp extends UnsignedUpdateOp { + sig: string; // base64url, unpadded +} + +export function buildUpdateOp(input: UpdateOpInput): UnsignedUpdateOp { + return { + type: "plc_operation", + prev: input.prev, + rotationKeys: input.rotationKeys, + verificationMethods: { atproto: input.verificationMethodAtproto }, + alsoKnownAs: input.alsoKnownAs, + services: input.services, + }; +} + +/** Sign an update op with a rotation key's private JWK. */ +export async function signUpdateOp( + unsigned: UnsignedUpdateOp, + signerPrivateJwk: JsonWebKey +): Promise<SignedUpdateOp> { + const encoded = encodeDagCbor(unsigned); + const sigBytes = await signBytes(signerPrivateJwk, encoded); + return { ...unsigned, sig: bytesToB64url(sigBytes) }; +} + +/** Compute the CIDv1 for a signed op (genesis, update, or tombstone). + * CIDv1 (0x01) + dag-cbor codec (0x71) + sha2-256 (0x12 0x20) + hash, + * base32-lower with multibase "b" prefix. + * + * The tombstone shape ({type, prev, sig}) is a strict subset of update — + * the DAG-CBOR encoder accepts all three uniformly, and PLC computes its + * stored CID from the same canonical encoding. */ +export async function cidForOp( + signedOp: SignedGenesisOp | SignedUpdateOp | SignedTombstoneOp +): Promise<string> { + const encoded = encodeDagCbor(signedOp); + const hash = new Uint8Array( + await crypto.subtle.digest("SHA-256", encoded as BufferSource) + ); + const cidBytes = new Uint8Array(4 + hash.length); + cidBytes[0] = 0x01; + cidBytes[1] = 0x71; + cidBytes[2] = 0x12; + cidBytes[3] = 0x20; + cidBytes.set(hash, 4); + return "b" + base32Lower(cidBytes); +} + +/** Fetch the CID of the most recent op in a DID's PLC log. Used during + * provision recovery to obtain the genesis op's CID at resume time (we can't + * recompute it locally because ECDSA signatures are randomized) and by the + * reap CLI to chain a tombstone onto the latest op. + * + * PLC's `/log/last` endpoint returns the bare signed op object — no envelope, + * no `cid` field. We compute the CID locally with the same DAG-CBOR encoder + * cidForOp uses; PLC computes its stored CID identically, so the result + * matches the entry's CID in `/log/audit`. */ +export async function getLastOpCid( + plcDirectory: string, + did: string, + opts: { fetch?: typeof fetch } = {} +): Promise<string> { + const f = opts.fetch ?? fetch; + const url = `${plcDirectory.replace(/\/$/, "")}/${did}/log/last`; + const res = await f(url); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`PLC log/last failed (${res.status}): ${text}`); + } + const op = (await res.json()) as + | SignedGenesisOp + | SignedUpdateOp + | SignedTombstoneOp; + return cidForOp(op); +} + +// ============================================================================ +// Tombstone op construction +// A tombstone op marks a DID's PLC log as terminated — no further ops will be +// accepted. Used by `contrail reap` to clean up DIDs whose PDS account is +// permanently unrecoverable. +// ============================================================================ + +export interface UnsignedTombstoneOp { + type: "plc_tombstone"; + prev: string; +} + +export interface SignedTombstoneOp extends UnsignedTombstoneOp { + sig: string; // base64url, unpadded +} + +export function buildTombstoneOp(prev: string): UnsignedTombstoneOp { + return { type: "plc_tombstone", prev }; +} + +/** Sign a tombstone op with a rotation key's private JWK. */ +export async function signTombstoneOp( + op: UnsignedTombstoneOp, + signerPrivateJwk: JsonWebKey +): Promise<SignedTombstoneOp> { + const encoded = encodeDagCbor(op); + const sigBytes = await signBytes(signerPrivateJwk, encoded); + return { ...op, sig: bytesToB64url(sigBytes) }; +} + +/** Submit a signed tombstone op to the PLC directory. PLC accepts genesis, + * update, and tombstone ops at the same `${plcDirectory}/${did}` endpoint. */ +export async function submitTombstoneOp( + plcDirectory: string, + did: string, + signedOp: SignedTombstoneOp, + opts: { fetch?: typeof fetch } = {} +): Promise<void> { + const f = opts.fetch ?? fetch; + const url = `${plcDirectory.replace(/\/$/, "")}/${did}`; + const res = await f(url, { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(signedOp), + }); + if (!res.ok) { + const text = await res.text().catch(() => ""); + throw new Error(`PLC tombstone submit failed (${res.status}): ${text}`); + } +} + /** Submit a signed genesis op to the PLC directory. */ export async function submitGenesisOp( plcDirectory: string, @@ -334,7 +482,7 @@ function base32Lower(bytes: Uint8Array): string { return out; } -function bytesToB64url(bytes: Uint8Array): string { +export function bytesToB64url(bytes: Uint8Array): string { let bin = ""; for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); diff --git a/packages/contrail-community/src/provision.ts b/packages/contrail-community/src/provision.ts new file mode 100644 index 0000000..b5ee50e --- /dev/null +++ b/packages/contrail-community/src/provision.ts @@ -0,0 +1,403 @@ +/** Provision orchestrator: runs the 5-RPC flow (genesis → createAccount → + * recommendedCreds → PLC update → activate), persisting status after each + * step so a stuck attempt is recognizable to the reap CLI. + * + * Steps and persisted statuses: + * Step 0 generate keys + persist row → keys_generated + * Step 1 PLC genesis op → genesis_submitted + * Step 2 PDS createAccount (service-auth JWT) → account_created + * Step 3 fetch recommended DID credentials (no status change) + * Step 4 PLC update op merging recommended credentials → did_doc_updated + * Step 5 PDS activateAccount → activated + */ + +import { + generateKeyPair, + buildGenesisOp, + signGenesisOp, + computeDidPlc, + buildUpdateOp, + signUpdateOp, + cidForOp, +} from "./plc"; +import type { RecommendedDidCredentials } from "./pds"; +import { decodeJwtExp } from "./pds"; +import { mintServiceAuthJwt } from "./service-auth"; +import type { CommunityAdapter } from "./adapter"; +import type { CredentialCipher } from "./credentials"; + +export interface PlcClient { + submit(did: string, op: any): Promise<unknown>; +} + +export interface PdsClient { + createAccount(input: { + pdsUrl: string; + serviceAuthJwt: string; + body: { + handle: string; + did: string; + email: string; + password: string; + inviteCode?: string; + }; + }): Promise<{ + did: string; + handle: string; + accessJwt: string; + refreshJwt: string; + }>; + getRecommendedDidCredentials(input: { + pdsUrl: string; + accessJwt: string; + }): Promise<RecommendedDidCredentials>; + activateAccount(input: { pdsUrl: string; accessJwt: string }): Promise<void>; + /** Mints a revocable app password on the freshly-activated account. Used by + * the self-sovereign custody mode so Contrail keeps publishing authority + * without holding the account's root password. */ + createAppPassword(input: { + pdsUrl: string; + accessJwt: string; + name: string; + }): Promise<{ password: string }>; + /** Used only by the C3 retry path: when a provision call failed at + * createAppPassword, retrying with the same attemptId needs a fresh + * accessJwt. The session-cache JWT from the failed attempt may have + * expired by the time the caller retries. Optional — non-retry callers + * never invoke this. */ + createSession?(input: { + pdsUrl: string; + identifier: string; + password: string; + }): Promise<{ accessJwt: string; refreshJwt: string; did: string }>; +} + +export interface ProvisionOrchestratorDeps { + adapter: CommunityAdapter; + cipher: CredentialCipher; + plc: PlcClient; + pds: PdsClient; + /** DID of the target PDS; used as `aud` in the service-auth JWT. */ + pdsDid: string; +} + +export interface ProvisionInput { + attemptId: string; + pdsEndpoint: string; + handle: string; + email: string; + password: string; + inviteCode?: string; + /** Caller-supplied rotation public key (did:key:z…). Sits at rotationKeys[0] + * in the genesis op; Contrail's generated key is the subordinate at [1]. + * After activation Contrail mints a revocable app password (via + * createAppPassword) for ongoing publishing — the user's account password + * is never persisted. */ + rotationKey: string; +} + +export interface ProvisionResult { + attemptId: string; + did: string; + status: "activated"; + /** The caller is expected to store these — Contrail does NOT retain the + * user's root password once the app password has been minted. */ + rootCredentials: { + handle: string; + password: string; + recoveryHint: string; + }; +} + +export class ProvisionOrchestrator { + constructor(private deps: ProvisionOrchestratorDeps) {} + + async provision(input: ProvisionInput): Promise<ProvisionResult> { + const { adapter, cipher, plc, pds, pdsDid } = this.deps; + + if (!isDidKeyZ(input.rotationKey)) { + throw new Error( + `rotationKey must be a did:key:z… string (got: ${input.rotationKey.slice(0, 24)}…)` + ); + } + + // Idempotent retry: a caller that gets a 5xx with an attemptId can + // re-invoke provision with the SAME attemptId and the orchestrator picks + // up where it left off. Two recoverable shapes: + // 1. status='activated', encryptedPassword present — the orchestrator + // itself completed successfully but a downstream graduation step + // (e.g. the route's createFromProvisioned, bootstrap of reserved + // spaces) failed. We return success without re-running any PLC/PDS + // work so the caller — or the route — can resume the post-orch path. + // 2. status='activated', no encryptedPassword — activation succeeded + // but the post-activation createAppPassword failed. retryAppPasswordOnly + // re-runs only that final step. + // Other partial states are not resumable through this entry point. + const existing = await adapter.getProvisionAttempt(input.attemptId); + if (existing) { + if (existing.status === "activated" && existing.encryptedPassword) { + return { + attemptId: input.attemptId, + did: existing.did, + status: "activated", + rootCredentials: { + handle: input.handle, + password: input.password, + recoveryHint: "store this — Contrail does not retain it", + }, + }; + } + if (existing.status === "activated" && !existing.encryptedPassword) { + return this.retryAppPasswordOnly(input, existing); + } + throw new Error( + `provision attempt ${input.attemptId} already exists at status="${existing.status}"; ` + + `retry is only supported for attempts that failed at createAppPassword` + ); + } + + // Step 0: keys + persist. Contrail generates a SUBORDINATE rotation key + // (rotationKeys[1]) so it retains a path to submit subsequent PLC ops; + // the caller's key sits at rotationKeys[0]. + const signingKey = await generateKeyPair(); + const contrailRotation = await generateKeyPair(); + const encryptedSigning = await cipher.encrypt( + JSON.stringify(signingKey.privateJwk) + ); + const encryptedRotation = await cipher.encrypt( + JSON.stringify(contrailRotation.privateJwk) + ); + + const unsigned = buildGenesisOp({ + rotationKeys: [input.rotationKey, contrailRotation.publicDidKey], + verificationMethodAtproto: signingKey.publicDidKey, + alsoKnownAs: [`at://${input.handle}`], + services: { + atproto_pds: { + type: "AtprotoPersonalDataServer", + endpoint: input.pdsEndpoint, + }, + }, + }); + // Genesis is signed with Contrail's subordinate rotation key — it's listed + // at rotationKeys[1] so PLC accepts the signature. + const signedGenesis = await signGenesisOp(unsigned, contrailRotation.privateJwk); + const did = await computeDidPlc(signedGenesis); + + await adapter.createProvisionAttempt({ + attemptId: input.attemptId, + did, + pdsEndpoint: input.pdsEndpoint, + handle: input.handle, + email: input.email, + inviteCode: input.inviteCode ?? null, + encryptedSigningKey: encryptedSigning, + encryptedRotationKey: encryptedRotation, + }); + + // Step 1: PLC genesis + try { + await plc.submit(did, signedGenesis); + await adapter.updateProvisionStatus(input.attemptId, "genesis_submitted"); + } catch (err: any) { + await adapter.updateProvisionStatus(input.attemptId, "keys_generated", { + lastError: `plc-genesis: ${err.message}`, + }); + throw err; + } + + // Step 2: createAccount + let session: { + did: string; + handle: string; + accessJwt: string; + refreshJwt: string; + }; + try { + const serviceAuthJwt = await mintServiceAuthJwt({ + privateJwk: signingKey.privateJwk, + iss: did, + aud: pdsDid, + lxm: "com.atproto.server.createAccount", + ttlSec: 60, + }); + session = await pds.createAccount({ + pdsUrl: input.pdsEndpoint, + serviceAuthJwt, + body: { + handle: input.handle, + did, + email: input.email, + password: input.password, + inviteCode: input.inviteCode, + }, + }); + await adapter.updateProvisionStatus(input.attemptId, "account_created"); + } catch (err: any) { + await adapter.updateProvisionStatus(input.attemptId, "genesis_submitted", { + lastError: `createAccount: ${err.message}`, + }); + throw err; + } + + // Step 3 + 4: getRecommendedDidCredentials + PLC update op + try { + const recommended = await pds.getRecommendedDidCredentials({ + pdsUrl: input.pdsEndpoint, + accessJwt: session.accessJwt, + }); + const baseRotationKeys = [input.rotationKey, contrailRotation.publicDidKey]; + const updatedRotationKeys = [ + ...baseRotationKeys, + ...recommended.rotationKeys.filter((k) => !baseRotationKeys.includes(k)), + ]; + const unsignedUpdate = buildUpdateOp({ + prev: await cidForOp(signedGenesis), + rotationKeys: updatedRotationKeys, + verificationMethodAtproto: recommended.verificationMethods.atproto, + alsoKnownAs: recommended.alsoKnownAs, + services: recommended.services, + }); + const signedUpdate = await signUpdateOp( + unsignedUpdate, + contrailRotation.privateJwk + ); + await plc.submit(did, signedUpdate); + await adapter.updateProvisionStatus(input.attemptId, "did_doc_updated"); + } catch (err: any) { + await adapter.updateProvisionStatus(input.attemptId, "account_created", { + lastError: `did-doc-update: ${err.message}`, + }); + throw err; + } + + // Step 5: activateAccount + try { + await pds.activateAccount({ + pdsUrl: input.pdsEndpoint, + accessJwt: session.accessJwt, + }); + await adapter.updateProvisionStatus(input.attemptId, "activated"); + } catch (err: any) { + await adapter.updateProvisionStatus(input.attemptId, "did_doc_updated", { + lastError: `activateAccount: ${err.message}`, + }); + throw err; + } + + // Seed the session cache with the JWTs createAccount returned, so the + // first publish doesn't waste a createSession round-trip. ensureSession + // refreshes or falls back to the stored password as the JWTs age out. + await adapter.upsertSession(did, { + accessJwt: session.accessJwt, + refreshJwt: session.refreshJwt, + accessExp: decodeJwtExp(session.accessJwt), + }); + + // Mint a revocable app password so we can publish without holding the + // user's root password. Failure here leaves the row at status=activated + // (the account IS activated upstream) with a last_error breadcrumb; the + // caller can retry with the same attemptId and it will pick up at + // createAppPassword only (see retryAppPasswordOnly). + try { + const minted = await pds.createAppPassword({ + pdsUrl: input.pdsEndpoint, + accessJwt: session.accessJwt, + name: `contrail-${input.attemptId}`, + }); + const encryptedPassword = await cipher.encrypt(minted.password); + await adapter.updateProvisionStatus(input.attemptId, "activated", { + encryptedPassword, + }); + } catch (err: any) { + await adapter.updateProvisionStatus(input.attemptId, "activated", { + lastError: `createAppPassword: ${err.message}`, + }); + // Wrap the error so the route handler / log readers see immediately + // that this is the recoverable retry case, not a generic PDS failure. + throw new Error(`createAppPassword: ${err.message}`); + } + + return { + attemptId: input.attemptId, + did, + status: "activated", + rootCredentials: { + handle: input.handle, + password: input.password, + recoveryHint: "store this — Contrail does not retain it", + }, + }; + } + + /** C3 retry path. Triggered when provision() is called with an attemptId + * whose row is at status='activated' + no encrypted_password. The DID is + * already on PLC; the PDS account is already created and activated; only + * the post-activation app-password mint failed. We need a fresh accessJwt + * (the cached one may have expired by the time the caller retries), then + * re-run createAppPassword. */ + private async retryAppPasswordOnly( + input: ProvisionInput, + existing: NonNullable<Awaited<ReturnType<CommunityAdapter["getProvisionAttempt"]>>> + ): Promise<ProvisionResult> { + const { adapter, cipher, pds } = this.deps; + if (!pds.createSession) { + throw new Error( + "retry path requires PdsClient.createSession to be wired (production buildOrchestrator does this; tests must stub it)" + ); + } + + const session = await pds.createSession({ + pdsUrl: existing.pdsEndpoint, + identifier: input.handle, + password: input.password, + }); + + let mintedPassword: string; + try { + const minted = await pds.createAppPassword({ + pdsUrl: existing.pdsEndpoint, + accessJwt: session.accessJwt, + name: `contrail-${input.attemptId}`, + }); + mintedPassword = minted.password; + } catch (err: any) { + await adapter.updateProvisionStatus(input.attemptId, "activated", { + lastError: `createAppPassword (retry): ${err.message}`, + }); + throw new Error(`createAppPassword (retry): ${err.message}`); + } + + const encryptedPassword = await cipher.encrypt(mintedPassword); + await adapter.updateProvisionStatus(input.attemptId, "activated", { + encryptedPassword, + }); + // Refresh the session cache with the JWTs we just obtained, so the + // first publish for this community doesn't need another createSession. + await adapter.upsertSession(existing.did, { + accessJwt: session.accessJwt, + refreshJwt: session.refreshJwt, + accessExp: decodeJwtExp(session.accessJwt), + }); + + return { + attemptId: input.attemptId, + did: existing.did, + status: "activated", + rootCredentials: { + handle: input.handle, + password: input.password, + recoveryHint: "store this — Contrail does not retain it", + }, + }; + } + +} + +/** Cheap structural check for did:key:z multibase identifiers. The orchestrator + * trusts the caller's submitted rotation key beyond this; PLC will reject any + * malformed key when the genesis op is submitted. */ +function isDidKeyZ(s: string): boolean { + return typeof s === "string" && s.startsWith("did:key:z") && s.length > 12; +} + diff --git a/packages/contrail-community/src/router.ts b/packages/contrail-community/src/router.ts index 23c3123..eb2e4ba 100644 --- a/packages/contrail-community/src/router.ts +++ b/packages/contrail-community/src/router.ts @@ -8,7 +8,13 @@ import type { import { buildSpaceUri, HostedAdapter } from "@atmo-dev/contrail"; import { CommunityAdapter } from "./adapter"; import { CredentialCipher } from "./credentials"; -import { resolveIdentity, createPdsSession } from "./pds"; +import { + resolveIdentity, + createPdsSession, + decodeJwtExp, + tryRefreshSession, + normalizePdsEndpoint, +} from "./pds"; import { generateKeyPair, buildGenesisOp, @@ -16,6 +22,18 @@ import { computeDidPlc, submitGenesisOp, } from "./plc"; +import { + pdsCreateAccount, + pdsGetRecommendedDidCredentials, + pdsActivateAccount, + pdsCreateAppPassword, + pdsDescribeServer, +} from "./pds"; +import { + ProvisionOrchestrator, + type PdsClient, + type PlcClient, +} from "./provision"; import { resolveEffectiveLevel, resolveReachableSpaces, wouldCycle } from "./acl"; import { reconcile } from "./reconcile"; import type { AccessLevel } from "./types"; @@ -212,6 +230,211 @@ export function registerCommunityRoutes( }); }); + app.post(`/xrpc/${NS}.provision`, auth, async (c) => { + // Default-deny gate. Every successful call burns an invite code on the + // target PDS and adds a permanent entry to PLC, so the route refuses + // unless the operator has explicitly opted in via cfg.allowProvisioning. + // Checked BEFORE auth-issued state inspection so an unauthorized op + // doesn't even surface that the route exists in a usable form. + if (!cfg.allowProvisioning) { + return c.json( + { + error: "ProvisioningDisabled", + message: "community.provision is disabled on this Contrail deployment", + }, + 403 + ); + } + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { + attemptId?: string; + handle?: string; + email?: string; + password?: string; + inviteCode?: string; + pdsEndpoint?: string; + rotationKey?: string; + } + | null; + if ( + !body?.handle || + !body.email || + !body.password || + !body.pdsEndpoint || + !body.rotationKey + ) { + return c.json( + { + error: "InvalidRequest", + message: "handle, email, password, pdsEndpoint, rotationKey required", + }, + 400 + ); + } + if ( + !(typeof body.rotationKey === "string" && body.rotationKey.startsWith("did:key:z")) + ) { + return c.json( + { + error: "InvalidRequest", + message: "rotationKey must be a did:key:z…", + }, + 400 + ); + } + + let normalizedPdsEndpoint: string; + try { + normalizedPdsEndpoint = normalizePdsEndpoint(body.pdsEndpoint); + } catch { + return c.json( + { + error: "InvalidRequest", + message: "pdsEndpoint must be a parseable URL", + }, + 400 + ); + } + + const allowed = cfg.allowedProvisionPdsEndpoints; + if (allowed && allowed.length > 0) { + const allowedNormalized = allowed.map((e) => { + try { + return normalizePdsEndpoint(e); + } catch { + // An unparseable allowlist entry can never match; treat as the + // original string so an obvious config typo at least produces a + // reject for the caller rather than a server crash. + return e; + } + }); + if (!allowedNormalized.includes(normalizedPdsEndpoint)) { + return c.json( + { + error: "InvalidRequest", + message: `pdsEndpoint not in allowlist`, + }, + 400 + ); + } + } else if (!cfg.allowAnyProvisionPdsEndpoint) { + // Fail closed: provisioning is enabled but no allowlist is configured. + // An empty/undefined allowlist must NOT mean "sign a genesis op for any + // caller-supplied PDS" — that's the exact attack the allowlist prevents. + // Refuse unless the operator has explicitly opted into the dangerous + // accept-any mode via allowAnyProvisionPdsEndpoint. + // + // SSRF note: allowAnyProvisionPdsEndpoint also bypasses any endpoint + // guard — the caller-supplied pdsEndpoint flows straight to + // describeServer + the createAccount fetches below with no private-IP / + // link-local / cloud-metadata protection. Only enable it behind trusted + // auth, and pair it with an egress network policy (block 169.254.0.0/16) + // + IMDSv2 on the host. The allowlist path above contains this by + // construction. + return c.json( + { + error: "ProvisioningMisconfigured", + message: + "community.provision requires a non-empty allowedProvisionPdsEndpoints allowlist; " + + "set the allowlist, or set allowAnyProvisionPdsEndpoint:true to deliberately accept any PDS", + }, + 403 + ); + } + body.pdsEndpoint = normalizedPdsEndpoint; + + // Resolve the target PDS's DID dynamically. The service-auth JWT's `aud` + // must match what the PDS publishes for itself via describeServer; the + // PDS rejects with BadJwtAudience otherwise. This is what allows a + // single Contrail to mint communities on multiple PDSes — using a + // cfg-pinned value would force a 1:1 Contrail-to-PDS deployment. + let pdsDid: string; + try { + const described = await pdsDescribeServer(body.pdsEndpoint, { + fetch: cfg.fetch, + }); + pdsDid = described.did; + } catch (err: any) { + return c.json( + { + error: "PdsUnreachable", + message: `describeServer failed for ${body.pdsEndpoint}: ${err.message}`, + }, + 502 + ); + } + const orchestrator = buildOrchestrator(cfg, community, cipher, pdsDid); + + const attemptId = body.attemptId ?? crypto.randomUUID(); + let result; + try { + result = await orchestrator.provision({ + attemptId, + pdsEndpoint: body.pdsEndpoint, + handle: body.handle, + email: body.email, + password: body.password, + inviteCode: body.inviteCode, + rotationKey: body.rotationKey, + }); + } catch (err: any) { + // attemptId must always come back to the caller so they can retry + // idempotently (see the C3 retry path in ProvisionOrchestrator). + return c.json( + { error: "ProvisioningFailed", message: err.message, attemptId }, + 502 + ); + } + + // Idempotent graduation: if the community row already exists, the first + // call already wrote both the row and the reserved spaces. A retry with + // the same attemptId should return success without double-creating. + const alreadyGraduated = (await community.getCommunity(result.did)) != null; + if (!alreadyGraduated) { + // Hand the already-encrypted password from the provision_attempts row + // to the communities row, keeping a single source of truth for the + // credential. + const attempt = await community.getProvisionAttempt(attemptId); + if (!attempt?.encryptedPassword) { + return c.json( + { + error: "ProvisioningFailed", + message: "provision attempt missing encryptedPassword after activation", + }, + 502 + ); + } + + await community.createFromProvisioned({ + did: result.did, + pdsEndpoint: body.pdsEndpoint, + handle: body.handle, + appPasswordEncrypted: attempt.encryptedPassword, + createdBy: sa.issuer, + }); + + await bootstrapReservedSpaces({ + communityDid: result.did, + creatorDid: sa.issuer, + spaces, + community, + type: spaceType, + serviceDid: spaceServiceDid, + }); + } + + const responseBody: { + communityDid: string; + status: string; + rootCredentials?: { handle: string; password: string; recoveryHint: string }; + } = { communityDid: result.did, status: result.status }; + if (result.rootCredentials) { + responseBody.rootCredentials = result.rootCredentials; + } + return c.json(responseBody); + }); + app.post(`/xrpc/${NS}.delete`, auth, async (c) => { const sa = getAuth(c); const body = (await c.req.json().catch(() => null)) as @@ -672,6 +895,8 @@ export function registerCommunityRoutes( 400 ); } + // adopt + provision modes both share the credential-proxy publishing path: + // both store {pds_endpoint, identifier, app_password_encrypted}. Falls through. // Caller must be member+ in $publishers. const publishersUri = buildSpaceUri({ @@ -695,7 +920,12 @@ export function registerCommunityRoutes( let session; try { const appPassword = await cipher.decryptString(raw.appPasswordEncrypted); - session = await createPdsSession(raw.pdsEndpoint, raw.identifier, appPassword, { + session = await ensureSession({ + community, + did: body.communityDid, + pdsEndpoint: raw.pdsEndpoint, + identifier: raw.identifier, + password: appPassword, fetch: cfg.fetch, }); } catch (err: any) { @@ -724,6 +954,11 @@ export function registerCommunityRoutes( }), } ); + if (res.status === 401) { + // Stale or revoked session: drop the cache so the next request goes + // cold through ensureSession. + await community.clearSession(body.communityDid); + } if (!res.ok) { const text = await res.text().catch(() => ""); return c.json( @@ -751,6 +986,7 @@ export function registerCommunityRoutes( if (row.mode === "mint") { return c.json({ error: "NotSupported" }, 400); } + // adopt + provision: same credential-proxy path; falls through. const publishersUri = buildSpaceUri({ ownerDid: body.communityDid, @@ -769,7 +1005,12 @@ export function registerCommunityRoutes( let session; try { const appPassword = await cipher.decryptString(raw.appPasswordEncrypted); - session = await createPdsSession(raw.pdsEndpoint, raw.identifier, appPassword, { + session = await ensureSession({ + community, + did: body.communityDid, + pdsEndpoint: raw.pdsEndpoint, + identifier: raw.identifier, + password: appPassword, fetch: cfg.fetch, }); } catch (err: any) { @@ -795,6 +1036,9 @@ export function registerCommunityRoutes( }), } ); + if (res.status === 401) { + await community.clearSession(body.communityDid); + } if (!res.ok) { const text = await res.text().catch(() => ""); return c.json( @@ -906,14 +1150,20 @@ export function registerCommunityRoutes( } } - // Adopted: attempt a session creation. + // Adopted + provisioned: both store an app password against an external PDS. + // Health = we can still create a session with the stored credentials. const raw = await community.getRawCredentials(communityDid); if (!raw?.appPasswordEncrypted || !raw.pdsEndpoint || !raw.identifier) { return c.json({ status: "expired" }); } try { const appPassword = await cipher.decryptString(raw.appPasswordEncrypted); - await createPdsSession(raw.pdsEndpoint, raw.identifier, appPassword, { + await ensureSession({ + community, + did: communityDid, + pdsEndpoint: raw.pdsEndpoint, + identifier: raw.identifier, + password: appPassword, fetch: cfg.fetch, }); return c.json({ status: "healthy" }); @@ -1010,6 +1260,41 @@ function generateKey(): string { return out; } +/** Build a ProvisionOrchestrator wired with real PDS/PLC clients backed by + * `cfg.fetch` (so tests can stub the network the same way they do for the + * mint/adopt routes). Mirrors the ad-hoc wrapper used in the live e2e test + * at apps/contrail-e2e/tests/provision.test.ts. */ +function buildOrchestrator( + cfg: import("./types").CommunityConfig, + adapter: CommunityAdapter, + cipher: CredentialCipher, + pdsDid: string +): ProvisionOrchestrator { + const plcDirectory = cfg.plcDirectory ?? "https://plc.directory"; + const fetchOpts = { fetch: cfg.fetch }; + + const plc: PlcClient = { + submit: (did, op) => submitGenesisOp(plcDirectory, did, op as any, fetchOpts), + }; + + const pds: PdsClient = { + createAccount: ({ pdsUrl, serviceAuthJwt, body }) => + pdsCreateAccount(pdsUrl, serviceAuthJwt, body, fetchOpts), + getRecommendedDidCredentials: ({ pdsUrl, accessJwt }) => + pdsGetRecommendedDidCredentials(pdsUrl, accessJwt, fetchOpts), + activateAccount: ({ pdsUrl, accessJwt }) => + pdsActivateAccount(pdsUrl, accessJwt, fetchOpts), + createAppPassword: async ({ pdsUrl, accessJwt, name }) => { + const r = await pdsCreateAppPassword(pdsUrl, accessJwt, name, fetchOpts); + return { password: r.password }; + }, + createSession: ({ pdsUrl, identifier, password }) => + createPdsSession(pdsUrl, identifier, password, fetchOpts), + }; + + return new ProvisionOrchestrator({ adapter, cipher, plc, pds, pdsDid }); +} + async function bootstrapReservedSpaces(args: { communityDid: string; creatorDid: string; @@ -1043,3 +1328,45 @@ async function bootstrapReservedSpaces(args: { await args.spaces.applyMembershipDiff(uri, [args.creatorDid], [], args.creatorDid); } } + +/** Ensure a usable PDS session for the given community DID. Tries the cached + * session first (with a 30s skew); if expired, tries refresh; if refresh fails + * (or there's no cache), falls back to creating a fresh session with the + * stored app password. The result is always written back to the cache. */ +async function ensureSession(args: { + community: CommunityAdapter; + did: string; + pdsEndpoint: string; + identifier: string; + password: string; + fetch?: typeof fetch; +}): Promise<{ accessJwt: string; refreshJwt: string }> { + const now = Math.floor(Date.now() / 1000); + const cached = await args.community.getSession(args.did); + if (cached && cached.accessExp > now + 30) { + return { accessJwt: cached.accessJwt, refreshJwt: cached.refreshJwt }; + } + if (cached) { + const refreshed = await tryRefreshSession({ + pdsUrl: args.pdsEndpoint, + refreshJwt: cached.refreshJwt, + fetch: args.fetch, + }); + if (refreshed) { + await args.community.upsertSession(args.did, refreshed); + return { accessJwt: refreshed.accessJwt, refreshJwt: refreshed.refreshJwt }; + } + } + const session = await createPdsSession( + args.pdsEndpoint, + args.identifier, + args.password, + { fetch: args.fetch } + ); + await args.community.upsertSession(args.did, { + accessJwt: session.accessJwt, + refreshJwt: session.refreshJwt, + accessExp: decodeJwtExp(session.accessJwt), + }); + return { accessJwt: session.accessJwt, refreshJwt: session.refreshJwt }; +} diff --git a/packages/contrail-community/src/schema.ts b/packages/contrail-community/src/schema.ts index cdcd56e..20d34f2 100644 --- a/packages/contrail-community/src/schema.ts +++ b/packages/contrail-community/src/schema.ts @@ -43,6 +43,57 @@ export function buildCommunitySchema(dialect: SqlDialect): string[] { note TEXT )`, `CREATE INDEX IF NOT EXISTS idx_community_invites_space ON community_invites(space_uri, created_at DESC)`, + + `CREATE TABLE IF NOT EXISTS provision_attempts ( + attempt_id TEXT PRIMARY KEY NOT NULL, + did TEXT NOT NULL, + status TEXT NOT NULL CHECK (status IN ( + 'keys_generated', + 'genesis_submitted', + 'account_created', + 'did_doc_updated', + 'activated' + )), + pds_endpoint TEXT NOT NULL, + handle TEXT NOT NULL, + email TEXT NOT NULL, + invite_code TEXT, + encrypted_signing_key TEXT, + encrypted_rotation_key TEXT, + encrypted_password TEXT, + genesis_submitted_at ${dialect.bigintType}, + account_created_at ${dialect.bigintType}, + did_doc_updated_at ${dialect.bigintType}, + activated_at ${dialect.bigintType}, + last_error TEXT, + created_at ${dialect.bigintType} NOT NULL, + updated_at ${dialect.bigintType} NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_provision_attempts_status ON provision_attempts(status, updated_at DESC)`, + `CREATE UNIQUE INDEX IF NOT EXISTS idx_provision_attempts_did ON provision_attempts(did)`, + + `CREATE TABLE IF NOT EXISTS provision_attempts_archive ( + attempt_id TEXT PRIMARY KEY NOT NULL, + did TEXT NOT NULL, + pds_endpoint TEXT NOT NULL, + handle TEXT NOT NULL, + email TEXT NOT NULL, + invite_code TEXT, + last_status TEXT, + last_error TEXT, + archived_at ${dialect.bigintType} NOT NULL, + tombstone_op_cid TEXT, + notes TEXT + )`, + `CREATE INDEX IF NOT EXISTS idx_provision_attempts_archive_archived_at ON provision_attempts_archive(archived_at DESC)`, + + `CREATE TABLE IF NOT EXISTS community_sessions ( + community_did TEXT PRIMARY KEY NOT NULL, + access_jwt TEXT NOT NULL, + refresh_jwt TEXT NOT NULL, + access_exp ${dialect.bigintType} NOT NULL, + updated_at ${dialect.bigintType} NOT NULL + )`, ]; } diff --git a/packages/contrail-community/src/service-auth.ts b/packages/contrail-community/src/service-auth.ts new file mode 100644 index 0000000..0c02073 --- /dev/null +++ b/packages/contrail-community/src/service-auth.ts @@ -0,0 +1,51 @@ +/** Mint an ES256 service-auth JWT for PDS XRPC calls. + * See com.atproto.server.createAccount handler: PDS verifies + * iss === did, aud === pds-did, lxm === lexicon-method, exp not in past. + * Signature is verified against the iss DID's atproto verificationMethod. */ + +import { signBytes, bytesToB64url } from "./plc"; + +function b64url(input: Uint8Array | string): string { + const b = typeof input === "string" ? new TextEncoder().encode(input) : input; + return bytesToB64url(b); +} + +function b64urlJson(obj: unknown): string { + return b64url(JSON.stringify(obj)); +} + +export interface MintServiceAuthInput { + /** Private JWK for the signing key (atproto verificationMethod). */ + privateJwk: JsonWebKey; + /** Issuer DID (the account's did:plc). */ + iss: string; + /** Audience DID (the target PDS, e.g. did:web:pds.example). */ + aud: string; + /** Lexicon method being authorized, e.g. com.atproto.server.createAccount. */ + lxm: string; + /** Token TTL in seconds. Defaults to 60. */ + ttlSec?: number; + /** Override "now" for deterministic tests; epoch milliseconds. */ + now?: number; +} + +export async function mintServiceAuthJwt(input: MintServiceAuthInput): Promise<string> { + const iat = Math.floor((input.now ?? Date.now()) / 1000); + const ttl = input.ttlSec ?? 60; + // Header: alg+typ only — atproto's service-auth verification doesn't use kid; + // the signing key is resolved from the iss DID's verificationMethod. + const header = { alg: "ES256", typ: "JWT" }; + const payload = { + iat, + iss: input.iss, + aud: input.aud, + exp: iat + ttl, + lxm: input.lxm, + jti: crypto.randomUUID(), + }; + const signingInput = `${b64urlJson(header)}.${b64urlJson(payload)}`; + // signBytes returns IEEE P1363 r||s (64 bytes), low-S normalized. + // JWT ES256 mandates raw r||s, NOT DER. atproto enforces low-S as well. + const sig = await signBytes(input.privateJwk, new TextEncoder().encode(signingInput)); + return `${signingInput}.${b64url(sig)}`; +} diff --git a/packages/contrail-community/src/types.ts b/packages/contrail-community/src/types.ts index 7f74698..4eba700 100644 --- a/packages/contrail-community/src/types.ts +++ b/packages/contrail-community/src/types.ts @@ -21,7 +21,47 @@ export function isAccessLevel(v: unknown): v is AccessLevel { return typeof v === "string" && ACCESS_LEVELS.includes(v as AccessLevel); } -export type CommunityMode = "adopt" | "mint"; +export type CommunityMode = "adopt" | "mint" | "provision"; + +export const PROVISION_STATUSES = [ + "keys_generated", + "genesis_submitted", + "account_created", + "did_doc_updated", + "activated", +] as const; +export type ProvisionStatus = (typeof PROVISION_STATUSES)[number]; + +export interface ProvisionAttemptRow { + attemptId: string; + did: string; + status: ProvisionStatus; + pdsEndpoint: string; + handle: string; + email: string; + inviteCode: string | null; + encryptedSigningKey: string | null; + encryptedRotationKey: string | null; + encryptedPassword: string | null; + genesisSubmittedAt: number | null; + accountCreatedAt: number | null; + didDocUpdatedAt: number | null; + activatedAt: number | null; + lastError: string | null; + createdAt: number; + updatedAt: number; +} + +export interface CreateProvisionAttemptInput { + attemptId: string; + did: string; + pdsEndpoint: string; + handle: string; + email: string; + inviteCode?: string | null; + encryptedSigningKey: string; + encryptedRotationKey: string; +} export interface CommunityConfig { /** Service DID for JWT verification. Falls back to spaces.serviceDid when both modules are enabled. */ @@ -35,6 +75,37 @@ export interface CommunityConfig { resolver?: DidDocumentResolver; /** Optional override for the fetch implementation (useful for tests). */ fetch?: typeof fetch; + /** Allowlist of PDS endpoints `community.provision` may create a community + * account on. Callers must supply a `pdsEndpoint` that matches one of these + * entries (after normalization); other values are rejected before any PLC + * op is signed. This gates ONLY provisioning — Contrail still reads/indexes + * records from every PDS on the network; this is not a global PDS filter. + * + * Fail-closed: when `allowProvisioning` is true this list MUST be non-empty, + * otherwise `community.provision` is refused. An empty/undefined allowlist + * no longer means "any PDS" — to genuinely accept any caller-supplied + * endpoint, set `allowAnyProvisionPdsEndpoint: true` (a separate, loud + * opt-in). Operators running a public/multi-tenant Contrail MUST keep a + * real allowlist here so callers can't mint PLC entries pointing at + * attacker-controlled PDSes signed by Contrail's rotation key. */ + allowedProvisionPdsEndpoints?: string[]; + /** Explicit, loud opt-in to accept ANY caller-supplied `pdsEndpoint` when + * provisioning is enabled. Only honored when `allowProvisioning` is true. + * This is the dangerous mode the allowlist exists to prevent: every + * successful call signs a permanent PLC genesis op pointing at a + * caller-controlled endpoint with Contrail's rotation key. Leave unset + * (or false) on any public/multi-tenant deployment and use + * `allowedProvisionPdsEndpoints` instead. */ + allowAnyProvisionPdsEndpoint?: boolean; + /** Top-level switch for the `community.provision` route. Default-deny: a + * call with the route present but this flag unset (or false) returns 403 + * ProvisioningDisabled BEFORE any PLC/PDS work runs. Set to `true` only + * when the operator has confirmed the upstream auth middleware restricts + * this route to authorized callers — every successful call burns a real + * invite code on the target PDS and adds a permanent entry to PLC. + * `mint` and `adopt` are not gated by this flag; they don't burn external + * resources. */ + allowProvisioning?: boolean; } /** Public view of a community row. Encrypted credentials are not included here diff --git a/packages/contrail-community/tests/cli-reap.test.ts b/packages/contrail-community/tests/cli-reap.test.ts new file mode 100644 index 0000000..0407475 --- /dev/null +++ b/packages/contrail-community/tests/cli-reap.test.ts @@ -0,0 +1,388 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import type { Database } from "@atmo-dev/contrail-base"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initCommunitySchema } from "../src/schema"; +import { CommunityAdapter } from "../src/adapter"; +import { CredentialCipher } from "../src/credentials"; +import { + generateKeyPair, + buildTombstoneOp, + signTombstoneOp, + submitTombstoneOp, + cidForOp, + type SignedGenesisOp, +} from "../src/plc"; +import { runReap, chooseReapDbSource } from "../src/cli/reap"; + +type SeedStatus = + | "keys_generated" + | "genesis_submitted" + | "account_created" + | "did_doc_updated" + | "activated"; + +interface SeedAttemptOpts { + attemptId: string; + did: string; + status: SeedStatus; +} + +async function seedAttempt( + adapter: CommunityAdapter, + cipher: CredentialCipher, + opts: SeedAttemptOpts +): Promise<{ rotationJwk: JsonWebKey }> { + const kp = await generateKeyPair(); + const encryptedRotation = await cipher.encrypt(JSON.stringify(kp.privateJwk)); + await adapter.createProvisionAttempt({ + attemptId: opts.attemptId, + did: opts.did, + pdsEndpoint: "https://pds.test", + handle: `${opts.attemptId}.pds.test`, + email: `${opts.attemptId}@x.test`, + encryptedSigningKey: await cipher.encrypt("{}"), + encryptedRotationKey: encryptedRotation, + }); + // Walk the row forward to its target status. The row starts at + // keys_generated after createProvisionAttempt. + const path: SeedStatus[] = [ + "genesis_submitted", + "account_created", + "did_doc_updated", + "activated", + ]; + for (const next of path) { + if (opts.status === "keys_generated") break; + await adapter.updateProvisionStatus(opts.attemptId, next); + if (next === opts.status) break; + } + return { rotationJwk: kp.privateJwk }; +} + +interface PlcCall { + url: string; + method: string; + body: any; +} + +/** Stand-in for what PLC's `/log/last` actually returns: the bare signed op + * object, no envelope. `getLastOpCid` computes the CID locally via cidForOp. */ +const FAKE_LAST_OP: SignedGenesisOp = { + type: "plc_operation", + prev: null, + rotationKeys: ["did:key:zQ3shfakerotation00000000000000000000000000000000000"], + verificationMethods: { atproto: "did:key:zQ3shfakeverif00000000000000000000000000000000000000" }, + alsoKnownAs: ["at://fixture.pds.test"], + services: { + atproto_pds: { type: "AtprotoPersonalDataServer", endpoint: "https://pds.test" }, + }, + sig: "fakesigfakesigfakesigfakesigfakesigfakesigfakesigfakesigfakesigfakesigfakesigfakesigfak", +}; + +function makeFakeFetch(calls: PlcCall[]): typeof fetch { + return (async (input: RequestInfo | URL, init?: RequestInit) => { + const url = String(input); + if (url.endsWith("/log/last")) { + return new Response(JSON.stringify(FAKE_LAST_OP), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + calls.push({ + url, + method: init?.method ?? "GET", + body: init?.body ? JSON.parse(String(init.body)) : null, + }); + return new Response("", { status: 200 }); + }) as typeof fetch; +} + +describe("runReap (cli reap)", () => { + let db: Database; + let adapter: CommunityAdapter; + let cipher: CredentialCipher; + + beforeEach(async () => { + db = createSqliteDatabase(":memory:"); + await initCommunitySchema(db); + cipher = new CredentialCipher(new Uint8Array(32).fill(7)); + adapter = new CommunityAdapter(db); + }); + + it("rejects when neither --attempt-id nor --all-stuck is set", async () => { + const result = await runReap({ + adapter, + cipher, + plcDirectory: "https://plc.test", + fetch: makeFakeFetch([]), + logger: { log: () => {}, error: () => {} }, + yes: true, + }); + expect(result.ok).toBe(false); + expect(result.error).toMatch(/--attempt-id|--all-stuck/i); + }); + + it("rejects when both --attempt-id and --all-stuck are set", async () => { + const result = await runReap({ + adapter, + cipher, + plcDirectory: "https://plc.test", + fetch: makeFakeFetch([]), + logger: { log: () => {}, error: () => {} }, + yes: true, + attemptId: "a1", + allStuck: true, + }); + expect(result.ok).toBe(false); + expect(result.error).toMatch(/mutually exclusive|both|exactly one/i); + }); + + it("real run with --attempt-id submits a tombstone and archives the row", async () => { + await seedAttempt(adapter, cipher, { + attemptId: "a-stuck", + did: "did:plc:stuck", + status: "genesis_submitted", + }); + + const calls: PlcCall[] = []; + const result = await runReap({ + adapter, + cipher, + plcDirectory: "https://plc.test", + fetch: makeFakeFetch(calls), + logger: { log: () => {}, error: () => {} }, + yes: true, + attemptId: "a-stuck", + dryRun: false, + }); + + expect(result.ok).toBe(true); + expect(result.reaped).toBe(1); + expect(result.errors).toBe(0); + expect(calls.length).toBe(1); + expect(calls[0]!.url).toBe("https://plc.test/did:plc:stuck"); + expect(calls[0]!.body.type).toBe("plc_tombstone"); + expect(calls[0]!.body.prev).toBe(await cidForOp(FAKE_LAST_OP)); + + // Original row removed from provision_attempts. + expect(await adapter.getProvisionAttempt("a-stuck")).toBeNull(); + // Archive row populated with the row's last live status. + const archive = await db + .prepare( + "SELECT * FROM provision_attempts_archive WHERE attempt_id = ?" + ) + .bind("a-stuck") + .first<Record<string, any>>(); + expect(archive).not.toBeNull(); + expect(archive!.did).toBe("did:plc:stuck"); + expect(archive!.last_status).toBe("genesis_submitted"); + expect(archive!.tombstone_op_cid).toBeTruthy(); + }); + + it("defaults to dry-run when dryRun is unspecified (safety default)", async () => { + await seedAttempt(adapter, cipher, { + attemptId: "a-stuck", + did: "did:plc:stuck", + status: "did_doc_updated", + }); + + const calls: PlcCall[] = []; + const result = await runReap({ + adapter, + cipher, + plcDirectory: "https://plc.test", + fetch: makeFakeFetch(calls), + logger: { log: () => {}, error: () => {} }, + yes: true, + attemptId: "a-stuck", + // dryRun INTENTIONALLY OMITTED — must default to dry-run. + }); + + expect(result.ok).toBe(true); + expect(result.reaped).toBe(0); + expect(result.dryRunSkipped).toBe(1); + expect(calls.length).toBe(0); + const row = await adapter.getProvisionAttempt("a-stuck"); + expect(row?.status).toBe("did_doc_updated"); + }); + + it("with --all-stuck reaps every non-activated row, regardless of status", async () => { + await seedAttempt(adapter, cipher, { + attemptId: "s1", + did: "did:plc:s1", + status: "keys_generated", + }); + await seedAttempt(adapter, cipher, { + attemptId: "s2", + did: "did:plc:s2", + status: "genesis_submitted", + }); + await seedAttempt(adapter, cipher, { + attemptId: "s3", + did: "did:plc:s3", + status: "did_doc_updated", + }); + // An activated row must NOT be reaped. + await seedAttempt(adapter, cipher, { + attemptId: "live", + did: "did:plc:live", + status: "activated", + }); + + const calls: PlcCall[] = []; + const result = await runReap({ + adapter, + cipher, + plcDirectory: "https://plc.test", + fetch: makeFakeFetch(calls), + logger: { log: () => {}, error: () => {} }, + yes: true, + allStuck: true, + dryRun: false, + olderThanMs: 0, // these rows are freshly seeded; disable the age floor + }); + + expect(result.ok).toBe(true); + expect(result.reaped).toBe(3); + expect(calls.map((c) => c.url).sort()).toEqual([ + "https://plc.test/did:plc:s1", + "https://plc.test/did:plc:s2", + "https://plc.test/did:plc:s3", + ]); + // The activated row is untouched. + const live = await adapter.getProvisionAttempt("live"); + expect(live?.status).toBe("activated"); + }); + + it("--all-stuck skips freshly-updated in-flight rows under the default age floor", async () => { + // A row mid-state-machine (updated_at ~ now) must survive a default + // --all-stuck run so reap can't tombstone a DID about to activate. + await seedAttempt(adapter, cipher, { + attemptId: "in-flight", + did: "did:plc:inflight", + status: "genesis_submitted", + }); + + const calls: PlcCall[] = []; + const result = await runReap({ + adapter, + cipher, + plcDirectory: "https://plc.test", + fetch: makeFakeFetch(calls), + logger: { log: () => {}, error: () => {} }, + yes: true, + allStuck: true, + dryRun: false, + // olderThanMs OMITTED — must apply the default 30-min floor. + }); + + expect(result.ok).toBe(true); + expect(result.reaped).toBe(0); + expect(calls.length).toBe(0); + // The in-flight row is untouched. + expect((await adapter.getProvisionAttempt("in-flight"))?.status).toBe( + "genesis_submitted" + ); + }); + + it("refuses to reap an activated row passed via --attempt-id", async () => { + await seedAttempt(adapter, cipher, { + attemptId: "live", + did: "did:plc:live", + status: "activated", + }); + + const calls: PlcCall[] = []; + const result = await runReap({ + adapter, + cipher, + plcDirectory: "https://plc.test", + fetch: makeFakeFetch(calls), + logger: { log: () => {}, error: () => {} }, + yes: true, + attemptId: "live", + dryRun: false, + }); + + expect(calls.length).toBe(0); + expect(result.errors).toBeGreaterThanOrEqual(1); + const live = await adapter.getProvisionAttempt("live"); + expect(live?.status).toBe("activated"); + }); +}); + +describe("chooseReapDbSource", () => { + it("selects Postgres when --db is given", () => { + expect( + chooseReapDbSource({ db: "postgres://x", databaseUrl: undefined }) + ).toEqual({ kind: "postgres", url: "postgres://x" }); + }); + + it("selects Postgres from DATABASE_URL when --db is absent", () => { + expect( + chooseReapDbSource({ db: undefined, databaseUrl: "postgres://env" }) + ).toEqual({ kind: "postgres", url: "postgres://env" }); + }); + + it("prefers an explicit --db over DATABASE_URL", () => { + expect( + chooseReapDbSource({ db: "postgres://flag", databaseUrl: "postgres://env" }) + ).toEqual({ kind: "postgres", url: "postgres://flag" }); + }); + + it("falls back to the D1 binding when neither is set", () => { + expect(chooseReapDbSource({ db: undefined, databaseUrl: undefined })).toEqual({ + kind: "d1", + }); + }); +}); + +describe("plc tombstone helpers", () => { + it("signTombstoneOp adds a base64url sig", async () => { + const kp = await generateKeyPair(); + const op = buildTombstoneOp("bafyreigenesis"); + const signed = await signTombstoneOp(op, kp.privateJwk); + expect(signed.type).toBe("plc_tombstone"); + expect(signed.prev).toBe("bafyreigenesis"); + expect(signed.sig).toMatch(/^[A-Za-z0-9_-]+$/); + }); + + it("submitTombstoneOp POSTs to the PLC directory at the DID URL", async () => { + const kp = await generateKeyPair(); + const signed = await signTombstoneOp( + buildTombstoneOp("bafyreigenesis"), + kp.privateJwk + ); + + let calledUrl = ""; + let calledBody: any = null; + const fakeFetch: typeof fetch = (async (input: RequestInfo | URL, init?: RequestInit) => { + calledUrl = String(input); + calledBody = init?.body ? JSON.parse(String(init.body)) : null; + return new Response("", { status: 200 }); + }) as typeof fetch; + + await submitTombstoneOp("https://plc.test", "did:plc:abc", signed, { + fetch: fakeFetch, + }); + expect(calledUrl).toBe("https://plc.test/did:plc:abc"); + expect(calledBody.type).toBe("plc_tombstone"); + expect(calledBody.prev).toBe("bafyreigenesis"); + expect(calledBody.sig).toBe(signed.sig); + }); + + it("submitTombstoneOp throws on non-2xx", async () => { + const kp = await generateKeyPair(); + const signed = await signTombstoneOp( + buildTombstoneOp("bafyreigenesis"), + kp.privateJwk + ); + const fakeFetch: typeof fetch = (async () => + new Response("denied", { status: 400 })) as typeof fetch; + await expect( + submitTombstoneOp("https://plc.test", "did:plc:abc", signed, { + fetch: fakeFetch, + }) + ).rejects.toThrow(/400.*denied/); + }); +}); diff --git a/packages/contrail-community/tests/community-provision-attempts.test.ts b/packages/contrail-community/tests/community-provision-attempts.test.ts new file mode 100644 index 0000000..486888b --- /dev/null +++ b/packages/contrail-community/tests/community-provision-attempts.test.ts @@ -0,0 +1,247 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import type { Database } from "@atmo-dev/contrail-base"; +import { initCommunitySchema } from "../src/schema"; +import { CommunityAdapter } from "../src/adapter"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; + +describe("provision_attempts adapter", () => { + let db: Database; + let adapter: CommunityAdapter; + + beforeEach(async () => { + db = createSqliteDatabase(":memory:"); + await initCommunitySchema(db); + adapter = new CommunityAdapter(db); + }); + + it("creates and reads a provision attempt", async () => { + const before = Date.now(); + await adapter.createProvisionAttempt({ + attemptId: "a1", + did: "did:plc:abc", + pdsEndpoint: "https://pds.test", + handle: "abc.pds.test", + email: "abc@x.test", + inviteCode: "code-1", + encryptedSigningKey: "sk-enc", + encryptedRotationKey: "rk-enc", + }); + + const row = await adapter.getProvisionAttempt("a1"); + expect(row).not.toBeNull(); + expect(row?.attemptId).toBe("a1"); + expect(row?.did).toBe("did:plc:abc"); + expect(row?.status).toBe("keys_generated"); + expect(row?.pdsEndpoint).toBe("https://pds.test"); + expect(row?.handle).toBe("abc.pds.test"); + expect(row?.email).toBe("abc@x.test"); + expect(row?.inviteCode).toBe("code-1"); + expect(row?.encryptedSigningKey).toBe("sk-enc"); + expect(row?.encryptedRotationKey).toBe("rk-enc"); + expect(row?.encryptedPassword).toBeNull(); + expect(row?.lastError).toBeNull(); + expect(row?.genesisSubmittedAt).toBeNull(); + expect(row?.accountCreatedAt).toBeNull(); + expect(row?.didDocUpdatedAt).toBeNull(); + expect(row?.activatedAt).toBeNull(); + expect(row?.createdAt).toBeGreaterThanOrEqual(before); + expect(row?.updatedAt).toBeGreaterThanOrEqual(before); + }); + + it("getProvisionAttempt returns null for unknown attempt", async () => { + const row = await adapter.getProvisionAttempt("no-such-attempt"); + expect(row).toBeNull(); + }); + + it("treats missing inviteCode as null", async () => { + await adapter.createProvisionAttempt({ + attemptId: "a-no-invite", + did: "did:plc:noinv", + pdsEndpoint: "https://pds.test", + handle: "noinv.pds.test", + email: "noinv@x.test", + encryptedSigningKey: "sk", + encryptedRotationKey: "rk", + }); + const row = await adapter.getProvisionAttempt("a-no-invite"); + expect(row?.inviteCode).toBeNull(); + }); + + it("advances status, stamps the matching timestamp, and persists last_error", async () => { + await adapter.createProvisionAttempt({ + attemptId: "a1", + did: "did:plc:abc", + pdsEndpoint: "https://pds.test", + handle: "abc.pds.test", + email: "abc@x.test", + encryptedSigningKey: "sk-enc", + encryptedRotationKey: "rk-enc", + }); + + const initial = await adapter.getProvisionAttempt("a1"); + const initialUpdated = initial!.updatedAt; + + // Small wait so updated_at can advance on millisecond clocks. + await new Promise((r) => setTimeout(r, 2)); + + await adapter.updateProvisionStatus("a1", "genesis_submitted"); + let row = await adapter.getProvisionAttempt("a1"); + expect(row?.status).toBe("genesis_submitted"); + expect(row?.genesisSubmittedAt).toBeTruthy(); + expect(row?.accountCreatedAt).toBeNull(); + expect(row?.didDocUpdatedAt).toBeNull(); + expect(row?.activatedAt).toBeNull(); + expect(row?.updatedAt).toBeGreaterThanOrEqual(initialUpdated); + + await adapter.updateProvisionStatus("a1", "account_created"); + row = await adapter.getProvisionAttempt("a1"); + expect(row?.status).toBe("account_created"); + expect(row?.accountCreatedAt).toBeTruthy(); + // Earlier stamp must be preserved on subsequent updates. + expect(row?.genesisSubmittedAt).toBeTruthy(); + + await adapter.updateProvisionStatus("a1", "did_doc_updated", { lastError: "transient PLC error" }); + row = await adapter.getProvisionAttempt("a1"); + expect(row?.status).toBe("did_doc_updated"); + expect(row?.lastError).toBe("transient PLC error"); + // Earlier stamps still preserved. + expect(row?.genesisSubmittedAt).toBeTruthy(); + expect(row?.accountCreatedAt).toBeTruthy(); + }); + + it("persists encryptedPassword via updateProvisionStatus", async () => { + await adapter.createProvisionAttempt({ + attemptId: "a-pwd", + did: "did:plc:pwd", + pdsEndpoint: "https://pds.test", + handle: "pwd.pds.test", + email: "pwd@x.test", + encryptedSigningKey: "sk", + encryptedRotationKey: "rk", + }); + expect((await adapter.getProvisionAttempt("a-pwd"))?.encryptedPassword).toBeNull(); + + await adapter.updateProvisionStatus("a-pwd", "account_created", { + encryptedPassword: "pwd-enc", + }); + const row = await adapter.getProvisionAttempt("a-pwd"); + expect(row?.encryptedPassword).toBe("pwd-enc"); + expect(row?.accountCreatedAt).toBeTruthy(); + }); + + describe("listStuckAttempts age threshold", () => { + async function seedStuck(attemptId: string, did: string): Promise<void> { + await adapter.createProvisionAttempt({ + attemptId, + did, + pdsEndpoint: "https://pds.test", + handle: `${attemptId}.pds.test`, + email: `${attemptId}@x.test`, + encryptedSigningKey: "sk", + encryptedRotationKey: "rk", + }); + } + /** Backdate a row's updated_at so it looks old to the age filter. */ + async function ageRow(attemptId: string, ageMs: number): Promise<void> { + await db + .prepare(`UPDATE provision_attempts SET updated_at = ? WHERE attempt_id = ?`) + .bind(Date.now() - ageMs, attemptId) + .run(); + } + + it("excludes a freshly-updated (in-flight) non-activated row", async () => { + await seedStuck("fresh", "did:plc:fresh"); + // updated_at is ~now; a 30-minute floor must not select it. + const rows = await adapter.listStuckAttempts(30 * 60 * 1000); + expect(rows.map((r) => r.attemptId)).not.toContain("fresh"); + }); + + it("includes a row older than the threshold", async () => { + await seedStuck("old", "did:plc:old"); + await ageRow("old", 2 * 60 * 60 * 1000); // 2 hours ago + const rows = await adapter.listStuckAttempts(30 * 60 * 1000); + expect(rows.map((r) => r.attemptId)).toContain("old"); + }); + + it("with a zero threshold returns every non-activated row", async () => { + await seedStuck("a", "did:plc:a"); + await seedStuck("b", "did:plc:b"); + const rows = await adapter.listStuckAttempts(0); + expect(rows.map((r) => r.attemptId).sort()).toEqual(["a", "b"]); + }); + }); + + describe("archiveStuckAttempt idempotency", () => { + it("retry after a partial failure (archive row already present, live row stranded) does not throw and finishes the move", async () => { + await adapter.createProvisionAttempt({ + attemptId: "partial", + did: "did:plc:partial", + pdsEndpoint: "https://pds.test", + handle: "partial.pds.test", + email: "partial@x.test", + encryptedSigningKey: "sk", + encryptedRotationKey: "rk", + }); + // Simulate the first reap: its archive INSERT landed, but the live-row + // DELETE failed, leaving the row in BOTH tables. + await db + .prepare( + `INSERT INTO provision_attempts_archive + (attempt_id, did, pds_endpoint, handle, email, invite_code, + last_status, last_error, archived_at, tombstone_op_cid, notes) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)` + ) + .bind( + "partial", + "did:plc:partial", + "https://pds.test", + "partial.pds.test", + "partial@x.test", + null, + "genesis_submitted", + null, + Date.now(), + "cid-first", + null + ) + .run(); + + // The retry must not hit a PRIMARY KEY conflict on the archive INSERT. + await expect( + adapter.archiveStuckAttempt("partial", { tombstoneOpCid: "cid-retry" }) + ).resolves.toBeUndefined(); + + // Live row is now gone; the archive row remains (the original landed copy). + expect(await adapter.getProvisionAttempt("partial")).toBeNull(); + const archive = await db + .prepare("SELECT * FROM provision_attempts_archive WHERE attempt_id = ?") + .bind("partial") + .first<Record<string, any>>(); + expect(archive).not.toBeNull(); + expect(archive!.tombstone_op_cid).toBe("cid-first"); + }); + }); + + it("enforces did uniqueness across attempts", async () => { + await adapter.createProvisionAttempt({ + attemptId: "first", + did: "did:plc:dupe", + pdsEndpoint: "https://pds.test", + handle: "first.pds.test", + email: "first@x.test", + encryptedSigningKey: "sk", + encryptedRotationKey: "rk", + }); + await expect( + adapter.createProvisionAttempt({ + attemptId: "second", + did: "did:plc:dupe", + pdsEndpoint: "https://pds.test", + handle: "second.pds.test", + email: "second@x.test", + encryptedSigningKey: "sk", + encryptedRotationKey: "rk", + }) + ).rejects.toThrow(); + }); +}); diff --git a/packages/contrail-community/tests/community-provision-pds-allowlist.test.ts b/packages/contrail-community/tests/community-provision-pds-allowlist.test.ts new file mode 100644 index 0000000..8cf2abf --- /dev/null +++ b/packages/contrail-community/tests/community-provision-pds-allowlist.test.ts @@ -0,0 +1,301 @@ +import { describe, it, expect } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; +import { normalizePdsEndpoint } from "../src/pds"; + +const ALICE = "did:plc:alice"; +const MASTER_KEY = new Uint8Array(32).fill(99); +const ALLOWED_PDS = "https://allowed.pds.test"; +const ATTACKER_PDS = "https://attacker.pds.test"; +const PLC_DIRECTORY = "https://plc.test"; + +const FAKE_ACCESS_JWT = "head.body.sig"; + +async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + const method = init?.method ?? "GET"; + const body = init?.body ? JSON.parse(init.body as string) : {}; + + if (url === `${ALLOWED_PDS}/xrpc/com.atproto.server.describeServer`) { + return new Response(JSON.stringify({ did: "did:web:allowed.pds.test" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (url.startsWith(`${PLC_DIRECTORY}/`) && !url.endsWith("/log/last") && method === "POST") { + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + } + if (url.endsWith("/log/last") && method === "GET") { + return new Response(JSON.stringify({ cid: "bafyreitestcid" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + if (url === `${ALLOWED_PDS}/xrpc/com.atproto.server.createAccount` && method === "POST") { + return new Response( + JSON.stringify({ + did: body.did, + handle: body.handle, + accessJwt: FAKE_ACCESS_JWT, + refreshJwt: "RT", + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + if (url === `${ALLOWED_PDS}/xrpc/com.atproto.identity.getRecommendedDidCredentials`) { + return new Response( + JSON.stringify({ + rotationKeys: [], + verificationMethods: { atproto: "did:key:zPdsSig" }, + alsoKnownAs: ["at://newcomm.allowed.pds.test"], + services: { + atproto_pds: { type: "AtprotoPersonalDataServer", endpoint: ALLOWED_PDS }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + if (url === `${ALLOWED_PDS}/xrpc/com.atproto.server.activateAccount` && method === "POST") { + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + } + if (url === `${ALLOWED_PDS}/xrpc/com.atproto.server.createAppPassword` && method === "POST") { + return new Response( + JSON.stringify({ name: body.name, password: "minted-app-pw" }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + return new Response(`unmocked: ${method} ${url}`, { status: 404 }); +} + +function buildConfig( + allowedProvisionPdsEndpoints: string[] | undefined, + extra: { allowAnyProvisionPdsEndpoint?: boolean } = {} +): ContrailConfig { + return { + namespace: "test.comm", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, + }, + community: { + masterKey: MASTER_KEY, + plcDirectory: PLC_DIRECTORY, + fetch: mockFetch, + allowedProvisionPdsEndpoints, + allowProvisioning: true, + allowAnyProvisionPdsEndpoint: extra.allowAnyProvisionPdsEndpoint, + }, + }; +} + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { issuer: did, audience: "did:web:test.example#svc", lxm: undefined }); + await next(); + }; +} + +async function makeApp( + allowedProvisionPdsEndpoints: string[] | undefined, + extra: { allowAnyProvisionPdsEndpoint?: boolean } = {} +): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const cfg = buildConfig(allowedProvisionPdsEndpoints, extra); + const resolved = resolveConfig(cfg); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, community }); +} + +async function call(app: Hono, body: any): Promise<Response> { + return await app.fetch( + new Request(`http://localhost/xrpc/test.comm.community.provision`, { + method: "POST", + headers: { "X-Test-Did": ALICE, "Content-Type": "application/json" }, + body: JSON.stringify(body), + }) + ); +} + +describe("provision pdsEndpoint allowlist (M3)", () => { + it("rejects pdsEndpoint not in allowedProvisionPdsEndpoints", async () => { + const app = await makeApp([ALLOWED_PDS]); + const res = await call(app, { + handle: "newcomm.attacker.pds.test", + email: "x@x.test", + password: "secret", + pdsEndpoint: ATTACKER_PDS, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(400); + const j = (await res.json()) as { error: string; message: string }; + expect(j.error).toBe("InvalidRequest"); + expect(j.message).toMatch(/pdsEndpoint/i); + }); + + it("accepts pdsEndpoint that is in allowedProvisionPdsEndpoints", async () => { + const app = await makeApp([ALLOWED_PDS]); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: ALLOWED_PDS, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(200); + }); + + it("fails closed: provisioning enabled + undefined allowlist → rejected", async () => { + const app = await makeApp(undefined); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: ALLOWED_PDS, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(403); + const j = (await res.json()) as { error: string; message: string }; + expect(j.error).toBe("ProvisioningMisconfigured"); + expect(j.message).toMatch( + /allowlist|allowedProvisionPdsEndpoints|allowAnyProvisionPdsEndpoint/i + ); + }); + + it("fails closed: provisioning enabled + empty allowlist → rejected", async () => { + const app = await makeApp([]); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: ALLOWED_PDS, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(403); + const j = (await res.json()) as { error: string; message: string }; + expect(j.error).toBe("ProvisioningMisconfigured"); + }); + + it("allowAnyProvisionPdsEndpoint=true is a loud opt-in: empty allowlist accepts any pdsEndpoint", async () => { + const app = await makeApp([], { allowAnyProvisionPdsEndpoint: true }); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: ALLOWED_PDS, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(200); + }); + + it("matches when caller adds a trailing slash to a slash-less allowlist entry", async () => { + const app = await makeApp([ALLOWED_PDS]); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: `${ALLOWED_PDS}/`, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(200); + }); + + it("matches when caller uppercases the scheme on an allowlisted endpoint", async () => { + const app = await makeApp([ALLOWED_PDS]); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: ALLOWED_PDS.replace(/^https/, "HTTPS"), + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(200); + }); + + it("matches when caller appends the default :443 port", async () => { + const app = await makeApp([ALLOWED_PDS]); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: ALLOWED_PDS.replace(/^https:\/\/([^/]+)/, "https://$1:443"), + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(200); + }); + + it("rejects pdsEndpoint that is not a parseable URL", async () => { + const app = await makeApp([ALLOWED_PDS]); + const res = await call(app, { + handle: "newcomm.allowed.pds.test", + email: "x@x.test", + password: "secret", + pdsEndpoint: "not a url", + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(400); + const j = (await res.json()) as { error: string; message: string }; + expect(j.error).toBe("InvalidRequest"); + expect(j.message).toMatch(/parseable|url/i); + }); +}); + +describe("normalizePdsEndpoint", () => { + it("collapses scheme case", () => { + expect(normalizePdsEndpoint("HTTPS://pds.example.com")).toBe( + "https://pds.example.com" + ); + }); + it("collapses host case", () => { + expect(normalizePdsEndpoint("https://PDS.Example.com")).toBe( + "https://pds.example.com" + ); + }); + it("strips trailing slash", () => { + expect(normalizePdsEndpoint("https://pds.example.com/")).toBe( + "https://pds.example.com" + ); + }); + it("strips default :443 for https", () => { + expect(normalizePdsEndpoint("https://pds.example.com:443")).toBe( + "https://pds.example.com" + ); + }); + it("strips default :80 for http", () => { + expect(normalizePdsEndpoint("http://pds.example.com:80")).toBe( + "http://pds.example.com" + ); + }); + it("preserves a non-default port", () => { + expect(normalizePdsEndpoint("https://pds.example.com:8443")).toBe( + "https://pds.example.com:8443" + ); + }); + it("converts an IDN hostname to its punycode form", () => { + expect(normalizePdsEndpoint("https://exämple.com")).toBe( + "https://xn--exmple-cua.com" + ); + }); + it("throws on an unparseable URL", () => { + expect(() => normalizePdsEndpoint("not a url")).toThrow(); + }); +}); diff --git a/packages/contrail-community/tests/community-provision-router.test.ts b/packages/contrail-community/tests/community-provision-router.test.ts new file mode 100644 index 0000000..535c70e --- /dev/null +++ b/packages/contrail-community/tests/community-provision-router.test.ts @@ -0,0 +1,336 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; + +const ALICE = "did:plc:alice"; +const MASTER_KEY = new Uint8Array(32).fill(99); +const PDS_ENDPOINT = "https://pds.test"; +const PLC_DIRECTORY = "https://plc.test"; +/** The DID describeServer claims for this PDS. INTENTIONALLY DIFFERENT from + * CONFIG.spaces.serviceDid so tests can detect a regression where the route + * falls back to the spaces DID instead of resolving the PDS DID dynamically. */ +const PDS_DESCRIBE_DID = "did:web:pds.test"; + +/** Captures upstream calls so we can assert the right RPCs ran. */ +const upstreamCalls: Array<{ url: string; method: string; body: any; authorization?: string }> = []; + +// Placeholder JWT — the orchestrator passes accessJwt through to PDS calls +// untouched; nothing in the contrail flow parses its claims. +const FAKE_ACCESS_JWT = "head.body.sig"; + +async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + const method = init?.method ?? "GET"; + const body = init?.body ? JSON.parse(init.body as string) : {}; + const headers = new Headers(init?.headers ?? {}); + upstreamCalls.push({ + url, + method, + body, + authorization: headers.get("authorization") ?? undefined, + }); + + // PDS describeServer — used by the route to resolve the target PDS's DID + // for service-auth JWT `aud`. + if (url === `${PDS_ENDPOINT}/xrpc/com.atproto.server.describeServer`) { + return new Response(JSON.stringify({ did: PDS_DESCRIBE_DID }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + // PLC submit: POST {plcDirectory}/{did} (genesis + update share the URL). + if (url.startsWith(`${PLC_DIRECTORY}/`) && url.endsWith("/log/last") === false && method === "POST") { + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + } + // PLC log/last: not used in the happy path but be defensive. + if (url.endsWith("/log/last") && method === "GET") { + return new Response(JSON.stringify({ cid: "bafyreitestcid" }), { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + // PDS createAccount. + if (url === `${PDS_ENDPOINT}/xrpc/com.atproto.server.createAccount` && method === "POST") { + return new Response( + JSON.stringify({ + did: body.did, + handle: body.handle, + accessJwt: FAKE_ACCESS_JWT, + refreshJwt: "RT", + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + // PDS getRecommendedDidCredentials. + if ( + url === `${PDS_ENDPOINT}/xrpc/com.atproto.identity.getRecommendedDidCredentials` + ) { + return new Response( + JSON.stringify({ + rotationKeys: [], + verificationMethods: { atproto: "did:key:zPdsSig" }, + alsoKnownAs: ["at://newcomm.pds.test"], + services: { + atproto_pds: { + type: "AtprotoPersonalDataServer", + endpoint: PDS_ENDPOINT, + }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + // PDS activateAccount. + if (url === `${PDS_ENDPOINT}/xrpc/com.atproto.server.activateAccount` && method === "POST") { + return new Response("{}", { status: 200, headers: { "content-type": "application/json" } }); + } + // PDS createAppPassword (post-activation, mints publishing credential). + if (url === `${PDS_ENDPOINT}/xrpc/com.atproto.server.createAppPassword` && method === "POST") { + return new Response( + JSON.stringify({ name: body.name, password: "minted-app-pw" }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + + return new Response(`unmocked: ${method} ${url}`, { status: 404 }); +} + +const CONFIG: ContrailConfig = { + namespace: "test.comm", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, + }, + community: { + masterKey: MASTER_KEY, + plcDirectory: PLC_DIRECTORY, + fetch: mockFetch, + allowProvisioning: true, + // Fail-closed requires a non-empty allowlist when provisioning is enabled. + allowedProvisionPdsEndpoints: [PDS_ENDPOINT], + }, +}; + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); + await next(); + }; +} + +async function makeApp(): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const resolved = resolveConfig(CONFIG); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, community }); +} + +async function call( + app: Hono, + method: string, + path: string, + did: string | null, + body?: any +): Promise<Response> { + const headers: Record<string, string> = {}; + if (did !== null) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return await app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +describe("POST /xrpc/{ns}.community.provision (allowProvisioning gate)", () => { + // Builds an app whose community config OMITS allowProvisioning. The route + // is expected to refuse with 403 ProvisioningDisabled — operators must + // explicitly opt in. The default-deny posture protects deployments where + // the auth middleware allows broader audiences than "operator only" from + // having any authenticated caller mint communities + burn invite codes. + async function makeAppWithoutAllowProvisioning(): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const configWithoutFlag: ContrailConfig = { + ...CONFIG, + community: { ...CONFIG.community!, allowProvisioning: undefined } as any, + }; + const resolved = resolveConfig(configWithoutFlag); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, community }); + } + + it("returns 403 ProvisioningDisabled when allowProvisioning is not set", async () => { + const app = await makeAppWithoutAllowProvisioning(); + const res = await call(app, "POST", "/xrpc/test.comm.community.provision", ALICE, { + handle: "newcomm.pds.test", + email: "newcomm@x.test", + password: "secret", + pdsEndpoint: PDS_ENDPOINT, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(403); + const body = (await res.json()) as { error: string }; + expect(body.error).toBe("ProvisioningDisabled"); + }); +}); + +describe("POST /xrpc/{ns}.community.provision", () => { + let app: Hono; + + beforeAll(async () => { + app = await makeApp(); + }); + + it("requires auth", async () => { + const res = await call(app, "POST", "/xrpc/test.comm.community.provision", null, { + handle: "x.pds.test", + email: "x@x.test", + password: "p", + pdsEndpoint: PDS_ENDPOINT, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(401); + }); + + it("rejects missing required fields", async () => { + const res = await call(app, "POST", "/xrpc/test.comm.community.provision", ALICE, {}); + expect(res.status).toBe(400); + const j = (await res.json()) as { error: string }; + expect(j.error).toBe("InvalidRequest"); + }); + + it("provisions a community and returns did + status=activated", async () => { + const before = upstreamCalls.length; + const res = await call(app, "POST", "/xrpc/test.comm.community.provision", ALICE, { + handle: "newcomm.pds.test", + email: "newcomm@x.test", + password: "secret", + inviteCode: "code-x", + pdsEndpoint: PDS_ENDPOINT, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(200); + const body = (await res.json()) as { communityDid: string; status: string }; + + expect(body.communityDid).toMatch(/^did:plc:[a-z2-7]{24}$/); + expect(body.status).toBe("activated"); + + // Verify the row was inserted into communities with mode='provision'. + // We round-trip via the GET list endpoint so we don't have to reach into + // the adapter — the route bootstrapped reserved spaces with the caller as + // owner, which makes the community reachable. + const listRes = await call(app, "GET", "/xrpc/test.comm.community.list", ALICE); + expect(listRes.status).toBe(200); + const list = (await listRes.json()) as { + communities: Array<{ did: string; mode: string }>; + }; + const row = list.communities.find((r) => r.did === body.communityDid); + expect(row).toBeDefined(); + expect(row!.mode).toBe("provision"); + + // Confirm we touched all five upstream RPCs: 2 PLC posts (genesis + update), + // createAccount, getRecommendedDidCredentials, activateAccount. + const ourCalls = upstreamCalls.slice(before); + const plcPosts = ourCalls.filter( + (c) => c.url.startsWith(`${PLC_DIRECTORY}/`) && c.method === "POST" + ); + expect(plcPosts.length).toBe(2); + expect( + ourCalls.some((c) => + c.url.endsWith("/xrpc/com.atproto.server.createAccount") + ) + ).toBe(true); + expect( + ourCalls.some((c) => + c.url.endsWith("/xrpc/com.atproto.identity.getRecommendedDidCredentials") + ) + ).toBe(true); + expect( + ourCalls.some((c) => + c.url.endsWith("/xrpc/com.atproto.server.activateAccount") + ) + ).toBe(true); + }); + + it("is idempotent on retry with the same attemptId after a fully-completed first call", async () => { + // The route already returns attemptId on every error response so a + // caller can retry. This guards the case where the first call + // succeeded end-to-end (orchestrator + graduation + reserved spaces) + // but the caller didn't receive the 200 (e.g. lost connection): a + // resent request with the same attemptId must still 200, return the + // same DID, and not double-create rows. + const attemptId = "retry-idem-1"; + const body = { + attemptId, + handle: "retryidem.pds.test", + email: "retryidem@x.test", + password: "secret", + pdsEndpoint: PDS_ENDPOINT, + rotationKey: "did:key:zStubCallerRotationKey", + }; + + const first = await call(app, "POST", "/xrpc/test.comm.community.provision", ALICE, body); + expect(first.status).toBe(200); + const firstJson = (await first.json()) as { communityDid: string }; + + const second = await call(app, "POST", "/xrpc/test.comm.community.provision", ALICE, body); + expect(second.status).toBe(200); + const secondJson = (await second.json()) as { communityDid: string }; + + expect(secondJson.communityDid).toBe(firstJson.communityDid); + }); + + it("uses the describeServer-returned DID as the service-auth JWT audience (not cfg.serviceDid)", async () => { + const before = upstreamCalls.length; + const res = await call(app, "POST", "/xrpc/test.comm.community.provision", ALICE, { + handle: "audtest.pds.test", + email: "audtest@x.test", + password: "secret", + pdsEndpoint: PDS_ENDPOINT, + rotationKey: "did:key:zStubCallerRotationKey", + }); + expect(res.status).toBe(200); + + const ourCalls = upstreamCalls.slice(before); + + // 1. The route must call describeServer on the target PDS. + const describeCall = ourCalls.find( + (c) => c.url === `${PDS_ENDPOINT}/xrpc/com.atproto.server.describeServer` + ); + expect(describeCall).toBeDefined(); + + // 2. The createAccount call's Authorization Bearer JWT must have + // `aud` === the describeServer-returned DID, NOT cfg.serviceDid. + const createAccountCall = ourCalls.find( + (c) => c.url === `${PDS_ENDPOINT}/xrpc/com.atproto.server.createAccount` + ); + expect(createAccountCall).toBeDefined(); + expect(createAccountCall!.authorization).toMatch(/^Bearer /); + + const jwt = createAccountCall!.authorization!.replace(/^Bearer /, ""); + const payloadSeg = jwt.split(".")[1]!; + const padded = payloadSeg.replace(/-/g, "+").replace(/_/g, "/"); + const padding = "=".repeat((4 - (padded.length % 4)) % 4); + const claims = JSON.parse(atob(padded + padding)) as { aud?: string }; + + expect(claims.aud).toBe(PDS_DESCRIBE_DID); + // Sanity: it is NOT the spaces serviceDid (the previous hardcoded value). + expect(claims.aud).not.toBe(CONFIG.spaces!.authority!.serviceDid); + }); +}); diff --git a/packages/contrail-community/tests/community-publish-401-clears-session.test.ts b/packages/contrail-community/tests/community-publish-401-clears-session.test.ts new file mode 100644 index 0000000..fb4f999 --- /dev/null +++ b/packages/contrail-community/tests/community-publish-401-clears-session.test.ts @@ -0,0 +1,150 @@ +/** L6: A 401 from the publish path used to leave the bad session in the + * cache, so every subsequent publish hit the same 401 permanently. The fix + * is small: on 401, drop the cached session row. The next request goes cold + * through ensureSession, which mints a fresh session from the stored app + * password (or fails permanently if the app password itself was revoked). */ + +import { describe, it, expect, beforeEach } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { + CommunityAdapter, + CredentialCipher, + RESERVED_KEYS, +} from "../src"; +import { HostedAdapter } from "@atmo-dev/contrail"; +import { buildSpaceUri } from "@atmo-dev/contrail"; + +const ALICE = "did:plc:alice"; +const COMMUNITY_DID = "did:plc:l6comm"; +const HANDLE = "l6.pds.test"; +const PDS = "https://pds.example"; +const MASTER_KEY = new Uint8Array(32).fill(13); +const APP_PASSWORD = "correct-pw"; + +function fakeAuth(spaceServiceDid: string): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: spaceServiceDid, + lxm: undefined, + }); + await next(); + }; +} + +async function build(): Promise<{ app: Hono; adapter: CommunityAdapter }> { + const fetchImpl: typeof fetch = (async (input: RequestInfo | URL) => { + const url = + typeof input === "string" + ? input + : input instanceof URL + ? input.href + : input.url; + if (url.endsWith("/xrpc/com.atproto.repo.createRecord")) { + return new Response(JSON.stringify({ error: "AuthRequired" }), { + status: 401, + }); + } + return new Response("not found", { status: 404 }); + }) as typeof fetch; + + const config: ContrailConfig = { + namespace: "test.comm", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, + }, + community: { masterKey: MASTER_KEY, fetch: fetchImpl }, + }; + const db = createSqliteDatabase(":memory:"); + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [communityIntegration.applySchema] }); + const app = createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth(config.spaces!.authority!.serviceDid) }, + community: communityIntegration, + }); + + const cipher = new CredentialCipher(MASTER_KEY); + const community = new CommunityAdapter(db); + const spaces = new HostedAdapter(db, resolved); + await community.createFromProvisioned({ + did: COMMUNITY_DID, + pdsEndpoint: PDS, + handle: HANDLE, + appPasswordEncrypted: await cipher.encrypt(APP_PASSWORD), + createdBy: ALICE, + }); + for (const key of RESERVED_KEYS) { + const uri = buildSpaceUri({ + ownerDid: COMMUNITY_DID, + type: config.spaces!.authority!.type, + key, + }); + await spaces.createSpace({ + uri, + ownerDid: COMMUNITY_DID, + type: config.spaces!.authority!.type, + key, + serviceDid: config.spaces!.authority!.serviceDid, + appPolicyRef: null, + appPolicy: null, + }); + await community.grant({ + spaceUri: uri, + subjectDid: ALICE, + accessLevel: "owner", + grantedBy: ALICE, + }); + await spaces.applyMembershipDiff(uri, [ALICE], [], ALICE); + } + return { app, adapter: community }; +} + +describe("publish path: 401 clears the session cache (L6)", () => { + let app: Hono; + let adapter: CommunityAdapter; + + beforeEach(async () => { + ({ app, adapter } = await build()); + }); + + it("removes the cached session row when createRecord returns 401", async () => { + // Seed a cached session that will be used (and rejected) by createRecord. + await adapter.upsertSession(COMMUNITY_DID, { + accessJwt: "stale-access", + refreshJwt: "stale-refresh", + accessExp: Math.floor(Date.now() / 1000) + 3600, + }); + expect(await adapter.getSession(COMMUNITY_DID)).not.toBeNull(); + + const res = await app.fetch( + new Request("http://localhost/xrpc/test.comm.community.putRecord", { + method: "POST", + headers: { "X-Test-Did": ALICE, "Content-Type": "application/json" }, + body: JSON.stringify({ + communityDid: COMMUNITY_DID, + collection: "app.event.message", + record: { text: "hello" }, + }), + }) + ); + expect(res.status).toBe(502); + + // The stale session must be gone, so the next attempt mints a fresh one. + expect(await adapter.getSession(COMMUNITY_DID)).toBeNull(); + }); +}); diff --git a/packages/contrail-community/tests/community-publishing.test.ts b/packages/contrail-community/tests/community-publishing.test.ts index a05ee0f..ccbac88 100644 --- a/packages/contrail-community/tests/community-publishing.test.ts +++ b/packages/contrail-community/tests/community-publishing.test.ts @@ -5,13 +5,18 @@ import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; import { initSchema } from "@atmo-dev/contrail"; import { createApp } from "@atmo-dev/contrail"; import { resolveConfig } from "@atmo-dev/contrail"; -import type { ContrailConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig, Database } from "@atmo-dev/contrail"; import { createCommunityIntegration } from "../src/integration"; +import { CommunityAdapter, CredentialCipher, RESERVED_KEYS } from "../src"; +import { HostedAdapter } from "@atmo-dev/contrail"; +import { buildSpaceUri } from "@atmo-dev/contrail"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; const CHARLIE = "did:plc:charlie"; const COMMUNITY_DID = "did:plc:pubcomm"; +const PROVISION_COMMUNITY_DID = "did:plc:provcomm"; +const PROVISION_HANDLE = "provcomm.pds.test"; const PDS_ENDPOINT = "https://pds.example"; const MASTER_KEY = new Uint8Array(32).fill(42); @@ -39,7 +44,9 @@ const CONFIG: ContrailConfig = { function mockResolver(): any { return { resolve: async (did: string) => { - if (did !== COMMUNITY_DID) throw new Error("unknown did"); + if (did !== COMMUNITY_DID && did !== PROVISION_COMMUNITY_DID) { + throw new Error("unknown did"); + } return { id: did, service: [ @@ -60,8 +67,12 @@ async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise< pdsCalls.push({ url, body }); if (url.endsWith("/xrpc/com.atproto.server.createSession")) { if (body.password === "correct-pw" || body.password === "new-correct-pw") { + // Echo back a DID that matches the identifier so adopt and provision flows + // both look right to any caller checking session.did. + const did = + body.identifier === PROVISION_HANDLE ? PROVISION_COMMUNITY_DID : COMMUNITY_DID; return new Response( - JSON.stringify({ accessJwt: "a.b.c", refreshJwt: "r.r.r", did: COMMUNITY_DID }), + JSON.stringify({ accessJwt: "a.b.c", refreshJwt: "r.r.r", did }), { status: 200, headers: { "content-type": "application/json" } } ); } @@ -70,7 +81,7 @@ async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise< if (url.endsWith("/xrpc/com.atproto.repo.createRecord")) { return new Response( JSON.stringify({ - uri: `at://${COMMUNITY_DID}/${body.collection}/fakerkey`, + uri: `at://${body.repo}/${body.collection}/fakerkey`, cid: "bafyfake", }), { status: 200, headers: { "content-type": "application/json" } } @@ -91,18 +102,63 @@ function fakeAuth(): MiddlewareHandler { }; } -async function makeApp(): Promise<Hono> { +async function makeApp(): Promise<{ app: Hono; db: Database }> { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); const community = createCommunityIntegration({ db, config: resolved }); await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); - return createApp(db, resolved, { + const app = createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, community, }); + return { app, db }; } -function call( +/** Seed a provision-mode community + its reserved spaces with `creator` as + * owner. Mirrors what the adopt/provision routes do via `bootstrapReservedSpaces`, + * but skips the route so we don't have to mock PLC + 5 PDS RPCs. */ +async function seedProvisionCommunity( + db: Database, + creator: string, + password: string +): Promise<void> { + const cipher = new CredentialCipher(MASTER_KEY); + const encrypted = await cipher.encrypt(password); + const community = new CommunityAdapter(db); + const spaces = new HostedAdapter(db, resolveConfig(CONFIG)); + await community.createFromProvisioned({ + did: PROVISION_COMMUNITY_DID, + pdsEndpoint: PDS_ENDPOINT, + handle: PROVISION_HANDLE, + appPasswordEncrypted: encrypted, + createdBy: creator, + }); + for (const key of RESERVED_KEYS) { + const uri = buildSpaceUri({ + ownerDid: PROVISION_COMMUNITY_DID, + type: CONFIG.spaces!.authority!.type, + key, + }); + await spaces.createSpace({ + uri, + ownerDid: PROVISION_COMMUNITY_DID, + type: CONFIG.spaces!.authority!.type, + key, + serviceDid: CONFIG.spaces!.authority!.serviceDid, + appPolicyRef: null, + appPolicy: null, + }); + await community.grant({ + spaceUri: uri, + subjectDid: creator, + accessLevel: "owner", + grantedBy: creator, + }); + await spaces.applyMembershipDiff(uri, [creator], [], creator); + } +} + +async function call( app: Hono, method: string, path: string, @@ -111,7 +167,7 @@ function call( ): Promise<Response> { const headers: Record<string, string> = { "X-Test-Did": did }; if (body !== undefined) headers["Content-Type"] = "application/json"; - return app.fetch( + return await app.fetch( new Request(`http://localhost${path}`, { method, headers, @@ -144,7 +200,7 @@ describe("community publishing + reauth — stage 3", () => { const admin = `ats://${COMMUNITY_DID}/tools.atmo.event.space/$admin`; beforeAll(async () => { - app = await makeApp(); + ({ app } = await makeApp()); await adopt(app, ALICE, "correct-pw"); }); @@ -273,3 +329,304 @@ describe("community publishing + reauth — stage 3", () => { expect(res.status).toBe(401); }); }); + +// Build a small base64url-encoded JWT with a given exp claim. The publishing +// path decodes payload.exp to decide whether to reuse a cached session. +function jwtWithExp(expSeconds: number): string { + // base64url("{}") padding stripped — header content irrelevant to our tests. + const header = "eyJhbGciOiJIUzI1NiJ9"; + const payloadJson = JSON.stringify({ exp: expSeconds }); + const payload = btoa(payloadJson) + .replace(/\+/g, "-") + .replace(/\//g, "_") + .replace(/=+$/, ""); + return `${header}.${payload}.sig`; +} + +/** Build an isolated app with a per-test scenario fetch + a fresh provision + * community. The scenario fetch records every call with its url + body + + * authorization header so individual tests can assert exact behavior. */ +async function makeScenarioApp(scenario: { + /** Override response for createSession. Default: success with default JWT. */ + onCreateSession?: () => Response; + /** Override response for refreshSession. Default: 400 (no refresh). */ + onRefreshSession?: () => Response; +}): Promise<{ + app: Hono; + db: Database; + calls: Array<{ url: string; body: any; authorization: string | null }>; +}> { + const calls: Array<{ url: string; body: any; authorization: string | null }> = []; + const scenarioFetch = async ( + input: RequestInfo | URL, + init?: RequestInit + ): Promise<Response> => { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + const body = init?.body ? JSON.parse(init.body as string) : {}; + const headers = new Headers((init?.headers as HeadersInit) ?? {}); + const authorization = headers.get("authorization"); + calls.push({ url, body, authorization }); + if (url.endsWith("/xrpc/com.atproto.server.createSession")) { + if (scenario.onCreateSession) return scenario.onCreateSession(); + return new Response( + JSON.stringify({ + accessJwt: jwtWithExp(Math.floor(Date.now() / 1000) + 3600), + refreshJwt: "r.r.r", + did: PROVISION_COMMUNITY_DID, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + if (url.endsWith("/xrpc/com.atproto.server.refreshSession")) { + if (scenario.onRefreshSession) return scenario.onRefreshSession(); + return new Response(JSON.stringify({ error: "ExpiredToken" }), { status: 400 }); + } + if (url.endsWith("/xrpc/com.atproto.repo.createRecord")) { + return new Response( + JSON.stringify({ + uri: `at://${body.repo}/${body.collection}/scenkey`, + cid: "bafyfake", + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + if (url.endsWith("/xrpc/com.atproto.repo.deleteRecord")) { + return new Response("{}", { + status: 200, + headers: { "content-type": "application/json" }, + }); + } + return new Response("not found", { status: 404 }); + }; + + const cfg: ContrailConfig = { + ...CONFIG, + community: { ...CONFIG.community!, fetch: scenarioFetch }, + }; + const db = createSqliteDatabase(":memory:"); + const resolved = resolveConfig(cfg); + const communityIntegration = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [communityIntegration.applySchema] }); + const app = createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community: communityIntegration, + }); + // Seed provision community + Alice as owner of $publishers. + const cipher = new CredentialCipher(MASTER_KEY); + const encrypted = await cipher.encrypt("correct-pw"); + const community = new CommunityAdapter(db); + const spacesAdp = new HostedAdapter(db, resolved); + await community.createFromProvisioned({ + did: PROVISION_COMMUNITY_DID, + pdsEndpoint: PDS_ENDPOINT, + handle: PROVISION_HANDLE, + appPasswordEncrypted: encrypted, + createdBy: ALICE, + }); + for (const key of RESERVED_KEYS) { + const uri = buildSpaceUri({ + ownerDid: PROVISION_COMMUNITY_DID, + type: CONFIG.spaces!.authority!.type, + key, + }); + await spacesAdp.createSpace({ + uri, + ownerDid: PROVISION_COMMUNITY_DID, + type: CONFIG.spaces!.authority!.type, + key, + serviceDid: CONFIG.spaces!.authority!.serviceDid, + appPolicyRef: null, + appPolicy: null, + }); + await community.grant({ + spaceUri: uri, + subjectDid: ALICE, + accessLevel: "owner", + grantedBy: ALICE, + }); + await spacesAdp.applyMembershipDiff(uri, [ALICE], [], ALICE); + } + return { app, db, calls }; +} + +describe("community publishing — session caching (Task 14)", () => { + it("caches PDS sessions across putRecord calls", async () => { + const { app, calls } = await makeScenarioApp({}); + + for (let i = 0; i < 3; i++) { + const res = await call(app, "POST", "/xrpc/test.comm.community.putRecord", ALICE, { + communityDid: PROVISION_COMMUNITY_DID, + collection: "app.event.message", + record: { text: `msg ${i}` }, + }); + expect(res.status).toBe(200); + } + + const createSessionCalls = calls.filter((c) => + c.url.endsWith("/xrpc/com.atproto.server.createSession") + ).length; + const createRecordCalls = calls.filter((c) => + c.url.endsWith("/xrpc/com.atproto.repo.createRecord") + ).length; + expect(createSessionCalls).toBe(1); + expect(createRecordCalls).toBe(3); + }); + + it("considers a session valid when accessExp is in the future", async () => { + const { app, db, calls } = await makeScenarioApp({}); + // Pre-seed cache with a clearly-future expiry. + const community = new CommunityAdapter(db); + const cachedAccess = jwtWithExp(Math.floor(Date.now() / 1000) + 3600); + await community.upsertSession(PROVISION_COMMUNITY_DID, { + accessJwt: cachedAccess, + refreshJwt: "cached-refresh", + accessExp: Math.floor(Date.now() / 1000) + 3600, + }); + + const res = await call(app, "POST", "/xrpc/test.comm.community.putRecord", ALICE, { + communityDid: PROVISION_COMMUNITY_DID, + collection: "app.event.message", + record: { text: "uses cached session" }, + }); + expect(res.status).toBe(200); + + const createSessionCalls = calls.filter((c) => + c.url.endsWith("/xrpc/com.atproto.server.createSession") + ).length; + expect(createSessionCalls).toBe(0); + // The createRecord call must have used the cached accessJwt. + const cr = calls.find((c) => c.url.endsWith("/xrpc/com.atproto.repo.createRecord")); + expect(cr).toBeDefined(); + expect(cr!.authorization).toBe(`Bearer ${cachedAccess}`); + }); + + it("refreshes a near-expired session via refreshSession", async () => { + const refreshedAccess = jwtWithExp(Math.floor(Date.now() / 1000) + 3600); + const { app, db, calls } = await makeScenarioApp({ + onRefreshSession: () => + new Response( + JSON.stringify({ accessJwt: refreshedAccess, refreshJwt: "new-refresh" }), + { status: 200, headers: { "content-type": "application/json" } } + ), + }); + const community = new CommunityAdapter(db); + await community.upsertSession(PROVISION_COMMUNITY_DID, { + accessJwt: jwtWithExp(Math.floor(Date.now() / 1000) - 60), + refreshJwt: "old-refresh", + accessExp: Math.floor(Date.now() / 1000) - 60, + }); + + const res = await call(app, "POST", "/xrpc/test.comm.community.putRecord", ALICE, { + communityDid: PROVISION_COMMUNITY_DID, + collection: "app.event.message", + record: { text: "after refresh" }, + }); + expect(res.status).toBe(200); + + const createSessionCalls = calls.filter((c) => + c.url.endsWith("/xrpc/com.atproto.server.createSession") + ).length; + const refreshSessionCalls = calls.filter((c) => + c.url.endsWith("/xrpc/com.atproto.server.refreshSession") + ).length; + expect(createSessionCalls).toBe(0); + expect(refreshSessionCalls).toBe(1); + // createRecord must use the refreshed access JWT. + const cr = calls.find((c) => c.url.endsWith("/xrpc/com.atproto.repo.createRecord")); + expect(cr!.authorization).toBe(`Bearer ${refreshedAccess}`); + }); + + it("falls back to createSession when refresh fails", async () => { + const { app, db, calls } = await makeScenarioApp({ + onRefreshSession: () => + new Response(JSON.stringify({ error: "ExpiredToken" }), { status: 400 }), + }); + const community = new CommunityAdapter(db); + await community.upsertSession(PROVISION_COMMUNITY_DID, { + accessJwt: jwtWithExp(Math.floor(Date.now() / 1000) - 60), + refreshJwt: "stale-refresh", + accessExp: Math.floor(Date.now() / 1000) - 60, + }); + + const res = await call(app, "POST", "/xrpc/test.comm.community.putRecord", ALICE, { + communityDid: PROVISION_COMMUNITY_DID, + collection: "app.event.message", + record: { text: "fallback to create" }, + }); + expect(res.status).toBe(200); + + const createSessionCalls = calls.filter((c) => + c.url.endsWith("/xrpc/com.atproto.server.createSession") + ).length; + const refreshSessionCalls = calls.filter((c) => + c.url.endsWith("/xrpc/com.atproto.server.refreshSession") + ).length; + expect(refreshSessionCalls).toBe(1); + expect(createSessionCalls).toBe(1); + }); +}); + +describe("community publishing — provision mode", () => { + let app: Hono; + + beforeAll(async () => { + const built = await makeApp(); + app = built.app; + await seedProvisionCommunity(built.db, ALICE, "correct-pw"); + }); + + it("publishes a record under a provision-mode community", async () => { + const before = pdsCalls.length; + const res = await call(app, "POST", "/xrpc/test.comm.community.putRecord", ALICE, { + communityDid: PROVISION_COMMUNITY_DID, + collection: "app.event.message", + record: { text: "hello from a provisioned community" }, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.uri).toBe(`at://${PROVISION_COMMUNITY_DID}/app.event.message/fakerkey`); + + const newCalls = pdsCalls.slice(before); + expect( + newCalls.some( + (c) => + c.url.endsWith("/xrpc/com.atproto.server.createSession") && + c.body.identifier === PROVISION_HANDLE + ) + ).toBe(true); + expect( + newCalls.some( + (c) => + c.url.endsWith("/xrpc/com.atproto.repo.createRecord") && + c.body.repo === PROVISION_COMMUNITY_DID + ) + ).toBe(true); + }); + + it("deletes a record under a provision-mode community", async () => { + const before = pdsCalls.length; + const res = await call(app, "POST", "/xrpc/test.comm.community.deleteRecord", ALICE, { + communityDid: PROVISION_COMMUNITY_DID, + collection: "app.event.message", + rkey: "fakerkey", + }); + expect(res.status).toBe(200); + expect(((await res.json()) as any).ok).toBe(true); + + const newCalls = pdsCalls.slice(before); + expect( + newCalls.some((c) => c.url.endsWith("/xrpc/com.atproto.repo.deleteRecord")) + ).toBe(true); + }); + + it("reports healthy for a provision-mode community", async () => { + const res = await call( + app, + "GET", + `/xrpc/test.comm.community.getHealth?communityDid=${PROVISION_COMMUNITY_DID}`, + ALICE + ); + expect(res.status).toBe(200); + expect(((await res.json()) as any).status).toBe("healthy"); + }); +}); diff --git a/packages/contrail-community/tests/community-sessions-cache.test.ts b/packages/contrail-community/tests/community-sessions-cache.test.ts new file mode 100644 index 0000000..6ead80c --- /dev/null +++ b/packages/contrail-community/tests/community-sessions-cache.test.ts @@ -0,0 +1,85 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { initCommunitySchema } from "../src/schema"; +import { CommunityAdapter } from "../src/adapter"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; + +describe("community_sessions cache", () => { + let adapter: CommunityAdapter; + + beforeEach(async () => { + const db = createSqliteDatabase(":memory:"); + await initCommunitySchema(db); + adapter = new CommunityAdapter(db); + }); + + it("upserts and reads a cached session", async () => { + await adapter.upsertSession("did:plc:x", { + accessJwt: "atok", + refreshJwt: "rtok", + accessExp: 1234, + }); + const got = await adapter.getSession("did:plc:x"); + expect(got).toEqual({ accessJwt: "atok", refreshJwt: "rtok", accessExp: 1234 }); + }); + + it("returns null for missing did", async () => { + const got = await adapter.getSession("did:plc:nope"); + expect(got).toBeNull(); + }); + + it("clears a session", async () => { + await adapter.upsertSession("did:plc:x", { + accessJwt: "a", + refreshJwt: "r", + accessExp: 1, + }); + await adapter.clearSession("did:plc:x"); + expect(await adapter.getSession("did:plc:x")).toBeNull(); + }); + + it("upsert overwrites existing session for the same did", async () => { + await adapter.upsertSession("did:plc:x", { + accessJwt: "old-a", + refreshJwt: "old-r", + accessExp: 100, + }); + await adapter.upsertSession("did:plc:x", { + accessJwt: "new-a", + refreshJwt: "new-r", + accessExp: 200, + }); + const got = await adapter.getSession("did:plc:x"); + expect(got).toEqual({ accessJwt: "new-a", refreshJwt: "new-r", accessExp: 200 }); + }); + + it("isolates sessions across communities", async () => { + await adapter.upsertSession("did:plc:a", { + accessJwt: "a-tok", + refreshJwt: "a-rtok", + accessExp: 1, + }); + await adapter.upsertSession("did:plc:b", { + accessJwt: "b-tok", + refreshJwt: "b-rtok", + accessExp: 2, + }); + expect(await adapter.getSession("did:plc:a")).toEqual({ + accessJwt: "a-tok", + refreshJwt: "a-rtok", + accessExp: 1, + }); + expect(await adapter.getSession("did:plc:b")).toEqual({ + accessJwt: "b-tok", + refreshJwt: "b-rtok", + accessExp: 2, + }); + await adapter.clearSession("did:plc:a"); + expect(await adapter.getSession("did:plc:a")).toBeNull(); + // Clearing one DID must not affect the other. + expect(await adapter.getSession("did:plc:b")).toEqual({ + accessJwt: "b-tok", + refreshJwt: "b-rtok", + accessExp: 2, + }); + }); +}); diff --git a/packages/contrail-community/tests/pds-account-ops.test.ts b/packages/contrail-community/tests/pds-account-ops.test.ts new file mode 100644 index 0000000..348793b --- /dev/null +++ b/packages/contrail-community/tests/pds-account-ops.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from "vitest"; +import { + pdsGetRecommendedDidCredentials, + pdsActivateAccount, +} from "../src/pds"; + +describe("pdsGetRecommendedDidCredentials", () => { + it("issues GET to the identity endpoint with bearer accessJwt and parses response", async () => { + let received: { url: string; init: any } | null = null; + const fetch = (async (url: string, init: any) => { + received = { url, init }; + return new Response( + JSON.stringify({ + rotationKeys: ["did:key:zRot"], + verificationMethods: { atproto: "did:key:zSig" }, + alsoKnownAs: ["at://h.test"], + services: { + atproto_pds: { type: "AtprotoPersonalDataServer", endpoint: "https://pds.test" }, + }, + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }) as unknown as typeof globalThis.fetch; + + const result = await pdsGetRecommendedDidCredentials( + "https://pds.test", + "AT", + { fetch } + ); + + expect(received!.url).toBe( + "https://pds.test/xrpc/com.atproto.identity.getRecommendedDidCredentials" + ); + // Default fetch method is GET when none specified. + expect(received!.init?.method ?? "GET").toBe("GET"); + // Bearer is the session accessJwt, NOT a service-auth JWT. + expect(received!.init.headers.authorization).toBe("Bearer AT"); + expect(result.rotationKeys).toEqual(["did:key:zRot"]); + expect(result.verificationMethods).toEqual({ atproto: "did:key:zSig" }); + expect(result.alsoKnownAs).toEqual(["at://h.test"]); + expect(result.services).toEqual({ + atproto_pds: { type: "AtprotoPersonalDataServer", endpoint: "https://pds.test" }, + }); + }); + + it("throws with status and body on non-2xx", async () => { + const fetch = (async () => + new Response("session expired", { status: 401 })) as any; + await expect( + pdsGetRecommendedDidCredentials("https://pds.test", "AT", { fetch }) + ).rejects.toThrow(/getRecommendedDidCredentials failed.*401.*session expired/); + }); +}); + +describe("pdsActivateAccount", () => { + it("issues POST to activateAccount with bearer accessJwt and resolves to undefined", async () => { + let received: { url: string; init: any } | null = null; + const fetch = (async (url: string, init: any) => { + received = { url, init }; + return new Response("", { status: 200 }); + }) as unknown as typeof globalThis.fetch; + + const result = await pdsActivateAccount("https://pds.test", "AT", { fetch }); + + expect(received!.url).toBe( + "https://pds.test/xrpc/com.atproto.server.activateAccount" + ); + expect(received!.init.method).toBe("POST"); + // Bearer is the session accessJwt from pdsCreateAccount, NOT a service-auth JWT. + expect(received!.init.headers.authorization).toBe("Bearer AT"); + expect(result).toBeUndefined(); + }); + + it("throws with status and body on non-2xx", async () => { + const fetch = (async () => + new Response("nope", { status: 400 })) as any; + await expect( + pdsActivateAccount("https://pds.test", "AT", { fetch }) + ).rejects.toThrow(/activateAccount failed.*400.*nope/); + }); +}); diff --git a/packages/contrail-community/tests/pds-create-account.test.ts b/packages/contrail-community/tests/pds-create-account.test.ts new file mode 100644 index 0000000..33ebcd7 --- /dev/null +++ b/packages/contrail-community/tests/pds-create-account.test.ts @@ -0,0 +1,74 @@ +import { describe, it, expect } from "vitest"; +import { pdsCreateAccount } from "../src/pds"; + +describe("pdsCreateAccount", () => { + it("posts createAccount with bearer auth and returns session", async () => { + let received: { url: string; init: any } | null = null; + const fetch = (async (url: string, init: any) => { + received = { url, init }; + return new Response( + JSON.stringify({ + accessJwt: "AT", refreshJwt: "RT", handle: "h.test", did: "did:plc:x", + }), + { status: 200, headers: { "content-type": "application/json" } } + ); + }) as unknown as typeof globalThis.fetch; + + const result = await pdsCreateAccount( + "https://pds.test", + "JWT-VALUE", + { + handle: "h.test", + did: "did:plc:x", + email: "h@x.test", + password: "p", + inviteCode: "code", + }, + { fetch } + ); + + expect(received!.url).toBe("https://pds.test/xrpc/com.atproto.server.createAccount"); + expect(received!.init.method).toBe("POST"); + expect(received!.init.headers.authorization).toBe("Bearer JWT-VALUE"); + expect(JSON.parse(received!.init.body)).toEqual({ + handle: "h.test", + did: "did:plc:x", + email: "h@x.test", + password: "p", + inviteCode: "code", + }); + expect(result.accessJwt).toBe("AT"); + expect(result.did).toBe("did:plc:x"); + }); + + it("strips trailing slash from pdsEndpoint", async () => { + let receivedUrl = ""; + const fetch = (async (url: string) => { + receivedUrl = url; + return new Response( + JSON.stringify({ accessJwt: "AT", refreshJwt: "RT", handle: "h", did: "did:plc:x" }), + { status: 200 } + ); + }) as any; + await pdsCreateAccount( + "https://pds.test/", + "JWT", + { handle: "h", did: "did:plc:x", email: "e", password: "p" }, + { fetch } + ); + expect(receivedUrl).toBe("https://pds.test/xrpc/com.atproto.server.createAccount"); + }); + + it("throws on non-2xx", async () => { + const fetch = (async () => + new Response(JSON.stringify({ error: "InvalidRequest", message: "bad" }), { status: 400 })) as any; + await expect( + pdsCreateAccount( + "https://pds.test", + "x", + { handle: "h", did: "did:plc:x", email: "e", password: "p" }, + { fetch } + ) + ).rejects.toThrow(/createAccount failed.*400.*InvalidRequest/); + }); +}); diff --git a/packages/contrail-community/tests/plc-log-last.test.ts b/packages/contrail-community/tests/plc-log-last.test.ts new file mode 100644 index 0000000..975ede7 --- /dev/null +++ b/packages/contrail-community/tests/plc-log-last.test.ts @@ -0,0 +1,81 @@ +import { describe, it, expect } from "vitest"; +import { + cidForOp, + getLastOpCid, + type SignedGenesisOp, + type SignedTombstoneOp, +} from "../src/plc"; + +const GENESIS_OP: SignedGenesisOp = { + type: "plc_operation", + prev: null, + rotationKeys: ["did:key:zQ3shjNSBChNYuYsW41QDdm2D25zmQkdpfhgbaQBRG4ecg7sk"], + verificationMethods: { + atproto: "did:key:zQ3shmefuqey6KqP7M9cwFwywqTVuCZFXcCAGJ5JGktdAUdD2", + }, + alsoKnownAs: ["at://probe.devnet.test"], + services: { + atproto_pds: { + type: "AtprotoPersonalDataServer", + endpoint: "https://devnet.test", + }, + }, + sig: "xEZ7BS7bXJ-7KqExTH158uJFNhcTi21khw-rCHjt70EwGVhftk29Xjf1IR9JGhSmDPE76Xqc01ydF9TmmPHr2w", +}; + +const TOMBSTONE_OP: SignedTombstoneOp = { + type: "plc_tombstone", + prev: "bafyreiabmto3hekxoflemevicopvpud2k6ypf2fkp3v3g6iu36l4wxxfle", + sig: "abc123", +}; + +describe("getLastOpCid", () => { + it("returns the CID computed locally from the PLC log/last op response", async () => { + let calledUrl = ""; + const fakeFetch: typeof fetch = async (input) => { + calledUrl = String(input); + return new Response(JSON.stringify(GENESIS_OP), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }; + const cid = await getLastOpCid("https://plc.test", "did:plc:abc", { + fetch: fakeFetch, + }); + expect(calledUrl).toBe("https://plc.test/did:plc:abc/log/last"); + // PLC returns the bare op (no envelope). The function must compute the + // CID with the same DAG-CBOR encoder cidForOp uses so the value matches + // the CID PLC stored when it accepted the op. + expect(cid).toBe(await cidForOp(GENESIS_OP)); + }); + + it("computes the CID from a tombstone op response too", async () => { + const fakeFetch: typeof fetch = async () => + new Response(JSON.stringify(TOMBSTONE_OP), { + status: 200, + headers: { "content-type": "application/json" }, + }); + const cid = await getLastOpCid("https://plc.test", "did:plc:abc", { + fetch: fakeFetch, + }); + expect(cid).toBe(await cidForOp(TOMBSTONE_OP)); + }); + + it("strips a trailing slash from the directory base", async () => { + let calledUrl = ""; + const fakeFetch: typeof fetch = async (input) => { + calledUrl = String(input); + return new Response(JSON.stringify(GENESIS_OP), { status: 200 }); + }; + await getLastOpCid("https://plc.test/", "did:plc:xyz", { fetch: fakeFetch }); + expect(calledUrl).toBe("https://plc.test/did:plc:xyz/log/last"); + }); + + it("throws on a non-200 response, including the status and body", async () => { + const fakeFetch: typeof fetch = async () => + new Response("not found", { status: 404 }); + await expect( + getLastOpCid("https://plc.test", "did:plc:missing", { fetch: fakeFetch }) + ).rejects.toThrow(/404.*not found/); + }); +}); diff --git a/packages/contrail-community/tests/plc-update-op.test.ts b/packages/contrail-community/tests/plc-update-op.test.ts new file mode 100644 index 0000000..5f3b765 --- /dev/null +++ b/packages/contrail-community/tests/plc-update-op.test.ts @@ -0,0 +1,50 @@ +import { describe, it, expect } from "vitest"; +import { + generateKeyPair, + buildGenesisOp, + signGenesisOp, + buildUpdateOp, + signUpdateOp, + cidForOp, +} from "../src/plc"; + +describe("cidForOp", () => { + it("produces a CIDv1 dag-cbor sha256 base32-lower CID starting with bafyrei", async () => { + const kp = await generateKeyPair(); + const unsigned = buildGenesisOp({ + rotationKeys: [kp.publicDidKey], + verificationMethodAtproto: kp.publicDidKey, + alsoKnownAs: ["at://x.test"], + services: { atproto_pds: { type: "AtprotoPersonalDataServer", endpoint: "https://x.test" } }, + }); + const signed = await signGenesisOp(unsigned, kp.privateJwk); + const cid = await cidForOp(signed); + expect(cid).toMatch(/^bafyrei/); + expect(cid.length).toBeGreaterThan(50); + }); +}); + +describe("buildUpdateOp + signUpdateOp", () => { + it("produces a plc_operation with prev set and a sig segment", async () => { + const kp = await generateKeyPair(); + const genesis = buildGenesisOp({ + rotationKeys: [kp.publicDidKey], + verificationMethodAtproto: kp.publicDidKey, + alsoKnownAs: ["at://x.test"], + services: { atproto_pds: { type: "AtprotoPersonalDataServer", endpoint: "https://x.test" } }, + }); + const signedGenesis = await signGenesisOp(genesis, kp.privateJwk); + const prev = await cidForOp(signedGenesis); + + const update = buildUpdateOp({ + prev, + rotationKeys: [kp.publicDidKey, "did:key:zPdsRot"], + verificationMethodAtproto: "did:key:zPdsSig", + alsoKnownAs: ["at://x.test"], + services: { atproto_pds: { type: "AtprotoPersonalDataServer", endpoint: "https://x.test" } }, + }); + + const signedUpdate = await signUpdateOp(update, kp.privateJwk); + expect(signedUpdate.sig).toMatch(/^[A-Za-z0-9_-]+$/); + }); +}); diff --git a/packages/contrail-community/tests/provision-orchestrator.test.ts b/packages/contrail-community/tests/provision-orchestrator.test.ts new file mode 100644 index 0000000..9dd832c --- /dev/null +++ b/packages/contrail-community/tests/provision-orchestrator.test.ts @@ -0,0 +1,259 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { initCommunitySchema } from "../src/schema"; +import { CommunityAdapter } from "../src/adapter"; +import { CredentialCipher } from "../src/credentials"; +import { ProvisionOrchestrator } from "../src/provision"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; + +const STUB_ROTATION_KEY = "did:key:zStubCallerRotationKeyForTests"; + +function mockPlc() { + const ops: any[] = []; + return { + ops, + async submit(did: string, op: any) { + ops.push({ did, op }); + return { ok: true }; + }, + }; +} + +function mockPds() { + return { + async createAccount() { + return { + did: "did:plc:x", + handle: "h.test", + accessJwt: "AT", + refreshJwt: "RT", + }; + }, + async getRecommendedDidCredentials() { + return { + rotationKeys: ["did:key:zPdsRot"], + verificationMethods: { atproto: "did:key:zPdsSig" }, + alsoKnownAs: ["at://h.test"], + services: { + atproto_pds: { + type: "AtprotoPersonalDataServer", + endpoint: "https://pds.test", + }, + }, + }; + }, + async activateAccount() { + return; + }, + async createAppPassword() { + return { password: "minted-app-pw" }; + }, + }; +} + +describe("ProvisionOrchestrator", () => { + let adapter: CommunityAdapter; + let cipher: CredentialCipher; + beforeEach(async () => { + const db = createSqliteDatabase(":memory:"); + await initCommunitySchema(db); + cipher = new CredentialCipher(new Uint8Array(32).fill(99)); + adapter = new CommunityAdapter(db); + }); + + it("runs end-to-end and lands at status=activated", async () => { + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: mockPlc(), + pds: mockPds(), + pdsDid: "did:web:pds.test", + }); + + const result = await orch.provision({ + attemptId: "a1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "p", + inviteCode: "code", + rotationKey: STUB_ROTATION_KEY, + }); + + expect(result.did).toBeTruthy(); + expect(result.status).toBe("activated"); + const row = await adapter.getProvisionAttempt("a1"); + expect(row?.status).toBe("activated"); + expect(row?.encryptedSigningKey).toBeTruthy(); + expect(row?.encryptedRotationKey).toBeTruthy(); + expect(row?.encryptedPassword).toBeTruthy(); + }); + + it("seeds the community_sessions cache with the createAccount JWTs after activation", async () => { + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: mockPlc(), + pds: mockPds(), + pdsDid: "did:web:pds.test", + }); + + const result = await orch.provision({ + attemptId: "a1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "p", + inviteCode: "code", + rotationKey: STUB_ROTATION_KEY, + }); + + const cached = await adapter.getSession(result.did); + expect(cached).not.toBeNull(); + expect(cached?.accessJwt).toBe("AT"); + expect(cached?.refreshJwt).toBe("RT"); + }); + + it("persists status=genesis_submitted before createAccount runs", async () => { + let createCalled = false; + const pds = { + async createAccount() { + // Inspect state at this exact moment. + const row = await adapter.getProvisionAttempt("a1"); + expect(row?.status).toBe("genesis_submitted"); + createCalled = true; + return { + did: "did:plc:x", + handle: "h.test", + accessJwt: "AT", + refreshJwt: "RT", + }; + }, + async getRecommendedDidCredentials() { + return { + rotationKeys: [], + verificationMethods: { atproto: "did:key:zSig" }, + alsoKnownAs: ["at://h.test"], + services: { + atproto_pds: { type: "x", endpoint: "https://pds.test" }, + }, + }; + }, + async activateAccount() {}, + async createAppPassword() { + return { password: "minted" }; + }, + }; + + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: mockPlc(), + pds, + pdsDid: "did:web:pds.test", + }); + await orch.provision({ + attemptId: "a1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "p", + rotationKey: STUB_ROTATION_KEY, + }); + expect(createCalled).toBe(true); + }); + + it("marks last_error and rethrows when createAccount fails", async () => { + const pds = { + ...mockPds(), + async createAccount() { + throw new Error("createAccount 400: bad invite"); + }, + } as any; + + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: mockPlc(), + pds, + pdsDid: "did:web:pds.test", + }); + + await expect( + orch.provision({ + attemptId: "a1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "p", + rotationKey: STUB_ROTATION_KEY, + }) + ).rejects.toThrow(/bad invite/); + const row = await adapter.getProvisionAttempt("a1"); + expect(row?.status).toBe("genesis_submitted"); // last successful step + expect(row?.lastError).toMatch(/bad invite/); + }); + + it("re-invoking with the same attemptId on a fully-completed row returns success without redoing PLC/PDS work", async () => { + // Scenario: the orchestrator finished cleanly (status=activated + + // encryptedPassword set), but a *downstream* step (router's + // createFromProvisioned or bootstrapReservedSpaces) failed and the + // caller retries with the same attemptId. The orchestrator must not + // throw "already exists" — it should report success so the route can + // resume the graduation steps. + const plc = mockPlc(); + const pds: any = mockPds(); + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc, + pds, + pdsDid: "did:web:pds.test", + }); + + const first = await orch.provision({ + attemptId: "a1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "p", + inviteCode: "code", + rotationKey: STUB_ROTATION_KEY, + }); + expect(first.status).toBe("activated"); + const opsAfterFirst = plc.ops.length; + + // Wire a fresh PDS mock whose every method throws — if the retry path + // calls any of them, the test fails loudly. createSession is allowed + // because the C3 retry path is wired for the not-yet-completed case; + // a fully-completed row should NOT hit it either. + const explodingPds: any = { + createAccount: () => { throw new Error("createAccount should not be called on a completed retry"); }, + getRecommendedDidCredentials: () => { throw new Error("getRecommendedDidCredentials should not be called"); }, + activateAccount: () => { throw new Error("activateAccount should not be called"); }, + createAppPassword: () => { throw new Error("createAppPassword should not be called on a completed retry"); }, + createSession: () => { throw new Error("createSession should not be called on a completed retry"); }, + }; + const retryOrch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: { submit: () => { throw new Error("plc.submit should not be called on a completed retry"); } }, + pds: explodingPds, + pdsDid: "did:web:pds.test", + }); + + const second = await retryOrch.provision({ + attemptId: "a1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "p", + inviteCode: "code", + rotationKey: STUB_ROTATION_KEY, + }); + expect(second.status).toBe("activated"); + expect(second.did).toBe(first.did); + expect(second.attemptId).toBe("a1"); + expect(plc.ops.length).toBe(opsAfterFirst); + }); + +}); diff --git a/packages/contrail-community/tests/provision-self-sovereign.test.ts b/packages/contrail-community/tests/provision-self-sovereign.test.ts new file mode 100644 index 0000000..df41bbe --- /dev/null +++ b/packages/contrail-community/tests/provision-self-sovereign.test.ts @@ -0,0 +1,353 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { initCommunitySchema } from "../src/schema"; +import { CommunityAdapter } from "../src/adapter"; +import { CredentialCipher } from "../src/credentials"; +import { ProvisionOrchestrator } from "../src/provision"; +import { generateKeyPair } from "../src/plc"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; + +/** Mock PLC client that records every submitted op so tests can inspect the + * genesis op (in particular, its rotationKeys array). */ +function mockPlc() { + const ops: Array<{ did: string; op: any }> = []; + return { + ops, + async submit(did: string, op: any) { + ops.push({ did, op }); + return { ok: true }; + }, + }; +} + +/** Mock PDS client that records calls to createAppPassword so tests can assert + * on its arguments (or its absence). The minted password is deterministic so + * decryption assertions can compare. */ +function mockPds(opts: { mintedPassword?: string } = {}) { + const calls: { createAppPassword: Array<{ pdsUrl: string; accessJwt: string; name: string }> } = { + createAppPassword: [], + }; + return { + calls, + async createAccount() { + return { + did: "did:plc:x", + handle: "h.test", + accessJwt: "AT", + refreshJwt: "RT", + }; + }, + async getRecommendedDidCredentials() { + return { + rotationKeys: ["did:key:zPdsRot"], + verificationMethods: { atproto: "did:key:zPdsSig" }, + alsoKnownAs: ["at://h.test"], + services: { + atproto_pds: { + type: "AtprotoPersonalDataServer", + endpoint: "https://pds.test", + }, + }, + }; + }, + async activateAccount() { + return; + }, + async createAppPassword(input: { pdsUrl: string; accessJwt: string; name: string }) { + calls.createAppPassword.push(input); + return { password: opts.mintedPassword ?? "minted-app-pass-XXXX" }; + }, + }; +} + +describe("ProvisionOrchestrator — caller-supplied rotation key", () => { + let adapter: CommunityAdapter; + let cipher: CredentialCipher; + beforeEach(async () => { + const db = createSqliteDatabase(":memory:"); + await initCommunitySchema(db); + cipher = new CredentialCipher(new Uint8Array(32).fill(99)); + adapter = new CommunityAdapter(db); + }); + + it("genesis includes caller rotation key, mints app password, response carries rootCredentials", async () => { + const callerKeyPair = await generateKeyPair(); + const callerRotationDidKey = callerKeyPair.publicDidKey; + const userPassword = "user-supplied-root-pw"; + const mintedPassword = "minted-app-pw-1234"; + + const plc = mockPlc(); + const pds = mockPds({ mintedPassword }); + + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc, + pds, + pdsDid: "did:web:pds.test", + }); + + const result = await orch.provision({ + attemptId: "ss1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: userPassword, + inviteCode: "code", + rotationKey: callerRotationDidKey, + }); + + // Status unchanged in shape. + expect(result.status).toBe("activated"); + expect(result.did).toBeTruthy(); + + // Response carries root credentials so the caller can keep their root password. + expect(result.rootCredentials).toBeDefined(); + expect(result.rootCredentials!.password).toBe(userPassword); + expect(result.rootCredentials!.handle).toBe("h.test"); + expect(typeof result.rootCredentials!.recoveryHint).toBe("string"); + + // Persisted attempt row reaches activated status. + const row = await adapter.getProvisionAttempt("ss1"); + expect(row).toBeTruthy(); + expect(row!.status).toBe("activated"); + + // Genesis op submitted to PLC has BOTH rotation keys, with the caller's first. + expect(plc.ops.length).toBeGreaterThanOrEqual(1); + const genesis = plc.ops[0]!.op; + expect(Array.isArray(genesis.rotationKeys)).toBe(true); + expect(genesis.rotationKeys[0]).toBe(callerRotationDidKey); + expect(genesis.rotationKeys.length).toBe(2); + expect(genesis.rotationKeys[1]).toBeTruthy(); + expect(genesis.rotationKeys[1]).not.toBe(callerRotationDidKey); + + // createAppPassword was invoked post-activation with the session's accessJwt. + expect(pds.calls.createAppPassword.length).toBe(1); + const apCall = pds.calls.createAppPassword[0]!; + expect(apCall.pdsUrl).toBe("https://pds.test"); + expect(apCall.accessJwt).toBe("AT"); + expect(apCall.name).toContain("ss1"); + + // encrypted_password column re-decrypts to the MINTED app password, + // not the user's supplied password. + expect(row!.encryptedPassword).toBeTruthy(); + const decryptedPw = await cipher.decryptString(row!.encryptedPassword!); + expect(decryptedPw).toBe(mintedPassword); + expect(decryptedPw).not.toBe(userPassword); + + // Subordinate rotation private JWK persisted in encrypted_rotation_key + // must NOT decrypt to anything containing the caller's did:key fingerprint. + expect(row!.encryptedRotationKey).toBeTruthy(); + const decryptedRot = await cipher.decryptString(row!.encryptedRotationKey!); + expect(decryptedRot).not.toContain(callerRotationDidKey); + + // Negative invariant: caller's did:key must not appear in any encrypted + // column (after decryption). + const encryptedSigning = row!.encryptedSigningKey; + if (encryptedSigning) { + const decryptedSig = await cipher.decryptString(encryptedSigning); + expect(decryptedSig).not.toContain(callerRotationDidKey); + } + }); + + it("PLC update op preserves caller's rotation key at index 0", async () => { + // H2 regression guard. The update op (plc.ops[1]) must keep the caller's + // did:key as rotationKeys[0]. Without threading it through + // runUpdateAndActivate, the caller's key is dropped and Contrail's + // subordinate becomes the highest-priority rotation key — caller has 72h + // to nullify before losing rotation authority on a DID they own. + const callerKeyPair = await generateKeyPair(); + const callerRotationDidKey = callerKeyPair.publicDidKey; + + const plc = mockPlc(); + const pds = mockPds(); + + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc, + pds, + pdsDid: "did:web:pds.test", + }); + + await orch.provision({ + attemptId: "ss-update", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "pw", + inviteCode: "code", + rotationKey: callerRotationDidKey, + }); + + // Genesis op already asserted in the prior test; here we focus on the update op. + expect(plc.ops.length).toBeGreaterThanOrEqual(2); + const update = plc.ops[1]!.op; + expect(Array.isArray(update.rotationKeys)).toBe(true); + expect(update.rotationKeys[0]).toBe(callerRotationDidKey); + // Contrail's subordinate must remain in the chain (we still need to sign + // future update ops). + expect(update.rotationKeys.length).toBeGreaterThanOrEqual(2); + expect(update.rotationKeys.slice(1)).not.toContain(callerRotationDidKey); + // PDS-recommended key is merged in after the contrail subordinate. + expect(update.rotationKeys).toContain("did:key:zPdsRot"); + }); + + it("createAppPassword failure persists last_error at status=activated and throws (no encryptedPassword)", async () => { + const callerKeyPair = await generateKeyPair(); + const plc = mockPlc(); + const pds = { + ...mockPds(), + async createAppPassword(_: any) { + throw new Error("PDS rejected: rate limited"); + }, + }; + + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc, + pds: pds as any, + pdsDid: "did:web:pds.test", + }); + + await expect( + orch.provision({ + attemptId: "ss-fail", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "pw", + rotationKey: callerKeyPair.publicDidKey, + }) + ).rejects.toThrow(/createAppPassword/); + + const row = await adapter.getProvisionAttempt("ss-fail"); + expect(row).toBeTruthy(); + expect(row!.status).toBe("activated"); + expect(row!.encryptedPassword).toBeFalsy(); + expect(row!.lastError).toMatch(/createAppPassword/); + }); + + it("retry with same attemptId after createAppPassword failure picks up at createAppPassword (no re-mint, no re-createAccount)", async () => { + // Simulate the failure-then-retry shape: a first provision call got all + // the way to createAppPassword and failed; the caller retries with the + // same attemptId. The orchestrator must NOT re-submit PLC ops, NOT + // re-call createAccount, only run createAppPassword. + const callerKeyPair = await generateKeyPair(); + const callerDidKey = callerKeyPair.publicDidKey; + const mintedPassword = "minted-on-retry-99"; + + // First attempt: createAppPassword throws; everything else succeeds. + const firstPds = { + ...mockPds(), + async createAppPassword(_: any) { + throw new Error("PDS transient: 503"); + }, + }; + const firstPlc = mockPlc(); + const firstOrch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: firstPlc, + pds: firstPlc && (firstPds as any), + pdsDid: "did:web:pds.test", + }); + await expect( + firstOrch.provision({ + attemptId: "ss-retry", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "user-root-pw", + rotationKey: callerDidKey, + }) + ).rejects.toThrow(); + + // Sanity: row is in the failure state we expect. + const failRow = await adapter.getProvisionAttempt("ss-retry"); + expect(failRow!.status).toBe("activated"); + expect(failRow!.encryptedPassword).toBeFalsy(); + expect(firstPlc.ops.length).toBe(2); // genesis + update + + // Second attempt with the SAME attemptId. Use a fresh mock that would + // EXPLODE if createAccount or any PLC op was re-issued. + const retryPlc = { + ops: [] as Array<{ did: string; op: any }>, + async submit(_did: string, _op: any) { + throw new Error("retry must not re-submit PLC ops"); + }, + }; + const retryAppPasswordCalls: any[] = []; + const retryPds = { + async createAccount() { + throw new Error("retry must not re-call createAccount"); + }, + async getRecommendedDidCredentials() { + throw new Error("retry must not re-fetch recommended creds"); + }, + async activateAccount() { + throw new Error("retry must not re-activate"); + }, + async createAppPassword(input: any) { + retryAppPasswordCalls.push(input); + return { password: mintedPassword }; + }, + async createSession(input: { pdsUrl: string; identifier: string; password: string }) { + // Verify the retry uses the user's root password to obtain a fresh + // accessJwt (the cached one from the failed attempt may have expired). + expect(input.password).toBe("user-root-pw"); + return { accessJwt: "AT-fresh", refreshJwt: "RT-fresh", did: "did:plc:x" }; + }, + }; + const retryOrch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: retryPlc as any, + pds: retryPds as any, + pdsDid: "did:web:pds.test", + }); + + const result = await retryOrch.provision({ + attemptId: "ss-retry", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "user-root-pw", + rotationKey: callerDidKey, + }); + + // Retry succeeded. + expect(result.status).toBe("activated"); + expect(result.did).toBe(failRow!.did); // SAME DID, not a new one + expect(result.rootCredentials).toBeDefined(); + expect(retryAppPasswordCalls.length).toBe(1); + + // Row now has the encrypted (minted) password. + const finalRow = await adapter.getProvisionAttempt("ss-retry"); + expect(finalRow!.status).toBe("activated"); + expect(finalRow!.encryptedPassword).toBeTruthy(); + const decrypted = await cipher.decryptString(finalRow!.encryptedPassword!); + expect(decrypted).toBe(mintedPassword); + }); + + it("rejects rotationKey that is not did:key:z…", async () => { + const orch = new ProvisionOrchestrator({ + adapter, + cipher, + plc: mockPlc(), + pds: mockPds(), + pdsDid: "did:web:pds.test", + }); + + await expect( + orch.provision({ + attemptId: "bad1", + pdsEndpoint: "https://pds.test", + handle: "h.test", + email: "h@x.test", + password: "p", + rotationKey: "not-a-did-key", + }) + ).rejects.toThrow(/rotationKey/); + }); +}); diff --git a/packages/contrail-community/tests/schema.test.ts b/packages/contrail-community/tests/schema.test.ts new file mode 100644 index 0000000..e5e52f5 --- /dev/null +++ b/packages/contrail-community/tests/schema.test.ts @@ -0,0 +1,18 @@ +import { describe, it, expect } from "vitest"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initCommunitySchema } from "../src/schema"; + +describe("provision_attempts schema", () => { + it("enforces status enum", async () => { + const db = createSqliteDatabase(":memory:"); + await initCommunitySchema(db); + await expect( + db + .prepare( + "INSERT INTO provision_attempts (attempt_id, did, status, created_at, updated_at, pds_endpoint, handle, email) VALUES (?, ?, ?, ?, ?, ?, ?, ?)" + ) + .bind("a1", "did:plc:x", "bogus", 1, 1, "https://pds", "h.test", "x@x") + .run() + ).rejects.toThrow(); + }); +}); diff --git a/packages/contrail-community/tests/service-auth.test.ts b/packages/contrail-community/tests/service-auth.test.ts new file mode 100644 index 0000000..199e581 --- /dev/null +++ b/packages/contrail-community/tests/service-auth.test.ts @@ -0,0 +1,135 @@ +import { describe, it, expect } from "vitest"; +import { mintServiceAuthJwt } from "../src/service-auth"; +import { generateKeyPair } from "../src/plc"; + +function b64urlDecode(s: string): Uint8Array { + const normal = s.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function decodeJwtJson(seg: string): Record<string, unknown> { + return JSON.parse(new TextDecoder().decode(b64urlDecode(seg))); +} + +/** P-256 curve order; used for low-S threshold. */ +const P256_N = BigInt( + "0xFFFFFFFF00000000FFFFFFFFFFFFFFFFBCE6FAADA7179E84F3B9CAC2FC632551" +); +const P256_N_HALF = P256_N >> 1n; + +function bytesToBigInt(b: Uint8Array): bigint { + let v = 0n; + for (const byte of b) v = (v << 8n) | BigInt(byte); + return v; +} + +describe("mintServiceAuthJwt", () => { + it("round-trips: signature verifies against the keypair's public key (P1363, not DER)", async () => { + // This is the canary: if the signer accidentally returns DER instead of raw r||s, + // Web Crypto's verify with raw form will reject it — and so will atproto PDSes. + const kp = await generateKeyPair(); + const jwt = await mintServiceAuthJwt({ + privateJwk: kp.privateJwk, + iss: "did:plc:abc", + aud: "did:web:pds.test", + lxm: "com.atproto.server.createAccount", + }); + + // Reconstruct the public JWK from the private JWK (drop d, key_ops, ext). + const priv = kp.privateJwk as Record<string, unknown>; + const publicJwk: JsonWebKey = { + kty: priv.kty as string, + crv: priv.crv as string, + x: priv.x as string, + y: priv.y as string, + }; + const publicKey = await crypto.subtle.importKey( + "jwk", + publicJwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"] + ); + + const [h, p, s] = jwt.split("."); + const signedBytes = new TextEncoder().encode(`${h}.${p}`); + const sig = b64urlDecode(s!); + + // P1363 form is exactly 64 bytes for P-256. DER would be ~70-72 and start with 0x30. + expect(sig.length).toBe(64); + + const ok = await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + sig as BufferSource, + signedBytes as BufferSource + ); + expect(ok).toBe(true); + }); + + it("emits low-S signatures across many independent signings", async () => { + // Without normalization, ~50% of ECDSA signatures have s > n/2. + // Across 12 fresh keypairs, the probability of *all* being naturally low-S is + // ~1/4096. If the test ever sees a high-S signature, normalization is broken. + const kp = await generateKeyPair(); + for (let i = 0; i < 12; i++) { + const jwt = await mintServiceAuthJwt({ + privateJwk: kp.privateJwk, + iss: "did:plc:abc", + aud: "did:web:pds.test", + lxm: "com.atproto.server.createAccount", + // Force a fresh signature each iteration; ECDSA k is randomized per sign. + }); + const sig = b64urlDecode(jwt.split(".")[2]!); + const s = bytesToBigInt(sig.slice(32)); + expect(s).toBeLessThanOrEqual(P256_N_HALF); + } + }); + + it("encodes header and claims as base64url JSON with the expected shape", async () => { + const kp = await generateKeyPair(); + const fixedNow = 1_700_000_000_000; + const jwt = await mintServiceAuthJwt({ + privateJwk: kp.privateJwk, + iss: "did:plc:abc", + aud: "did:web:pds.test", + lxm: "com.atproto.server.createAccount", + ttlSec: 60, + now: fixedNow, + }); + + const [h, p] = jwt.split("."); + const header = decodeJwtJson(h!); + const payload = decodeJwtJson(p!); + + // Header MUST be exactly {alg,typ}; presence of kid would change the + // signed bytes and break PDS verification (which doesn't use kid). + expect(header).toEqual({ alg: "ES256", typ: "JWT" }); + + expect(payload.iss).toBe("did:plc:abc"); + expect(payload.aud).toBe("did:web:pds.test"); + expect(payload.lxm).toBe("com.atproto.server.createAccount"); + expect(payload.iat).toBe(Math.floor(fixedNow / 1000)); + expect(payload.exp).toBe(Math.floor(fixedNow / 1000) + 60); + expect(typeof payload.jti).toBe("string"); + expect((payload.jti as string).length).toBeGreaterThan(0); + }); + + it("uses unique jti per call (replay-protection sanity)", async () => { + const kp = await generateKeyPair(); + const mk = () => + mintServiceAuthJwt({ + privateJwk: kp.privateJwk, + iss: "did:plc:abc", + aud: "did:web:pds.test", + lxm: "com.atproto.server.createAccount", + }); + const a = decodeJwtJson((await mk()).split(".")[1]!); + const b = decodeJwtJson((await mk()).split(".")[1]!); + expect(a.jti).not.toBe(b.jti); + }); +}); diff --git a/packages/contrail-community/tsup.config.ts b/packages/contrail-community/tsup.config.ts index b033415..cbfbc7e 100644 --- a/packages/contrail-community/tsup.config.ts +++ b/packages/contrail-community/tsup.config.ts @@ -1,11 +1,11 @@ import { defineConfig } from "tsup"; export default defineConfig({ - entry: ["src/index.ts"], + entry: ["src/index.ts", "src/cli/index.ts"], format: ["esm"], dts: true, sourcemap: true, clean: true, tsconfig: "tsconfig.build.json", - external: ["@atmo-dev/contrail"], + external: ["@atmo-dev/contrail", "wrangler"], }); diff --git a/packages/contrail/src/cli.ts b/packages/contrail/src/cli.ts index 8b1978f..af648dd 100644 --- a/packages/contrail/src/cli.ts +++ b/packages/contrail/src/cli.ts @@ -10,6 +10,7 @@ import { registerBackfill } from "./cli/commands/backfill.js"; import { registerRefresh } from "./cli/commands/refresh.js"; import { registerDev } from "./cli/commands/dev.js"; import { registerAppendScheduled } from "./cli/commands/append-scheduled.js"; +import { resolveAndLoadConfig } from "./cli/shared.js"; const cli = cac("contrail"); @@ -18,6 +19,42 @@ registerRefresh(cli); registerDev(cli); registerAppendScheduled(cli); +// `reap` lives in @atmo-dev/contrail-community after the PR #30 package split. +// Dynamically import so contrail has no compile-time edge into the community +// package (contrail-community already depends on contrail; a static import +// here would create a build cycle). contrail declares contrail-community as an +// optional peer so a consumer that installs both gets `reap` wired up; when +// it's genuinely absent the subcommand is simply omitted. +try { + const mod = await import("@atmo-dev/contrail-community" as string); + if (typeof mod.registerReap === "function") { + mod.registerReap(cli, { resolveAndLoadConfig }); + } else { + // Loaded, but the expected export is missing — a real packaging problem, + // not "not installed". Surface it so a broken build is debuggable. + console.warn( + "[contrail] @atmo-dev/contrail-community loaded but does not export registerReap; `reap` unavailable." + ); + } +} catch (err) { + const code = (err as { code?: string })?.code; + if (code === "ERR_MODULE_NOT_FOUND") { + // Expected when contrail-community isn't installed alongside contrail. + // Debug-level so it doesn't nag, but is visible under DEBUG diagnostics. + if (process.env.DEBUG) { + console.debug("[contrail] contrail-community not installed; `reap` unavailable."); + } + } else { + // A different failure (broken export, missing transitive dep, syntax + // error) — do NOT swallow it, or `reap` silently vanishes with no clue. + console.warn( + `[contrail] failed to load @atmo-dev/contrail-community for \`reap\`: ${ + err instanceof Error ? err.message : String(err) + }` + ); + } +} + cli.help(); try { diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 63d07ec..a5ccd69 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -189,5 +189,5 @@ export type { PersistentLabelsOptions } from "./core/labels/subscribe"; export { resolveLabelerEndpoint } from "./core/labels/resolve"; // Community has moved to @atmo-dev/contrail-community. Import from there: -// import { createCommunityIntegration, CommunityAdapter, ... } from "@atmo-dev/contrail-community"; +// import { createCommunityIntegration, CommunityAdapter, ProvisionOrchestrator, ... } from "@atmo-dev/contrail-community"; // const app = createApp(db, config, { community: createCommunityIntegration(...) }); diff --git a/packages/contrail/tests/schema.test.ts b/packages/contrail/tests/schema.test.ts index 6197818..9b5aafa 100644 --- a/packages/contrail/tests/schema.test.ts +++ b/packages/contrail/tests/schema.test.ts @@ -55,3 +55,4 @@ describe("initSchema", () => { await initSchema(db, TEST_CONFIG); }); }); + diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index adc192d..3966d34 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -569,10 +569,25 @@ importers: '@atmo-dev/contrail-base': specifier: workspace:* version: link:../contrail-base + cac: + specifier: ^7.0.0 + version: 7.0.0 hono: specifier: ^4.12.8 version: 4.12.15 + wrangler: + specifier: ^4.0.0 + version: 4.84.1(@cloudflare/workers-types@4.20260424.1) devDependencies: + '@types/node': + specifier: ^25.5.0 + version: 25.6.0 + '@types/pg': + specifier: ^8.20.0 + version: 8.20.0 + pg: + specifier: ^8.20.0 + version: 8.20.0 tsup: specifier: ^8.5.0 version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) -- 2.51.2 From f26ad37d191f4109dce35a275b3084605ba9fcef Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:34:16 -0400 Subject: [PATCH 22/25] Version Packages (#45) Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> --- .changeset/itchy-rice-kick.md | 22 ----------- .changeset/private-network-overrides.md | 19 ---------- packages/contrail-appview/CHANGELOG.md | 25 ++++++++++++ packages/contrail-appview/package.json | 2 +- packages/contrail-authority/CHANGELOG.md | 7 ++++ packages/contrail-authority/package.json | 2 +- packages/contrail-base/CHANGELOG.md | 18 +++++++++ packages/contrail-base/package.json | 2 +- packages/contrail-community/CHANGELOG.md | 44 ++++++++++++++++++++++ packages/contrail-community/package.json | 2 +- packages/contrail-record-host/CHANGELOG.md | 7 ++++ packages/contrail-record-host/package.json | 2 +- packages/contrail/CHANGELOG.md | 36 ++++++++++++++++++ packages/contrail/package.json | 2 +- packages/lexicons/CHANGELOG.md | 10 +++++ packages/lexicons/package.json | 2 +- packages/sync/CHANGELOG.md | 5 +++ packages/sync/package.json | 2 +- 18 files changed, 160 insertions(+), 49 deletions(-) delete mode 100644 .changeset/itchy-rice-kick.md delete mode 100644 .changeset/private-network-overrides.md diff --git a/.changeset/itchy-rice-kick.md b/.changeset/itchy-rice-kick.md deleted file mode 100644 index 84f0750..0000000 --- a/.changeset/itchy-rice-kick.md +++ /dev/null @@ -1,22 +0,0 @@ ---- -"@atmo-dev/contrail-community": minor -"@atmo-dev/contrail": minor ---- - -A third community-creation mode: **provision**. alongside the existing `adopt` (caller already has a `did:plc`) and `mint` (caller wants a DID but brings their own PDS) modes, contrail can now provision a community on a stock `@atproto/pds` end-to-end — minting the `did:plc`, creating and activating the PDS account, generating an app password, and persisting credentials so the existing `community.putRecord` / `.deleteRecord` publish path keeps working. contrail never holds PDS admin credentials. - -**`xrpc/{ns}.community.provision`** runs the five-step PLC + PDS dance (key generation → PLC genesis → `createAccount` → `getRecommendedDidCredentials` + signed PLC update op → `activateAccount`), persists each step in a new `provision_attempts` table so a partially-failed attempt can be resumed, mints an app password, and seeds the session cache. - -**`contrail-community reap [--all-stuck] [--older-than <minutes>] [--db <url>] [--dry-run]`** new CLI (a bin shipped by `@atmo-dev/contrail-community`) that cleans up provision attempts which didn't reach `status='activated'` by tombstoning their PLC entries. `--dry-run` is the default; per-row confirmation is required for live reaping unless `--all-stuck` is given. `--all-stuck` only acts on rows idle at least `--older-than` minutes (default 30) so a bulk run can't tombstone an in-flight provision. Runs against the Cloudflare D1 binding by default, or against the decoupled Postgres index when `--db`/`DATABASE_URL` is set. It ships as a contrail-community bin because the PR #30 package split removed contrail's edge into community code: under pnpm's isolated `node_modules` the core `contrail` CLI can't resolve `@atmo-dev/contrail-community`, so `contrail reap` only registers in hoisted installs where both packages sit together. - -custody model: the caller supplies a `rotationKey` and that key sits at `rotationKeys[0]` — the highest-priority rotation slot on the resulting DID. contrail generates a subordinate keypair and persists it (AES-GCM-encrypted under `masterKey`) at `rotationKeys[1]`, so it can submit later PLC ops on the community's behalf — most importantly the post-activation PLC update during provision, and the tombstone op that `reap` issues to clean up stuck DIDs. - -the caller's key dominates: PLC's 72-hour nullification window means any op contrail signs with its subordinate key can be overridden within 72h by an op signed with the caller's key. with this caveat: a tombstone is irrevocable. a malicious or compromised contrail instance could tombstone any DID it provisioned. there is no managed code path, no shared rotation, and `rootCredentials` are returned to the caller in the response so they can also be persisted out-of-band. - -what you need to configure / know: - -- new `community` config block: `masterKey` (32-byte AES-GCM envelope key for the encrypted credential columns), `allowedProvisionPdsEndpoints` (URL-origin matching, collapses scheme case / default ports / trailing slash / IDN), optional `plcDirectory` override. - -- **provisioning fails closed.** When `allowProvisioning` is true, `allowedProvisionPdsEndpoints` MUST be non-empty — a missing/empty allowlist no longer means "accept any PDS" (that was a fail-open hole: any caller could have a PLC genesis op signed by Contrail's rotation key against an attacker-chosen PDS). To deliberately accept any endpoint, set the separate, loud `allowAnyProvisionPdsEndpoint: true`. The field was renamed from `allowedPdsEndpoints` to make clear it gates *provisioning* only, not which PDSes Contrail reads/indexes. - -- new tables `provision_attempts` and `community_credentials`. credentials are stored AES-GCM-encrypted under that key; lose the key, lose the ability to mint sessions for previously-provisioned communities. diff --git a/.changeset/private-network-overrides.md b/.changeset/private-network-overrides.md deleted file mode 100644 index 874794e..0000000 --- a/.changeset/private-network-overrides.md +++ /dev/null @@ -1,19 +0,0 @@ ---- -"@atmo-dev/contrail-base": minor -"@atmo-dev/contrail-appview": minor -"@atmo-dev/contrail-community": minor ---- - -Private-network deployment support via a new optional `ContrailConfig.networkOverrides` block. - -`networkOverrides` carries three optional subfields, all defaulting to the current public-internet behavior (omit the block entirely and nothing changes): - -- **`resolver`** — a custom `DidDocumentResolver` used during DID-doc PDS fallback, labeler-endpoint resolution, and spaces service-auth JWT verification. Lets a deployment point at a private PLC mirror or inject a custom fetch (mTLS, retry, instrumentation). Trusted; not SSRF-checked. -- **`slingshotUrl`** — override the slingshot identity-resolver endpoint. Trusted; not SSRF-checked. -- **`additionalAllowedHosts`** — hostnames that bypass the default SSRF guard when validating a resolved PDS or labeler endpoint. Match is exact, case-insensitive, port-agnostic (e.g. `["pds.dev.svc.cluster.local"]`). This is the only knob that widens the validator; there is no "disable SSRF" flag. - -The overrides are threaded through PDS/identity resolution (`resolvePDS`, `getPDS`, `getClient`, `resolveIdentity*`, `refreshStaleIdentities`), labeler endpoint resolution and ingest (`resolveLabelerEndpoint`, `getLabelerState`, label subscribe cycles), and service-auth verification (`buildVerifier` in both the appview router and the community integration). The in-scope `config` is now also passed at every appview call site that resolves identities or PDS endpoints — the live-ingest refresh cycle (`runIngestCycle` → `refreshStaleIdentities`), the on-demand `refresh` path, and the router actor/identity/PDS resolution paths (`getProfile`, `getFeed`, collection queries, profile hydration, notify) — so private-network deploys honor the override on those paths instead of silently falling back to the public resolver and un-widened SSRF guard. - -The SSRF guard is now a single shared validator: `validateExternalUrl(url, additionalAllowedHosts?)` is exported from `contrail-base` and consumed by both the PDS client and labeler-endpoint resolution. `validateEndpointUrl` remains exported as a thin alias for backward compatibility. This removes the previous duplicate validator (`validatePdsUrl` + `validateEndpointUrl`) where an allowlist or SSRF-rule edit could be applied to only one copy. - -Also hardens schema initialization for concurrent/Postgres deployments: a dialect-aware `addColumnIfNotExists` (Postgres `ADD COLUMN IF NOT EXISTS`; SQLite pre-check), narrow absorption of the Postgres concurrent-`CREATE` race (42P07 / 23505 on pg_type/pg_class/pg_namespace indexes), and per-statement (rather than batched) DDL during `initSchema` / `initSpacesSchema` / spaces schema. Genuine DDL errors (syntax, type mismatch, missing column/table) still propagate. diff --git a/packages/contrail-appview/CHANGELOG.md b/packages/contrail-appview/CHANGELOG.md index cd18649..371296c 100644 --- a/packages/contrail-appview/CHANGELOG.md +++ b/packages/contrail-appview/CHANGELOG.md @@ -1,5 +1,30 @@ # @atmo-dev/contrail-appview +## 0.8.0 + +### Minor Changes + +- d7e0936: Private-network deployment support via a new optional `ContrailConfig.networkOverrides` block. + + `networkOverrides` carries three optional subfields, all defaulting to the current public-internet behavior (omit the block entirely and nothing changes): + + - **`resolver`** — a custom `DidDocumentResolver` used during DID-doc PDS fallback, labeler-endpoint resolution, and spaces service-auth JWT verification. Lets a deployment point at a private PLC mirror or inject a custom fetch (mTLS, retry, instrumentation). Trusted; not SSRF-checked. + - **`slingshotUrl`** — override the slingshot identity-resolver endpoint. Trusted; not SSRF-checked. + - **`additionalAllowedHosts`** — hostnames that bypass the default SSRF guard when validating a resolved PDS or labeler endpoint. Match is exact, case-insensitive, port-agnostic (e.g. `["pds.dev.svc.cluster.local"]`). This is the only knob that widens the validator; there is no "disable SSRF" flag. + + The overrides are threaded through PDS/identity resolution (`resolvePDS`, `getPDS`, `getClient`, `resolveIdentity*`, `refreshStaleIdentities`), labeler endpoint resolution and ingest (`resolveLabelerEndpoint`, `getLabelerState`, label subscribe cycles), and service-auth verification (`buildVerifier` in both the appview router and the community integration). The in-scope `config` is now also passed at every appview call site that resolves identities or PDS endpoints — the live-ingest refresh cycle (`runIngestCycle` → `refreshStaleIdentities`), the on-demand `refresh` path, and the router actor/identity/PDS resolution paths (`getProfile`, `getFeed`, collection queries, profile hydration, notify) — so private-network deploys honor the override on those paths instead of silently falling back to the public resolver and un-widened SSRF guard. + + The SSRF guard is now a single shared validator: `validateExternalUrl(url, additionalAllowedHosts?)` is exported from `contrail-base` and consumed by both the PDS client and labeler-endpoint resolution. `validateEndpointUrl` remains exported as a thin alias for backward compatibility. This removes the previous duplicate validator (`validatePdsUrl` + `validateEndpointUrl`) where an allowlist or SSRF-rule edit could be applied to only one copy. + + Also hardens schema initialization for concurrent/Postgres deployments: a dialect-aware `addColumnIfNotExists` (Postgres `ADD COLUMN IF NOT EXISTS`; SQLite pre-check), narrow absorption of the Postgres concurrent-`CREATE` race (42P07 / 23505 on pg_type/pg_class/pg_namespace indexes), and per-statement (rather than batched) DDL during `initSchema` / `initSpacesSchema` / spaces schema. Genuine DDL errors (syntax, type mismatch, missing column/table) still propagate. + +### Patch Changes + +- Updated dependencies [d7e0936] + - @atmo-dev/contrail-base@0.8.0 + - @atmo-dev/contrail-authority@0.8.0 + - @atmo-dev/contrail-record-host@0.8.0 + ## 0.7.0 ### Patch Changes diff --git a/packages/contrail-appview/package.json b/packages/contrail-appview/package.json index f2627dd..d1eedd2 100644 --- a/packages/contrail-appview/package.json +++ b/packages/contrail-appview/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-appview", - "version": "0.7.0", + "version": "0.8.0", "description": "Public-records appview for contrail — jetstream ingestion, backfill, query layer, feeds, labels, profiles, per-collection XRPC routes.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-authority/CHANGELOG.md b/packages/contrail-authority/CHANGELOG.md index de82f64..bccf371 100644 --- a/packages/contrail-authority/CHANGELOG.md +++ b/packages/contrail-authority/CHANGELOG.md @@ -1,5 +1,12 @@ # @atmo-dev/contrail-authority +## 0.8.0 + +### Patch Changes + +- Updated dependencies [d7e0936] + - @atmo-dev/contrail-base@0.8.0 + ## 0.7.0 ### Patch Changes diff --git a/packages/contrail-authority/package.json b/packages/contrail-authority/package.json index a088146..3b2f6de 100644 --- a/packages/contrail-authority/package.json +++ b/packages/contrail-authority/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-authority", - "version": "0.7.0", + "version": "0.8.0", "description": "Default space-authority implementation for contrail — member list, invites, app policy, credential issuance. Contrail's binary-membership ACL flavor; for ladder-style access levels see @atmo-dev/contrail-community.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-base/CHANGELOG.md b/packages/contrail-base/CHANGELOG.md index e6ed047..4c6e069 100644 --- a/packages/contrail-base/CHANGELOG.md +++ b/packages/contrail-base/CHANGELOG.md @@ -1,3 +1,21 @@ # @atmo-dev/contrail-base +## 0.8.0 + +### Minor Changes + +- d7e0936: Private-network deployment support via a new optional `ContrailConfig.networkOverrides` block. + + `networkOverrides` carries three optional subfields, all defaulting to the current public-internet behavior (omit the block entirely and nothing changes): + + - **`resolver`** — a custom `DidDocumentResolver` used during DID-doc PDS fallback, labeler-endpoint resolution, and spaces service-auth JWT verification. Lets a deployment point at a private PLC mirror or inject a custom fetch (mTLS, retry, instrumentation). Trusted; not SSRF-checked. + - **`slingshotUrl`** — override the slingshot identity-resolver endpoint. Trusted; not SSRF-checked. + - **`additionalAllowedHosts`** — hostnames that bypass the default SSRF guard when validating a resolved PDS or labeler endpoint. Match is exact, case-insensitive, port-agnostic (e.g. `["pds.dev.svc.cluster.local"]`). This is the only knob that widens the validator; there is no "disable SSRF" flag. + + The overrides are threaded through PDS/identity resolution (`resolvePDS`, `getPDS`, `getClient`, `resolveIdentity*`, `refreshStaleIdentities`), labeler endpoint resolution and ingest (`resolveLabelerEndpoint`, `getLabelerState`, label subscribe cycles), and service-auth verification (`buildVerifier` in both the appview router and the community integration). The in-scope `config` is now also passed at every appview call site that resolves identities or PDS endpoints — the live-ingest refresh cycle (`runIngestCycle` → `refreshStaleIdentities`), the on-demand `refresh` path, and the router actor/identity/PDS resolution paths (`getProfile`, `getFeed`, collection queries, profile hydration, notify) — so private-network deploys honor the override on those paths instead of silently falling back to the public resolver and un-widened SSRF guard. + + The SSRF guard is now a single shared validator: `validateExternalUrl(url, additionalAllowedHosts?)` is exported from `contrail-base` and consumed by both the PDS client and labeler-endpoint resolution. `validateEndpointUrl` remains exported as a thin alias for backward compatibility. This removes the previous duplicate validator (`validatePdsUrl` + `validateEndpointUrl`) where an allowlist or SSRF-rule edit could be applied to only one copy. + + Also hardens schema initialization for concurrent/Postgres deployments: a dialect-aware `addColumnIfNotExists` (Postgres `ADD COLUMN IF NOT EXISTS`; SQLite pre-check), narrow absorption of the Postgres concurrent-`CREATE` race (42P07 / 23505 on pg_type/pg_class/pg_namespace indexes), and per-statement (rather than batched) DDL during `initSchema` / `initSpacesSchema` / spaces schema. Genuine DDL errors (syntax, type mismatch, missing column/table) still propagate. + ## 0.7.0 diff --git a/packages/contrail-base/package.json b/packages/contrail-base/package.json index 5535411..a97edb4 100644 --- a/packages/contrail-base/package.json +++ b/packages/contrail-base/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-base", - "version": "0.7.0", + "version": "0.8.0", "description": "Shared infrastructure for the contrail family of packages — interfaces (SpaceAuthority, RecordHost, CommunityIntegration), credential primitives, binding resolvers, realtime infra, schema scaffolding. No routes, no tables of its own.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-community/CHANGELOG.md b/packages/contrail-community/CHANGELOG.md index 163f0e5..ffe7196 100644 --- a/packages/contrail-community/CHANGELOG.md +++ b/packages/contrail-community/CHANGELOG.md @@ -1,5 +1,48 @@ # @atmo-dev/contrail-community +## 0.8.0 + +### Minor Changes + +- bea0dd2: A third community-creation mode: **provision**. alongside the existing `adopt` (caller already has a `did:plc`) and `mint` (caller wants a DID but brings their own PDS) modes, contrail can now provision a community on a stock `@atproto/pds` end-to-end — minting the `did:plc`, creating and activating the PDS account, generating an app password, and persisting credentials so the existing `community.putRecord` / `.deleteRecord` publish path keeps working. contrail never holds PDS admin credentials. + + **`xrpc/{ns}.community.provision`** runs the five-step PLC + PDS dance (key generation → PLC genesis → `createAccount` → `getRecommendedDidCredentials` + signed PLC update op → `activateAccount`), persists each step in a new `provision_attempts` table so a partially-failed attempt can be resumed, mints an app password, and seeds the session cache. + + **`contrail-community reap [--all-stuck] [--older-than <minutes>] [--db <url>] [--dry-run]`** new CLI (a bin shipped by `@atmo-dev/contrail-community`) that cleans up provision attempts which didn't reach `status='activated'` by tombstoning their PLC entries. `--dry-run` is the default; per-row confirmation is required for live reaping unless `--all-stuck` is given. `--all-stuck` only acts on rows idle at least `--older-than` minutes (default 30) so a bulk run can't tombstone an in-flight provision. Runs against the Cloudflare D1 binding by default, or against the decoupled Postgres index when `--db`/`DATABASE_URL` is set. It ships as a contrail-community bin because the PR #30 package split removed contrail's edge into community code: under pnpm's isolated `node_modules` the core `contrail` CLI can't resolve `@atmo-dev/contrail-community`, so `contrail reap` only registers in hoisted installs where both packages sit together. + + custody model: the caller supplies a `rotationKey` and that key sits at `rotationKeys[0]` — the highest-priority rotation slot on the resulting DID. contrail generates a subordinate keypair and persists it (AES-GCM-encrypted under `masterKey`) at `rotationKeys[1]`, so it can submit later PLC ops on the community's behalf — most importantly the post-activation PLC update during provision, and the tombstone op that `reap` issues to clean up stuck DIDs. + + the caller's key dominates: PLC's 72-hour nullification window means any op contrail signs with its subordinate key can be overridden within 72h by an op signed with the caller's key. with this caveat: a tombstone is irrevocable. a malicious or compromised contrail instance could tombstone any DID it provisioned. there is no managed code path, no shared rotation, and `rootCredentials` are returned to the caller in the response so they can also be persisted out-of-band. + + what you need to configure / know: + + - new `community` config block: `masterKey` (32-byte AES-GCM envelope key for the encrypted credential columns), `allowedProvisionPdsEndpoints` (URL-origin matching, collapses scheme case / default ports / trailing slash / IDN), optional `plcDirectory` override. + + - **provisioning fails closed.** When `allowProvisioning` is true, `allowedProvisionPdsEndpoints` MUST be non-empty — a missing/empty allowlist no longer means "accept any PDS" (that was a fail-open hole: any caller could have a PLC genesis op signed by Contrail's rotation key against an attacker-chosen PDS). To deliberately accept any endpoint, set the separate, loud `allowAnyProvisionPdsEndpoint: true`. The field was renamed from `allowedPdsEndpoints` to make clear it gates _provisioning_ only, not which PDSes Contrail reads/indexes. + + - new tables `provision_attempts` and `community_credentials`. credentials are stored AES-GCM-encrypted under that key; lose the key, lose the ability to mint sessions for previously-provisioned communities. + +- d7e0936: Private-network deployment support via a new optional `ContrailConfig.networkOverrides` block. + + `networkOverrides` carries three optional subfields, all defaulting to the current public-internet behavior (omit the block entirely and nothing changes): + + - **`resolver`** — a custom `DidDocumentResolver` used during DID-doc PDS fallback, labeler-endpoint resolution, and spaces service-auth JWT verification. Lets a deployment point at a private PLC mirror or inject a custom fetch (mTLS, retry, instrumentation). Trusted; not SSRF-checked. + - **`slingshotUrl`** — override the slingshot identity-resolver endpoint. Trusted; not SSRF-checked. + - **`additionalAllowedHosts`** — hostnames that bypass the default SSRF guard when validating a resolved PDS or labeler endpoint. Match is exact, case-insensitive, port-agnostic (e.g. `["pds.dev.svc.cluster.local"]`). This is the only knob that widens the validator; there is no "disable SSRF" flag. + + The overrides are threaded through PDS/identity resolution (`resolvePDS`, `getPDS`, `getClient`, `resolveIdentity*`, `refreshStaleIdentities`), labeler endpoint resolution and ingest (`resolveLabelerEndpoint`, `getLabelerState`, label subscribe cycles), and service-auth verification (`buildVerifier` in both the appview router and the community integration). The in-scope `config` is now also passed at every appview call site that resolves identities or PDS endpoints — the live-ingest refresh cycle (`runIngestCycle` → `refreshStaleIdentities`), the on-demand `refresh` path, and the router actor/identity/PDS resolution paths (`getProfile`, `getFeed`, collection queries, profile hydration, notify) — so private-network deploys honor the override on those paths instead of silently falling back to the public resolver and un-widened SSRF guard. + + The SSRF guard is now a single shared validator: `validateExternalUrl(url, additionalAllowedHosts?)` is exported from `contrail-base` and consumed by both the PDS client and labeler-endpoint resolution. `validateEndpointUrl` remains exported as a thin alias for backward compatibility. This removes the previous duplicate validator (`validatePdsUrl` + `validateEndpointUrl`) where an allowlist or SSRF-rule edit could be applied to only one copy. + + Also hardens schema initialization for concurrent/Postgres deployments: a dialect-aware `addColumnIfNotExists` (Postgres `ADD COLUMN IF NOT EXISTS`; SQLite pre-check), narrow absorption of the Postgres concurrent-`CREATE` race (42P07 / 23505 on pg_type/pg_class/pg_namespace indexes), and per-statement (rather than batched) DDL during `initSchema` / `initSpacesSchema` / spaces schema. Genuine DDL errors (syntax, type mismatch, missing column/table) still propagate. + +### Patch Changes + +- Updated dependencies [bea0dd2] +- Updated dependencies [d7e0936] + - @atmo-dev/contrail@0.8.0 + - @atmo-dev/contrail-base@0.8.0 + ## 0.7.0 ### Minor Changes @@ -90,6 +133,7 @@ **Migration** For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. 2. Generate and store an authority signing key diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index b0f2c47..a5afdb6 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-community", - "version": "0.7.0", + "version": "0.8.0", "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-record-host/CHANGELOG.md b/packages/contrail-record-host/CHANGELOG.md index bbc13ac..5dad428 100644 --- a/packages/contrail-record-host/CHANGELOG.md +++ b/packages/contrail-record-host/CHANGELOG.md @@ -1,5 +1,12 @@ # @atmo-dev/contrail-record-host +## 0.8.0 + +### Patch Changes + +- Updated dependencies [d7e0936] + - @atmo-dev/contrail-base@0.8.0 + ## 0.7.0 ### Patch Changes diff --git a/packages/contrail-record-host/package.json b/packages/contrail-record-host/package.json index b0f50f9..2beced2 100644 --- a/packages/contrail-record-host/package.json +++ b/packages/contrail-record-host/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-record-host", - "version": "0.7.0", + "version": "0.8.0", "description": "Default record-host implementation for contrail — stores records and blobs for permissioned spaces, enforces local enrollment as the host's consent layer.", "type": "module", "sideEffects": false, diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index d404678..d50a581 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,35 @@ # @atmo-dev/contrail +## 0.8.0 + +### Minor Changes + +- bea0dd2: A third community-creation mode: **provision**. alongside the existing `adopt` (caller already has a `did:plc`) and `mint` (caller wants a DID but brings their own PDS) modes, contrail can now provision a community on a stock `@atproto/pds` end-to-end — minting the `did:plc`, creating and activating the PDS account, generating an app password, and persisting credentials so the existing `community.putRecord` / `.deleteRecord` publish path keeps working. contrail never holds PDS admin credentials. + + **`xrpc/{ns}.community.provision`** runs the five-step PLC + PDS dance (key generation → PLC genesis → `createAccount` → `getRecommendedDidCredentials` + signed PLC update op → `activateAccount`), persists each step in a new `provision_attempts` table so a partially-failed attempt can be resumed, mints an app password, and seeds the session cache. + + **`contrail-community reap [--all-stuck] [--older-than <minutes>] [--db <url>] [--dry-run]`** new CLI (a bin shipped by `@atmo-dev/contrail-community`) that cleans up provision attempts which didn't reach `status='activated'` by tombstoning their PLC entries. `--dry-run` is the default; per-row confirmation is required for live reaping unless `--all-stuck` is given. `--all-stuck` only acts on rows idle at least `--older-than` minutes (default 30) so a bulk run can't tombstone an in-flight provision. Runs against the Cloudflare D1 binding by default, or against the decoupled Postgres index when `--db`/`DATABASE_URL` is set. It ships as a contrail-community bin because the PR #30 package split removed contrail's edge into community code: under pnpm's isolated `node_modules` the core `contrail` CLI can't resolve `@atmo-dev/contrail-community`, so `contrail reap` only registers in hoisted installs where both packages sit together. + + custody model: the caller supplies a `rotationKey` and that key sits at `rotationKeys[0]` — the highest-priority rotation slot on the resulting DID. contrail generates a subordinate keypair and persists it (AES-GCM-encrypted under `masterKey`) at `rotationKeys[1]`, so it can submit later PLC ops on the community's behalf — most importantly the post-activation PLC update during provision, and the tombstone op that `reap` issues to clean up stuck DIDs. + + the caller's key dominates: PLC's 72-hour nullification window means any op contrail signs with its subordinate key can be overridden within 72h by an op signed with the caller's key. with this caveat: a tombstone is irrevocable. a malicious or compromised contrail instance could tombstone any DID it provisioned. there is no managed code path, no shared rotation, and `rootCredentials` are returned to the caller in the response so they can also be persisted out-of-band. + + what you need to configure / know: + + - new `community` config block: `masterKey` (32-byte AES-GCM envelope key for the encrypted credential columns), `allowedProvisionPdsEndpoints` (URL-origin matching, collapses scheme case / default ports / trailing slash / IDN), optional `plcDirectory` override. + + - **provisioning fails closed.** When `allowProvisioning` is true, `allowedProvisionPdsEndpoints` MUST be non-empty — a missing/empty allowlist no longer means "accept any PDS" (that was a fail-open hole: any caller could have a PLC genesis op signed by Contrail's rotation key against an attacker-chosen PDS). To deliberately accept any endpoint, set the separate, loud `allowAnyProvisionPdsEndpoint: true`. The field was renamed from `allowedPdsEndpoints` to make clear it gates _provisioning_ only, not which PDSes Contrail reads/indexes. + + - new tables `provision_attempts` and `community_credentials`. credentials are stored AES-GCM-encrypted under that key; lose the key, lose the ability to mint sessions for previously-provisioned communities. + +### Patch Changes + +- Updated dependencies [d7e0936] + - @atmo-dev/contrail-base@0.8.0 + - @atmo-dev/contrail-appview@0.8.0 + - @atmo-dev/contrail-authority@0.8.0 + - @atmo-dev/contrail-record-host@0.8.0 + ## 0.7.0 ### Minor Changes @@ -90,6 +120,7 @@ **Migration** For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. 2. Generate and store an authority signing key @@ -179,6 +210,7 @@ ``` what changed: + - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -205,6 +237,7 @@ ``` changes: + - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. @@ -244,6 +277,7 @@ - ad3a61d: add `contrail dev` — local dev wrapper for cloudflare workers deployments. replaces `wrangler dev --test-scheduled` + a separate cron-trigger script with one command. on start it: + 1. connects to your local D1 via wrangler's `getPlatformProxy`, inspects state 2. prompts to run `backfillAll` if no completed backfills exist yet 3. prompts to run `refresh` if the ingest cursor is older than 60 minutes (configurable with `--stale-after`) @@ -271,6 +305,7 @@ options: `binding` (D1 binding name, default `"DB"`), `lexicons` (see below), `onInit` (one-shot app-specific setup). **`/xrpc/<ns>.lexicons` endpoint + `contrail-lex pull-service`** lets consumer apps typegen against a deployed contrail over HTTP, no PDS or DNS required: + - `contrail-lex generate` now emits a barrel `lexicons/generated/index.ts` that imports every lexicon the deployment speaks: generated + pulled + custom. The pulled lexicons are needed so consumer typegen can resolve `$ref`s out of the generated schemas. - Pass `{ lexicons }` to `createWorker` (or `createHandler(contrail, { lexicons })`) and the service exposes them at `GET /xrpc/<namespace>.lexicons`. - From a consumer app: @@ -288,6 +323,7 @@ unlike `backfillAll`, it ignores the `backfills` state table and sweeps fresh. useful after jetstream outages or after leaving a dev deployment idle for days. each record in each configured collection is classified as: + - **missing** — PDS has it, DB doesn't - **stale update** — DB has it with a different CID, _and_ the DB row was written before the ignore window (default 60s, configurable) - **in sync** — same CID, or DB row is within the ignore window diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 0219561..88158b2 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.7.0", + "version": "0.8.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, diff --git a/packages/lexicons/CHANGELOG.md b/packages/lexicons/CHANGELOG.md index ab5abd0..a51f8b5 100644 --- a/packages/lexicons/CHANGELOG.md +++ b/packages/lexicons/CHANGELOG.md @@ -1,5 +1,12 @@ # @atmo-dev/contrail-lexicons +## 0.4.8 + +### Patch Changes + +- Updated dependencies [bea0dd2] + - @atmo-dev/contrail@0.8.0 + ## 0.4.7 ### Patch Changes @@ -60,6 +67,7 @@ ``` what changed: + - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -92,6 +100,7 @@ ``` changes: + - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. @@ -115,6 +124,7 @@ options: `binding` (D1 binding name, default `"DB"`), `lexicons` (see below), `onInit` (one-shot app-specific setup). **`/xrpc/<ns>.lexicons` endpoint + `contrail-lex pull-service`** lets consumer apps typegen against a deployed contrail over HTTP, no PDS or DNS required: + - `contrail-lex generate` now emits a barrel `lexicons/generated/index.ts` that imports every lexicon the deployment speaks: generated + pulled + custom. The pulled lexicons are needed so consumer typegen can resolve `$ref`s out of the generated schemas. - Pass `{ lexicons }` to `createWorker` (or `createHandler(contrail, { lexicons })`) and the service exposes them at `GET /xrpc/<namespace>.lexicons`. - From a consumer app: diff --git a/packages/lexicons/package.json b/packages/lexicons/package.json index 066e06c..c35fcb9 100644 --- a/packages/lexicons/package.json +++ b/packages/lexicons/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-lexicons", - "version": "0.4.7", + "version": "0.4.8", "description": "Generate atproto lexicon JSON (and optionally TypeScript types via @atcute/lex-cli) from a Contrail config.", "type": "module", "files": [ diff --git a/packages/sync/CHANGELOG.md b/packages/sync/CHANGELOG.md index aa4b041..444bc23 100644 --- a/packages/sync/CHANGELOG.md +++ b/packages/sync/CHANGELOG.md @@ -1,5 +1,7 @@ # @atmo-dev/contrail-sync +## 0.8.0 + ## 0.7.0 ### Minor Changes @@ -90,6 +92,7 @@ **Migration** For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. 2. Generate and store an authority signing key @@ -128,6 +131,7 @@ ``` what changed: + - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -154,6 +158,7 @@ ``` changes: + - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. diff --git a/packages/sync/package.json b/packages/sync/package.json index 9bd77e4..d914e2a 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-sync", - "version": "0.7.0", + "version": "0.8.0", "description": "Client-side reactive watch-store over contrail's watchRecords endpoints. SSE + WebSocket transports, optimistic updates, optional IndexedDB cache.", "type": "module", "sideEffects": false, -- 2.51.2 From 8f0b87ed1b618b1630e411084d0306e38dd7dec7 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 7 Jun 2026 17:05:39 +0200 Subject: [PATCH 23/25] fix: bounded prune of follow feeds --- .changeset/feed-prune-bounded-sweep.md | 34 ++++ .../contrail-appview/src/core/db/index.ts | 4 +- .../contrail-appview/src/core/db/records.ts | 192 +++++++++++++++--- .../contrail-appview/src/core/db/schema.ts | 5 + .../contrail-appview/src/core/jetstream.ts | 36 +++- .../contrail-appview/src/core/persistent.ts | 27 ++- packages/contrail/tests/feed-prune.test.ts | 189 +++++++++++++++++ 7 files changed, 435 insertions(+), 52 deletions(-) create mode 100644 .changeset/feed-prune-bounded-sweep.md create mode 100644 packages/contrail/tests/feed-prune.test.ts diff --git a/.changeset/feed-prune-bounded-sweep.md b/.changeset/feed-prune-bounded-sweep.md new file mode 100644 index 0000000..e459924 --- /dev/null +++ b/.changeset/feed-prune-bounded-sweep.md @@ -0,0 +1,34 @@ +--- +"@atmo-dev/contrail-appview": minor +--- + +fix(feeds): make feed_items pruning bounded so it can't reset the D1 DO + +The hourly feed prune ran a single global `ROW_NUMBER() OVER (PARTITION BY actor)` +window + `(actor, uri) NOT IN (...)` anti-join across the entire `feed_items` +table — O(n) CPU in one statement. Once the table grew large this exceeded D1's +per-query CPU limit and reset the shared Durable Object, taking down any +concurrent read on the same SQLite instance (unrelated user requests 500'd with +`was reset` / `Network connection lost`). Because the statement reset before +completing, caps were never enforced, the table kept growing, and the prune got +more expensive — a death spiral. + +Changes: + +- **Bounded per-actor prune.** Pruning is now an index-backed cutoff delete per + `(actor, collection)` using `idx_feed_actor_coll_time`, cost O(cap), never + O(table). New `pruneActorFeed` / `sweepFeedItems` exports; the ingest loops + run one bounded `sweepFeedItems` slice per tick (`FEED_PRUNE_SWEEP_ACTORS` + actors), which also serves as recovery for already-bloated tables. +- **Persisted prune cursor.** A new `feed_prune_cursor` row tracks the rolling + sweep position, so progress survives the cron isolate recycling that + previously made the in-memory hourly gate a no-op (it pruned on essentially + every tick). The time gate is removed from the cron path; the long-lived + persistent loop keeps a short in-memory throttle. +- **API:** `pruneFeedItems(db, caps)` now accepts only the per-collection + `Map<collection, cap>` (the legacy global-number form is removed) and is + reimplemented as a bounded full-table recovery loop — keep it off the hot + path. + +The follow fan-out's `subject` lookup is already covered by `idx_<follow>_subject`, +so no unbounded statement remains in the ingest path. diff --git a/packages/contrail-appview/src/core/db/index.ts b/packages/contrail-appview/src/core/db/index.ts index 2469f31..fcc52bd 100644 --- a/packages/contrail-appview/src/core/db/index.ts +++ b/packages/contrail-appview/src/core/db/index.ts @@ -1,4 +1,4 @@ export { initSchema } from "./schema"; -export { getLastCursor, saveCursor, applyEvents, lookupExistingRecords, queryRecords, queryAcrossSources, pruneFeedItems } from "./records"; -export type { QueryOptions, SortOption, ExistingRecordInfo } from "./records"; +export { getLastCursor, saveCursor, applyEvents, lookupExistingRecords, queryRecords, queryAcrossSources, pruneFeedItems, pruneActorFeed, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./records"; +export type { QueryOptions, SortOption, ExistingRecordInfo, FeedSweepResult } from "./records"; export type { RecordSource } from "../types"; diff --git a/packages/contrail-appview/src/core/db/records.ts b/packages/contrail-appview/src/core/db/records.ts index 51414b8..57f696e 100644 --- a/packages/contrail-appview/src/core/db/records.ts +++ b/packages/contrail-appview/src/core/db/records.ts @@ -295,49 +295,175 @@ function buildFeedStatements( // --- Feed pruning --- -/** Prune feed_items per (actor, collection) to the given cap. +/** db.batch chunk size for the sweep — caps statements per transaction. */ +const SWEEP_BATCH_SIZE = 50; +/** Actor page size for the full-table {@link pruneFeedItems} recovery loop. */ +const FEED_PRUNE_RECOVERY_BATCH = 200; + +/** + * Build the bounded per-actor cutoff DELETE for one (actor, collection). + * + * Deletes everything older than the newest `cap` rows, driven directly by + * idx_feed_actor_coll_time(actor, collection, time_us DESC). Cost is + * O(cap + deleted) — never O(table). This is the ONLY prune shape contrail + * issues: an unbounded window/anti-join over the whole table can exhaust D1's + * per-query CPU budget and reset the shared Durable Object, which kills any + * concurrent read against the same SQLite instance. * - * - If `caps` is a number: legacy behavior — global per-actor cap across all collections. - * - If `caps` is a Map<collection-NSID, cap>: each collection is pruned independently per actor, - * so high-volume collections (e.g. RSVPs) can't squeeze out lower-volume ones (e.g. events). - * Collections not present in the map are left alone. + * The cutoff is the cap-th newest row (`OFFSET cap - 1`); we delete strictly + * older rows. Actors with `cap` or fewer rows: the OFFSET subquery yields no + * row, the cutoff is NULL, and `time_us < NULL` matches nothing — a cheap + * index no-op. On a tie at the cutoff time_us we keep the extra rows rather + * than risk deleting a row we meant to keep (feed_items is a cache; a few over + * cap is harmless, dropping a wanted item is not). */ -export async function pruneFeedItems( +function actorCutoffDelete( + db: Database, + actor: string, + collection: string, + cap: number +): Statement { + // Plain `?` placeholders (bound repeatedly) rather than numbered params, so + // the Postgres adapter's positional `?`→`$n` rewrite stays correct. + return db + .prepare( + `DELETE FROM feed_items + WHERE actor = ? AND collection = ? + AND time_us < ( + SELECT time_us FROM feed_items + WHERE actor = ? AND collection = ? + ORDER BY time_us DESC LIMIT 1 OFFSET ? + )` + ) + .bind(actor, collection, actor, collection, Math.max(0, cap - 1)); +} + +/** Prune a single actor's feed for one collection to `cap`. Bounded O(cap). */ +export async function pruneActorFeed( db: Database, - caps: number | Map<string, number> + actor: string, + collection: string, + cap: number ): Promise<number> { - if (typeof caps === "number") { - const result = await db - .prepare( - `DELETE FROM feed_items WHERE (actor, uri) NOT IN ( - SELECT actor, uri FROM ( - SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn - FROM feed_items - ) sub WHERE rn <= ? - )` - ) - .bind(caps) - .run(); - return (result as any)?.changes ?? 0; + const result = await actorCutoffDelete(db, actor, collection, cap).run(); + return (result as any)?.changes ?? 0; +} + +export interface FeedSweepResult { + /** Rows deleted this slice. */ + pruned: number; + /** Actor to resume after; null once a full pass completed (wrap to start). */ + nextCursor: string | null; + /** True when this slice reached the end of the actor list. */ + done: boolean; +} + +/** + * One bounded slice of a rolling feed-items prune. + * + * Pages at most `actorBudget` distinct actors (resuming after `cursor`, via the + * feed_items (actor, uri) PK) and applies the per-(actor, collection) cutoff + * delete for every cap in `caps`. Every issued statement is index-backed and + * O(cap), so the slice's per-query CPU stays flat no matter how large + * feed_items grows — the property the old global window query lacked. + * + * Drive it across ticks with a persisted cursor (see getFeedPruneCursor): + * feed back `nextCursor` until `done`, at which point the cursor wraps to null + * and the next pass starts from the beginning. Because each pass visits every + * actor, this doubles as the recovery path for an already-bloated table. + */ +export async function sweepFeedItems( + db: Database, + caps: Map<string, number>, + cursor: string | null, + actorBudget: number +): Promise<FeedSweepResult> { + if (caps.size === 0 || actorBudget <= 0) { + return { pruned: 0, nextCursor: null, done: true }; + } + + const actorsRes = cursor + ? await db + .prepare( + "SELECT DISTINCT actor FROM feed_items WHERE actor > ? ORDER BY actor LIMIT ?" + ) + .bind(cursor, actorBudget) + .all<{ actor: string }>() + : await db + .prepare("SELECT DISTINCT actor FROM feed_items ORDER BY actor LIMIT ?") + .bind(actorBudget) + .all<{ actor: string }>(); + + const actors = (actorsRes.results ?? []).map((r) => r.actor); + if (actors.length === 0) { + // Ran off the end (cursor pointed past the last actor) — wrap next tick. + return { pruned: 0, nextCursor: null, done: true }; + } + + const stmts: Statement[] = []; + for (const actor of actors) { + for (const [collection, cap] of caps) { + stmts.push(actorCutoffDelete(db, actor, collection, cap)); + } + } + + let pruned = 0; + for (let i = 0; i < stmts.length; i += SWEEP_BATCH_SIZE) { + const results = await db.batch(stmts.slice(i, i + SWEEP_BATCH_SIZE)); + for (const r of results) pruned += (r as any)?.changes ?? 0; } + + // A short page means we exhausted the actor list this slice. + const done = actors.length < actorBudget; + return { pruned, nextCursor: done ? null : actors[actors.length - 1], done }; +} + +/** + * Prune the ENTIRE feed_items table to the per-collection `caps` by looping the + * bounded {@link sweepFeedItems} until a full pass completes. + * + * Every statement is O(cap) and safe against D1's per-query CPU limit, but the + * statement count is O(distinct actors), so keep this OFF the hot ingest path — + * the cron/persistent loops issue a single bounded slice per tick instead. Use + * it for one-shot recovery or admin tooling. + */ +export async function pruneFeedItems( + db: Database, + caps: Map<string, number> +): Promise<number> { let total = 0; - for (const [collection, cap] of caps) { - const result = await db - .prepare( - `DELETE FROM feed_items WHERE collection = ? AND (actor, uri) NOT IN ( - SELECT actor, uri FROM ( - SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn - FROM feed_items WHERE collection = ? - ) sub WHERE rn <= ? - )` - ) - .bind(collection, collection, cap) - .run(); - total += (result as any)?.changes ?? 0; + let cursor: string | null = null; + for (;;) { + const res = await sweepFeedItems(db, caps, cursor, FEED_PRUNE_RECOVERY_BATCH); + total += res.pruned; + if (res.done) break; + cursor = res.nextCursor; } return total; } +// --- Feed prune cursor --- + +/** Last actor swept by the rolling feed prune; null = start of a fresh pass. */ +export async function getFeedPruneCursor(db: Database): Promise<string | null> { + const row = await db + .prepare("SELECT actor FROM feed_prune_cursor WHERE id = 1") + .first<{ actor: string | null }>(); + return row?.actor ?? null; +} + +export async function saveFeedPruneCursor( + db: Database, + actor: string | null +): Promise<void> { + await db + .prepare( + "INSERT INTO feed_prune_cursor (id, actor) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET actor = excluded.actor" + ) + .bind(actor) + .run(); +} + // --- Cursor --- export async function getLastCursor(db: Database): Promise<number | null> { diff --git a/packages/contrail-appview/src/core/db/schema.ts b/packages/contrail-appview/src/core/db/schema.ts index df93e72..7089827 100644 --- a/packages/contrail-appview/src/core/db/schema.ts +++ b/packages/contrail-appview/src/core/db/schema.ts @@ -338,6 +338,11 @@ function buildFeedTables(config: ContrailConfig, dialect: SqlDialect): string[] )`, `CREATE INDEX IF NOT EXISTS idx_feed_actor_coll_time ON feed_items(actor, collection, time_us DESC)`, `CREATE INDEX IF NOT EXISTS idx_feed_actor_time ON feed_items(actor, time_us DESC)`, + // Single-row cursor for the rolling, bounded feed prune (see sweepFeedItems). + `CREATE TABLE IF NOT EXISTS feed_prune_cursor ( + id INTEGER PRIMARY KEY CHECK (id = 1), + actor TEXT + )`, `CREATE TABLE IF NOT EXISTS feed_backfills ( actor TEXT NOT NULL, feed TEXT NOT NULL, diff --git a/packages/contrail-appview/src/core/jetstream.ts b/packages/contrail-appview/src/core/jetstream.ts index 486d398..ee5cab3 100644 --- a/packages/contrail-appview/src/core/jetstream.ts +++ b/packages/contrail-appview/src/core/jetstream.ts @@ -6,22 +6,28 @@ import { shortNameForNsid, buildFeedTargetCaps, } from "./types"; -import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; +import { initSchema, getLastCursor, saveCursor, applyEvents, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./db"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; const BATCH_SIZE = 50; -const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour +/** Distinct actors pruned per ingest tick by the rolling feed sweep. Each + * actor costs a handful of index-backed O(cap) deletes, so this bounds the + * prune's per-tick CPU regardless of how large feed_items grows. */ +export const FEED_PRUNE_SWEEP_ACTORS = 500; /** Mutable state that persists across ingest cycles within the same process. */ export interface IngestState { cachedKnownDids?: Set<string>; schemaInitialized: boolean; - lastFeedPruneMs: number; + /** Wall-clock of the last feed sweep — used only by the long-lived + * persistent loop to throttle; the recycling cron isolate sweeps every + * tick and relies on the persisted cursor instead. */ + lastFeedSweepMs: number; } export function createIngestState(): IngestState { - return { schemaInitialized: false, lastFeedPruneMs: 0 }; + return { schemaInitialized: false, lastFeedSweepMs: 0 }; } function getLogger(config: ContrailConfig): Logger { @@ -336,15 +342,25 @@ export async function runIngestCycle( } } - // Prune feed items hourly, per-target so high-volume targets don't - // squeeze out lower-volume ones. - if (config.feeds && Date.now() - s.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { + // Prune feed_items to per-collection caps with a bounded, cursored sweep. + // Every statement is index-backed and O(cap) (see sweepFeedItems), so it can + // never exhaust D1's per-query CPU budget and reset the shared DO — unlike + // the old global window+anti-join. The cron isolate recycles each tick, so we + // persist the sweep cursor in the DB rather than gating on in-memory time, + // and run an unconditional bounded slice every tick. + if (config.feeds) { const caps = buildFeedTargetCaps(config); if (caps.size > 0) { - const pruned = await pruneFeedItems(db, caps); - if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); + const cursor = await getFeedPruneCursor(db); + const { pruned, nextCursor } = await sweepFeedItems( + db, + caps, + cursor, + FEED_PRUNE_SWEEP_ACTORS + ); + await saveFeedPruneCursor(db, nextCursor); + if (pruned > 0) log.log(`Pruned ${pruned} feed items (sweep)`); } - s.lastFeedPruneMs = Date.now(); } log.log(`[ingest] cycle complete. stored=${events.length}`); diff --git a/packages/contrail-appview/src/core/persistent.ts b/packages/contrail-appview/src/core/persistent.ts index 94ca666..63a3ff0 100644 --- a/packages/contrail-appview/src/core/persistent.ts +++ b/packages/contrail-appview/src/core/persistent.ts @@ -7,13 +7,16 @@ import { resolveConfig, shortNameForNsid, } from "./types"; -import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; +import { initSchema, getLastCursor, saveCursor, applyEvents, sweepFeedItems, getFeedPruneCursor, saveFeedPruneCursor } from "./db"; import { refreshStaleIdentities, applyIdentityEvent } from "./identity"; import { backfillFollowersFromConstellation } from "./constellation"; -import { createIngestState } from "./jetstream"; +import { createIngestState, FEED_PRUNE_SWEEP_ACTORS } from "./jetstream"; import type { IngestState } from "./jetstream"; -const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; +/** How often the long-lived persistent loop runs a bounded feed sweep. The + * process stays resident, so this in-memory throttle is reliable here (unlike + * the recycling cron isolate). */ +const FEED_SWEEP_INTERVAL_MS = 10_000; export interface PersistentIngestOptions { batchSize?: number; @@ -176,13 +179,23 @@ async function streamAndFlush( } } - if (config.feeds && Date.now() - state.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { + // Bounded, cursored feed prune (see sweepFeedItems). This process is + // long-lived, so the in-memory interval is a reliable throttle; the + // cursor is still persisted so progress carries across restarts. + if (config.feeds && Date.now() - state.lastFeedSweepMs > FEED_SWEEP_INTERVAL_MS) { const caps = buildFeedTargetCaps(config); if (caps.size > 0) { - const pruned = await pruneFeedItems(db, caps); - if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); + const cursor = await getFeedPruneCursor(db); + const { pruned, nextCursor } = await sweepFeedItems( + db, + caps, + cursor, + FEED_PRUNE_SWEEP_ACTORS + ); + await saveFeedPruneCursor(db, nextCursor); + if (pruned > 0) log.log(`Pruned ${pruned} feed items (sweep)`); } - state.lastFeedPruneMs = Date.now(); + state.lastFeedSweepMs = Date.now(); } log.log(`Flushed ${batch.length} events. Cursor: ${lastTimeUs}`); diff --git a/packages/contrail/tests/feed-prune.test.ts b/packages/contrail/tests/feed-prune.test.ts new file mode 100644 index 0000000..a1f6e5c --- /dev/null +++ b/packages/contrail/tests/feed-prune.test.ts @@ -0,0 +1,189 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import type { Database, ResolvedContrailConfig } from "../src/core/types"; +import { resolveConfig } from "../src/core/types"; +import { initSchema } from "../src/core/db/schema"; +import { + pruneActorFeed, + sweepFeedItems, + pruneFeedItems, + getFeedPruneCursor, + saveFeedPruneCursor, +} from "../src/core/db/records"; + +const EVENT = "community.lexicon.calendar.event"; +const RSVP = "community.lexicon.calendar.rsvp"; + +// Feeds config: event capped at 2 per actor, rsvp at 3. resolveConfig +// auto-adds the `follow` collection, so initSchema builds feed_items, the +// idx_feed_actor_coll_time index, and feed_prune_cursor. +const CONFIG: ResolvedContrailConfig = resolveConfig({ + namespace: "com.example", + collections: { + event: { collection: EVENT }, + rsvp: { collection: RSVP }, + }, + feeds: { + main: { + targets: [ + { collection: "event", maxItems: 2 }, + { collection: "rsvp", maxItems: 3 }, + ], + }, + }, +}); + +// caps keyed by NSID, matching what buildFeedTargetCaps / the fanout produce. +const CAPS = new Map<string, number>([ + [EVENT, 2], + [RSVP, 3], +]); + +let db: Database; + +async function insertItem( + actor: string, + collection: string, + n: number, + timeUs: number +): Promise<void> { + await db + .prepare( + "INSERT INTO feed_items (actor, uri, collection, time_us) VALUES (?, ?, ?, ?)" + ) + .bind(actor, `at://${actor}/${collection}/${n}`, collection, timeUs) + .run(); +} + +/** Insert `count` items for (actor, collection) with increasing time_us. */ +async function seed( + actor: string, + collection: string, + count: number +): Promise<void> { + for (let i = 0; i < count; i++) { + await insertItem(actor, collection, i, 1000 + i); + } +} + +async function rows( + actor: string, + collection: string +): Promise<number[]> { + const res = await db + .prepare( + "SELECT time_us FROM feed_items WHERE actor = ? AND collection = ? ORDER BY time_us DESC" + ) + .bind(actor, collection) + .all<{ time_us: number }>(); + return (res.results ?? []).map((r) => Number(r.time_us)); +} + +beforeEach(async () => { + db = createSqliteDatabase(":memory:"); + await initSchema(db, CONFIG); +}); + +describe("pruneActorFeed", () => { + it("keeps the newest `cap` rows and deletes the rest", async () => { + await seed("alice", EVENT, 5); // time_us 1000..1004 + const deleted = await pruneActorFeed(db, "alice", EVENT, 2); + expect(deleted).toBe(3); + expect(await rows("alice", EVENT)).toEqual([1004, 1003]); + }); + + it("is a no-op when the actor is at or under cap", async () => { + await seed("bob", EVENT, 2); + expect(await pruneActorFeed(db, "bob", EVENT, 2)).toBe(0); + expect(await pruneActorFeed(db, "bob", EVENT, 5)).toBe(0); + expect((await rows("bob", EVENT)).length).toBe(2); + }); + + it("only touches the named collection", async () => { + await seed("alice", EVENT, 4); + await seed("alice", RSVP, 4); + await pruneActorFeed(db, "alice", EVENT, 2); + expect((await rows("alice", EVENT)).length).toBe(2); + expect((await rows("alice", RSVP)).length).toBe(4); // untouched + }); +}); + +describe("sweepFeedItems", () => { + it("prunes every actor to the per-collection caps in one pass", async () => { + await seed("alice", EVENT, 5); + await seed("alice", RSVP, 6); + await seed("bob", EVENT, 1); + await seed("carol", RSVP, 10); + + const res = await sweepFeedItems(db, CAPS, null, 100); + + expect(res.done).toBe(true); + expect(res.nextCursor).toBeNull(); + expect(res.pruned).toBe(3 + 3 + 0 + 7); // alice event/rsvp, bob, carol + expect((await rows("alice", EVENT)).length).toBe(2); + expect((await rows("alice", RSVP)).length).toBe(3); + expect((await rows("bob", EVENT)).length).toBe(1); + expect((await rows("carol", RSVP)).length).toBe(3); + }); + + it("pages by actor and resumes via the cursor", async () => { + // Three actors, each over the event cap. + for (const a of ["a-actor", "b-actor", "c-actor"]) await seed(a, EVENT, 5); + + // Budget of 1 actor per slice: first slice handles "a-actor". + const s1 = await sweepFeedItems(db, CAPS, null, 1); + expect(s1.done).toBe(false); + expect(s1.nextCursor).toBe("a-actor"); + expect((await rows("a-actor", EVENT)).length).toBe(2); + expect((await rows("b-actor", EVENT)).length).toBe(5); // not yet reached + + const s2 = await sweepFeedItems(db, CAPS, s1.nextCursor, 1); + expect(s2.nextCursor).toBe("b-actor"); + expect((await rows("b-actor", EVENT)).length).toBe(2); + + const s3 = await sweepFeedItems(db, CAPS, s2.nextCursor, 1); + // Last actor — still a full page, so not yet flagged done. + expect(s3.nextCursor).toBe("c-actor"); + expect((await rows("c-actor", EVENT)).length).toBe(2); + + // One more slice runs off the end and wraps. + const s4 = await sweepFeedItems(db, CAPS, s3.nextCursor, 1); + expect(s4.done).toBe(true); + expect(s4.nextCursor).toBeNull(); + expect(s4.pruned).toBe(0); + }); + + it("returns done with no work for empty caps", async () => { + await seed("alice", EVENT, 5); + const res = await sweepFeedItems(db, new Map(), null, 100); + expect(res).toEqual({ pruned: 0, nextCursor: null, done: true }); + expect((await rows("alice", EVENT)).length).toBe(5); + }); +}); + +describe("feed prune cursor", () => { + it("round-trips and defaults to null", async () => { + expect(await getFeedPruneCursor(db)).toBeNull(); + await saveFeedPruneCursor(db, "did:plc:xyz"); + expect(await getFeedPruneCursor(db)).toBe("did:plc:xyz"); + await saveFeedPruneCursor(db, null); + expect(await getFeedPruneCursor(db)).toBeNull(); + }); +}); + +describe("pruneFeedItems (full recovery loop)", () => { + it("brings an already-bloated table within caps in one call", async () => { + // Many actors well over cap — the bloated-table recovery scenario. + for (let i = 0; i < 25; i++) { + await seed(`actor-${String(i).padStart(2, "0")}`, EVENT, 8); + await seed(`actor-${String(i).padStart(2, "0")}`, RSVP, 8); + } + const total = await pruneFeedItems(db, CAPS); + expect(total).toBe(25 * (6 + 5)); // event: 8→2, rsvp: 8→3 + + const remaining = await db + .prepare("SELECT COUNT(*) AS c FROM feed_items") + .first<{ c: number }>(); + expect(Number(remaining?.c)).toBe(25 * (2 + 3)); + }); +}); -- 2.51.2 From 162bf96a6d660d19c3f408b0d11eba1c6d642058 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 7 Jun 2026 17:58:15 +0200 Subject: [PATCH 24/25] add test --- .../tests/feed-prune-guardrail.test.ts | 243 ++++++++++++++++++ 1 file changed, 243 insertions(+) create mode 100644 packages/contrail/tests/feed-prune-guardrail.test.ts diff --git a/packages/contrail/tests/feed-prune-guardrail.test.ts b/packages/contrail/tests/feed-prune-guardrail.test.ts new file mode 100644 index 0000000..a538902 --- /dev/null +++ b/packages/contrail/tests/feed-prune-guardrail.test.ts @@ -0,0 +1,243 @@ +import { describe, it, expect, beforeEach } from "vitest"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import type { + Database, + Statement, + ResolvedContrailConfig, + IngestEvent, +} from "../src/core/types"; +import { resolveConfig } from "../src/core/types"; +import { initSchema } from "../src/core/db/schema"; +import { + applyEvents, + sweepFeedItems, + pruneActorFeed, + pruneFeedItems, +} from "../src/core/db/records"; + +// --------------------------------------------------------------------------- +// Guardrail: no contrail-issued maintenance statement may be unbounded-O(n) +// over a table. A single full-table SCAN can exhaust D1's per-query CPU budget +// and reset the shared Durable Object, which kills every concurrent read on the +// same SQLite instance (the feed-prune outage). This test exercises the real +// prune + feed-fanout code, captures every SQL it issues, and asserts each one +// is index-bounded via EXPLAIN QUERY PLAN. +// --------------------------------------------------------------------------- + +const EVENT = "community.lexicon.calendar.event"; +const RSVP = "community.lexicon.calendar.rsvp"; +const FOLLOW = "app.bsky.graph.follow"; + +const CONFIG: ResolvedContrailConfig = resolveConfig({ + namespace: "com.example", + collections: { + event: { collection: EVENT }, + rsvp: { collection: RSVP }, + }, + feeds: { + main: { + targets: [ + { collection: "event", maxItems: 2 }, + { collection: "rsvp", maxItems: 3 }, + ], + }, + }, +}); + +const CAPS = new Map<string, number>([ + [EVENT, 2], + [RSVP, 3], +]); + +/** Wrap a Database so every SQL string passed to prepare() is recorded, while + * delegating to the real DB so the exercised code still reads/writes data. */ +function recordingDb(real: Database): { db: Database; sqls: string[] } { + const sqls: string[] = []; + const db: Database = { + prepare(sql: string): Statement { + sqls.push(sql); + return real.prepare(sql); + }, + batch(stmts: Statement[]): Promise<any[]> { + return real.batch(stmts); + }, + dialect: real.dialect, + }; + return { db, sqls }; +} + +/** Return the EXPLAIN QUERY PLAN `detail` lines for a statement. Params are + * irrelevant to the plan, so we bind dummy values to satisfy the placeholders. */ +async function queryPlan(db: Database, sql: string): Promise<string[]> { + const placeholders = (sql.match(/\?/g) ?? []).length; + const binds = Array.from({ length: placeholders }, () => 1); + const res = await db + .prepare("EXPLAIN QUERY PLAN " + sql) + .bind(...binds) + .all<{ detail: string }>(); + return (res.results ?? []).map((r) => r.detail); +} + +/** + * Plan lines that represent an UNBOUNDED full-table scan: a `SCAN <table>` that + * is not driven by an index. Index SEARCHes and LIMIT-bounded index SCANs are + * fine — their cost is keyed/bounded, not proportional to the whole table. + */ +function unboundedScans(plan: string[]): string[] { + return plan.filter( + (d) => /^SCAN\b/i.test(d.trim()) && !/\bINDEX\b/i.test(d) + ); +} + +/** Statements with no rows in their plan (e.g. INSERT ... VALUES) read nothing. */ +async function assertAllBounded(db: Database, sqls: string[]): Promise<void> { + for (const sql of [...new Set(sqls)]) { + const plan = await queryPlan(db, sql); + const bad = unboundedScans(plan); + expect( + bad, + `Unbounded full-table scan in maintenance SQL:\n ${sql}\n plan: ${plan.join(" | ")}` + ).toEqual([]); + } +} + +function makeFollowRow(db: Database, follower: string, subject: string) { + return db + .prepare( + "INSERT INTO records_follow (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?)" + ) + .bind( + `at://${follower}/${FOLLOW}/${subject}`, + follower, + subject, + "bafyfollow", + JSON.stringify({ subject }), + 1000, + 1000 + ) + .run(); +} + +let real: Database; + +beforeEach(async () => { + real = createSqliteDatabase(":memory:"); + await initSchema(real, CONFIG); +}); + +describe("feed maintenance stays index-bounded", () => { + it("sweepFeedItems issues only index-bounded statements", async () => { + // Several actors over cap so the sweep actually deletes. + for (const a of ["did:plc:a", "did:plc:b", "did:plc:c"]) { + for (let i = 0; i < 6; i++) { + await real + .prepare( + "INSERT INTO feed_items (actor, uri, collection, time_us) VALUES (?, ?, ?, ?)" + ) + .bind(a, `at://${a}/e/${i}`, EVENT, 1000 + i) + .run(); + } + } + + const { db, sqls } = recordingDb(real); + const res = await sweepFeedItems(db, CAPS, null, 100); + expect(res.pruned).toBeGreaterThan(0); + expect(sqls.length).toBeGreaterThan(0); + await assertAllBounded(real, sqls); + }); + + it("pruneActorFeed / pruneFeedItems issue only index-bounded statements", async () => { + for (let i = 0; i < 8; i++) { + await real + .prepare( + "INSERT INTO feed_items (actor, uri, collection, time_us) VALUES (?, ?, ?, ?)" + ) + .bind("did:plc:z", `at://did:plc:z/e/${i}`, EVENT, 2000 + i) + .run(); + } + + const a = recordingDb(real); + await pruneActorFeed(a.db, "did:plc:z", EVENT, 2); + await assertAllBounded(real, a.sqls); + + const b = recordingDb(real); + await pruneFeedItems(b.db, CAPS); + await assertAllBounded(real, b.sqls); + }); + + it("feed fan-out on a target create issues only index-bounded statements", async () => { + // A follower pointing at the event's author, so the fan-out has work. + await makeFollowRow(real, "did:plc:follower", "did:plc:author"); + + const { db, sqls } = recordingDb(real); + const event: IngestEvent = { + uri: "at://did:plc:author/" + EVENT + "/evt1", + did: "did:plc:author", + collection: EVENT, + rkey: "evt1", + cid: "bafyevt", + record: JSON.stringify({ name: "Party", startsAt: "2026-04-01T10:00:00Z" }), + time_us: 5000, + indexed_at: 5000, + operation: "create", + }; + await applyEvents(db, [event], CONFIG); + + // The fan-out INSERT must have been issued and it must hit the table. + const fanout = sqls.find((s) => /INSERT.*feed_items/is.test(s)); + expect(fanout, "expected a feed_items fan-out INSERT").toBeTruthy(); + expect((await real.prepare("SELECT COUNT(*) AS c FROM feed_items").first<{ c: number }>())?.c).toBe(1); + + await assertAllBounded(real, sqls); + }); + + it("the fan-out follower lookup uses idx_follow_subject (not a full scan)", async () => { + await makeFollowRow(real, "did:plc:follower", "did:plc:author"); + + const { db, sqls } = recordingDb(real); + await applyEvents( + db, + [ + { + uri: "at://did:plc:author/" + EVENT + "/evt2", + did: "did:plc:author", + collection: EVENT, + rkey: "evt2", + cid: "bafyevt2", + record: JSON.stringify({ name: "x" }), + time_us: 6000, + indexed_at: 6000, + operation: "create", + }, + ], + CONFIG + ); + + const fanout = sqls.find((s) => /records_follow/is.test(s))!; + const plan = (await queryPlan(real, fanout)).join(" | "); + expect(plan).toMatch(/idx_follow_subject/i); + }); +}); + +describe("the guardrail has teeth", () => { + it("rejects the old global window + anti-join prune", async () => { + // The original pruneFeedItems statement that reset the D1 DO in production. + const oldGlobalPrune = `DELETE FROM feed_items WHERE collection = ? AND (actor, uri) NOT IN ( + SELECT actor, uri FROM ( + SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn + FROM feed_items WHERE collection = ? + ) sub WHERE rn <= ? + )`; + const plan = await queryPlan(real, oldGlobalPrune); + // It must trip the guard with at least one full-table SCAN of feed_items. + expect(unboundedScans(plan).length).toBeGreaterThan(0); + }); + + it("flags a contrived unindexed scan", async () => { + const plan = await queryPlan( + real, + "SELECT * FROM feed_items WHERE time_us = ?" + ); + expect(unboundedScans(plan).length).toBeGreaterThan(0); + }); +}); -- 2.51.2 From 7780dab49fa2fef732b6f961af73ae7aeef68522 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Sun, 7 Jun 2026 15:59:51 +0000 Subject: [PATCH 25/25] Version Packages --- .changeset/feed-prune-bounded-sweep.md | 34 ------------------ packages/contrail-appview/CHANGELOG.md | 41 ++++++++++++++++++++++ packages/contrail-appview/package.json | 2 +- packages/contrail-authority/CHANGELOG.md | 6 ++++ packages/contrail-authority/package.json | 2 +- packages/contrail-base/CHANGELOG.md | 2 ++ packages/contrail-base/package.json | 2 +- packages/contrail-community/CHANGELOG.md | 7 ++++ packages/contrail-community/package.json | 2 +- packages/contrail-record-host/CHANGELOG.md | 6 ++++ packages/contrail-record-host/package.json | 2 +- packages/contrail/CHANGELOG.md | 10 ++++++ packages/contrail/package.json | 2 +- packages/lexicons/CHANGELOG.md | 6 ++++ packages/lexicons/package.json | 2 +- packages/sync/CHANGELOG.md | 2 ++ packages/sync/package.json | 2 +- 17 files changed, 88 insertions(+), 42 deletions(-) delete mode 100644 .changeset/feed-prune-bounded-sweep.md diff --git a/.changeset/feed-prune-bounded-sweep.md b/.changeset/feed-prune-bounded-sweep.md deleted file mode 100644 index e459924..0000000 --- a/.changeset/feed-prune-bounded-sweep.md +++ /dev/null @@ -1,34 +0,0 @@ ---- -"@atmo-dev/contrail-appview": minor ---- - -fix(feeds): make feed_items pruning bounded so it can't reset the D1 DO - -The hourly feed prune ran a single global `ROW_NUMBER() OVER (PARTITION BY actor)` -window + `(actor, uri) NOT IN (...)` anti-join across the entire `feed_items` -table — O(n) CPU in one statement. Once the table grew large this exceeded D1's -per-query CPU limit and reset the shared Durable Object, taking down any -concurrent read on the same SQLite instance (unrelated user requests 500'd with -`was reset` / `Network connection lost`). Because the statement reset before -completing, caps were never enforced, the table kept growing, and the prune got -more expensive — a death spiral. - -Changes: - -- **Bounded per-actor prune.** Pruning is now an index-backed cutoff delete per - `(actor, collection)` using `idx_feed_actor_coll_time`, cost O(cap), never - O(table). New `pruneActorFeed` / `sweepFeedItems` exports; the ingest loops - run one bounded `sweepFeedItems` slice per tick (`FEED_PRUNE_SWEEP_ACTORS` - actors), which also serves as recovery for already-bloated tables. -- **Persisted prune cursor.** A new `feed_prune_cursor` row tracks the rolling - sweep position, so progress survives the cron isolate recycling that - previously made the in-memory hourly gate a no-op (it pruned on essentially - every tick). The time gate is removed from the cron path; the long-lived - persistent loop keeps a short in-memory throttle. -- **API:** `pruneFeedItems(db, caps)` now accepts only the per-collection - `Map<collection, cap>` (the legacy global-number form is removed) and is - reimplemented as a bounded full-table recovery loop — keep it off the hot - path. - -The follow fan-out's `subject` lookup is already covered by `idx_<follow>_subject`, -so no unbounded statement remains in the ingest path. diff --git a/packages/contrail-appview/CHANGELOG.md b/packages/contrail-appview/CHANGELOG.md index 371296c..c1b9cbe 100644 --- a/packages/contrail-appview/CHANGELOG.md +++ b/packages/contrail-appview/CHANGELOG.md @@ -1,5 +1,46 @@ # @atmo-dev/contrail-appview +## 0.9.0 + +### Minor Changes + +- 8f0b87e: fix(feeds): make feed_items pruning bounded so it can't reset the D1 DO + + The hourly feed prune ran a single global `ROW_NUMBER() OVER (PARTITION BY actor)` + window + `(actor, uri) NOT IN (...)` anti-join across the entire `feed_items` + table — O(n) CPU in one statement. Once the table grew large this exceeded D1's + per-query CPU limit and reset the shared Durable Object, taking down any + concurrent read on the same SQLite instance (unrelated user requests 500'd with + `was reset` / `Network connection lost`). Because the statement reset before + completing, caps were never enforced, the table kept growing, and the prune got + more expensive — a death spiral. + + Changes: + + - **Bounded per-actor prune.** Pruning is now an index-backed cutoff delete per + `(actor, collection)` using `idx_feed_actor_coll_time`, cost O(cap), never + O(table). New `pruneActorFeed` / `sweepFeedItems` exports; the ingest loops + run one bounded `sweepFeedItems` slice per tick (`FEED_PRUNE_SWEEP_ACTORS` + actors), which also serves as recovery for already-bloated tables. + - **Persisted prune cursor.** A new `feed_prune_cursor` row tracks the rolling + sweep position, so progress survives the cron isolate recycling that + previously made the in-memory hourly gate a no-op (it pruned on essentially + every tick). The time gate is removed from the cron path; the long-lived + persistent loop keeps a short in-memory throttle. + - **API:** `pruneFeedItems(db, caps)` now accepts only the per-collection + `Map<collection, cap>` (the legacy global-number form is removed) and is + reimplemented as a bounded full-table recovery loop — keep it off the hot + path. + + The follow fan-out's `subject` lookup is already covered by `idx_<follow>_subject`, + so no unbounded statement remains in the ingest path. + +### Patch Changes + +- @atmo-dev/contrail-base@0.9.0 +- @atmo-dev/contrail-authority@0.9.0 +- @atmo-dev/contrail-record-host@0.9.0 + ## 0.8.0 ### Minor Changes diff --git a/packages/contrail-appview/package.json b/packages/contrail-appview/package.json index d1eedd2..7720226 100644 --- a/packages/contrail-appview/package.json +++ b/packages/contrail-appview/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-appview", - "version": "0.8.0", + "version": "0.9.0", "description": "Public-records appview for contrail — jetstream ingestion, backfill, query layer, feeds, labels, profiles, per-collection XRPC routes.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-authority/CHANGELOG.md b/packages/contrail-authority/CHANGELOG.md index bccf371..1766bb5 100644 --- a/packages/contrail-authority/CHANGELOG.md +++ b/packages/contrail-authority/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail-authority +## 0.9.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.9.0 + ## 0.8.0 ### Patch Changes diff --git a/packages/contrail-authority/package.json b/packages/contrail-authority/package.json index 3b2f6de..1e1f7ee 100644 --- a/packages/contrail-authority/package.json +++ b/packages/contrail-authority/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-authority", - "version": "0.8.0", + "version": "0.9.0", "description": "Default space-authority implementation for contrail — member list, invites, app policy, credential issuance. Contrail's binary-membership ACL flavor; for ladder-style access levels see @atmo-dev/contrail-community.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-base/CHANGELOG.md b/packages/contrail-base/CHANGELOG.md index 4c6e069..db1878d 100644 --- a/packages/contrail-base/CHANGELOG.md +++ b/packages/contrail-base/CHANGELOG.md @@ -1,5 +1,7 @@ # @atmo-dev/contrail-base +## 0.9.0 + ## 0.8.0 ### Minor Changes diff --git a/packages/contrail-base/package.json b/packages/contrail-base/package.json index a97edb4..d39fef0 100644 --- a/packages/contrail-base/package.json +++ b/packages/contrail-base/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-base", - "version": "0.8.0", + "version": "0.9.0", "description": "Shared infrastructure for the contrail family of packages — interfaces (SpaceAuthority, RecordHost, CommunityIntegration), credential primitives, binding resolvers, realtime infra, schema scaffolding. No routes, no tables of its own.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-community/CHANGELOG.md b/packages/contrail-community/CHANGELOG.md index ffe7196..3e55e9d 100644 --- a/packages/contrail-community/CHANGELOG.md +++ b/packages/contrail-community/CHANGELOG.md @@ -1,5 +1,12 @@ # @atmo-dev/contrail-community +## 0.9.0 + +### Patch Changes + +- @atmo-dev/contrail@0.9.0 +- @atmo-dev/contrail-base@0.9.0 + ## 0.8.0 ### Minor Changes diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index a5afdb6..9b21bd6 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-community", - "version": "0.8.0", + "version": "0.9.0", "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-record-host/CHANGELOG.md b/packages/contrail-record-host/CHANGELOG.md index 5dad428..07161ca 100644 --- a/packages/contrail-record-host/CHANGELOG.md +++ b/packages/contrail-record-host/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail-record-host +## 0.9.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.9.0 + ## 0.8.0 ### Patch Changes diff --git a/packages/contrail-record-host/package.json b/packages/contrail-record-host/package.json index 2beced2..e20fd4c 100644 --- a/packages/contrail-record-host/package.json +++ b/packages/contrail-record-host/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-record-host", - "version": "0.8.0", + "version": "0.9.0", "description": "Default record-host implementation for contrail — stores records and blobs for permissioned spaces, enforces local enrollment as the host's consent layer.", "type": "module", "sideEffects": false, diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index d50a581..b82bcb8 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,5 +1,15 @@ # @atmo-dev/contrail +## 0.9.0 + +### Patch Changes + +- Updated dependencies [8f0b87e] + - @atmo-dev/contrail-appview@0.9.0 + - @atmo-dev/contrail-base@0.9.0 + - @atmo-dev/contrail-authority@0.9.0 + - @atmo-dev/contrail-record-host@0.9.0 + ## 0.8.0 ### Minor Changes diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 88158b2..e784ff8 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.8.0", + "version": "0.9.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, diff --git a/packages/lexicons/CHANGELOG.md b/packages/lexicons/CHANGELOG.md index a51f8b5..63c0055 100644 --- a/packages/lexicons/CHANGELOG.md +++ b/packages/lexicons/CHANGELOG.md @@ -1,5 +1,11 @@ # @atmo-dev/contrail-lexicons +## 0.4.9 + +### Patch Changes + +- @atmo-dev/contrail@0.9.0 + ## 0.4.8 ### Patch Changes diff --git a/packages/lexicons/package.json b/packages/lexicons/package.json index c35fcb9..3759eca 100644 --- a/packages/lexicons/package.json +++ b/packages/lexicons/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-lexicons", - "version": "0.4.8", + "version": "0.4.9", "description": "Generate atproto lexicon JSON (and optionally TypeScript types via @atcute/lex-cli) from a Contrail config.", "type": "module", "files": [ diff --git a/packages/sync/CHANGELOG.md b/packages/sync/CHANGELOG.md index 444bc23..ad9a3de 100644 --- a/packages/sync/CHANGELOG.md +++ b/packages/sync/CHANGELOG.md @@ -1,5 +1,7 @@ # @atmo-dev/contrail-sync +## 0.9.0 + ## 0.8.0 ## 0.7.0 diff --git a/packages/sync/package.json b/packages/sync/package.json index d914e2a..02f533c 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-sync", - "version": "0.8.0", + "version": "0.9.0", "description": "Client-side reactive watch-store over contrail's watchRecords endpoints. SSE + WebSocket transports, optimistic updates, optional IndexedDB cache.", "type": "module", "sideEffects": false,