diff --git a/apps/group-chat/scripts/generate.ts b/apps/group-chat/scripts/generate.ts index c29853a..e6cf38f 100644 --- a/apps/group-chat/scripts/generate.ts +++ b/apps/group-chat/scripts/generate.ts @@ -11,7 +11,10 @@ const ROOT_DIR = join(dirname(fileURLToPath(import.meta.url)), '..'); // per-collection endpoints. Values are stubs — only the shape matters for codegen. const configForGen = { ...baseConfig, - spaces: { type: 'tools.atmo.chat.space', serviceDid: 'did:web:localhost' }, + spaces: { + authority: { type: 'tools.atmo.chat.space', serviceDid: 'did:web:localhost' }, + recordHost: {} + }, community: { masterKey: new Uint8Array(32), serviceDid: 'did:web:localhost' }, realtime: { ticketSecret: new Uint8Array(32) } }; diff --git a/apps/group-chat/src/lib/contrail/index.ts b/apps/group-chat/src/lib/contrail/index.ts index a75157a..85a6aa3 100644 --- a/apps/group-chat/src/lib/contrail/index.ts +++ b/apps/group-chat/src/lib/contrail/index.ts @@ -43,12 +43,16 @@ function build(env: Env): Bundle { const config: ContrailConfig = { ...baseConfig, spaces: { - type: 'tools.atmo.chat.space', - serviceDid: env.SERVICE_DID, - blobs: { - adapter: blobAdapter, - maxSize: 2 * 1024 * 1024, - accept: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'] + authority: { + type: 'tools.atmo.chat.space', + serviceDid: env.SERVICE_DID + }, + recordHost: { + blobs: { + adapter: blobAdapter, + maxSize: 2 * 1024 * 1024, + accept: ['image/png', 'image/jpeg', 'image/webp', 'image/gif'] + } } }, community: { diff --git a/packages/contrail/src/core/community/router.ts b/packages/contrail/src/core/community/router.ts index 2897ec0..84e86d1 100644 --- a/packages/contrail/src/core/community/router.ts +++ b/packages/contrail/src/core/community/router.ts @@ -45,8 +45,8 @@ export function registerCommunityRoutes( ): void { const cfg = config.community; if (!cfg) return; - if (!config.spaces) { - throw new Error("community module requires spaces to be enabled in config"); + if (!config.spaces?.authority) { + throw new Error("community module requires spaces.authority to be enabled in config"); } const community = @@ -64,8 +64,8 @@ export function registerCommunityRoutes( })(); const NS = `${config.namespace}.community`; - const spaceType = config.spaces.type; - const spaceServiceDid = cfg.serviceDid ?? config.spaces.serviceDid; + const spaceType = config.spaces.authority.type; + const spaceServiceDid = cfg.serviceDid ?? config.spaces.authority.serviceDid; // ========================================================================== // Community lifecycle diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 3f3c676..5ef78e8 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -307,7 +307,7 @@ export async function initSchema( await db.batch(all.map((s) => db.prepare(s))); - if (config.spaces) { + if (config.spaces?.authority || config.spaces?.recordHost) { await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); } diff --git a/packages/contrail/src/core/invite/router.ts b/packages/contrail/src/core/invite/router.ts index 0b9a4fc..440bc16 100644 --- a/packages/contrail/src/core/invite/router.ts +++ b/packages/contrail/src/core/invite/router.ts @@ -78,7 +78,7 @@ export function registerInviteRoutes( community: CommunityAdapter | null, options: InviteRoutesOptions ): void { - if (!config.spaces) return; + if (!config.spaces?.authority) return; const NS = `${config.namespace}.invite`; const auth = options.authMiddleware; diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index f5e6808..8c10854 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -115,14 +115,17 @@ export function createApp( // Shared spaces context — verifier + adapter — reused by both the per-collection // routes (for `?spaceUri=...` dispatch) and the `.space.*` routes. + // Built when an authority is configured (spaces are gated on the authority, + // not the record host — a record-host-only deployment still needs an + // authority somewhere, just possibly external). const spacesDb = options.spacesDb ?? db; let spacesCtx: SpacesContext | null = options.spacesCtx !== undefined ? options.spacesCtx - : config.spaces + : config.spaces?.authority ? { adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), - verifier: buildVerifier(config.spaces), + verifier: buildVerifier(config.spaces.authority), } : null; @@ -179,7 +182,7 @@ export function createApp( ); } - if (config.spaces && spacesCtx) { + if (config.spaces?.authority && spacesCtx) { // Unified invite surface: one `.invite.*` family that dispatches on // space ownership (user-owned → addMember; community-owned → grant). const authMiddleware = diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 4ad723b..85ba814 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -7,16 +7,17 @@ import { type DidDocumentResolver, } from "@atcute/identity-resolver"; import type { Did, Nsid } from "@atcute/lexicons"; -import type { SpacesConfig } from "./types"; +import type { AuthorityConfig } from "./types"; import { readInProcess } from "./in-process"; export { ServiceJwtVerifier }; -/** Build a ServiceJwtVerifier from a SpacesConfig, using the configured - * resolver or a default PLC+Web composite. */ -export function buildVerifier(spaces: SpacesConfig): ServiceJwtVerifier { +/** Build a ServiceJwtVerifier from an AuthorityConfig, using the configured + * resolver or a default PLC+Web composite. The verifier checks that incoming + * JWTs target this authority's serviceDid (aud claim). */ +export function buildVerifier(authority: AuthorityConfig): ServiceJwtVerifier { const resolver = - spaces.resolver ?? + authority.resolver ?? new CompositeDidDocumentResolver({ methods: { plc: new PlcDidDocumentResolver(), @@ -24,7 +25,7 @@ export function buildVerifier(spaces: SpacesConfig): ServiceJwtVerifier { }, }); return new ServiceJwtVerifier({ - serviceDid: spaces.serviceDid as Did, + serviceDid: authority.serviceDid as Did, resolver, }); } diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index e3fa1c7..f3e3fd5 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -8,7 +8,6 @@ import { checkInviteReadGrant, createServiceAuthMiddleware, extractInviteToken, - verifyServiceAuthRequest, } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; @@ -16,22 +15,29 @@ import { resolveEffectiveLevel } from "../community/acl"; import { buildSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, + type AuthorityConfig, + type RecordHostConfig, + type RecordHost, + type SpaceAuthority, type SpaceRow, - type SpacesConfig, type StorageAdapter, } from "./types"; import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; import { create as createCid, toString as cidToString } from "@atcute/cid"; -import type { Did } from "@atcute/lexicons"; export interface SpacesRoutesOptions { - /** Provide a custom middleware (e.g. for tests). If omitted and spaces.resolver is set, a real one is built. */ + /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ authMiddleware?: MiddlewareHandler; /** Storage adapter override. Defaults to HostedAdapter(db). */ adapter?: StorageAdapter; } +/** Umbrella registration: wires both the authority and the record-host + * routes against the same adapter. Today's deployments enable both via + * `config.spaces.authority` and `config.spaces.recordHost`. Either may be + * omitted in future split deployments — phase 5 lifts the assumption that + * one process runs both. */ export function registerSpacesRoutes( app: Hono, db: Database, @@ -42,45 +48,30 @@ export function registerSpacesRoutes( ): void { const spacesConfig = config.spaces; if (!spacesConfig) return; + const authorityConfig = spacesConfig.authority; + if (!authorityConfig) return; const adapter = options.adapter ?? ctx?.adapter ?? new HostedAdapter(db, config); - const verifier = ctx?.verifier ?? buildVerifier(spacesConfig); + const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is - * present so anonymous bearer reads don't 401 before the route handler can - * validate the token. The route handler is responsible for actually checking - * the token (via `authorizeRead`). */ - const readAuth: MiddlewareHandler = async (c, next) => { - if (extractInviteToken(c.req.raw)) { - await next(); - return; - } - return auth(c, next); - }; + registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, community ?? null); - /** Authorize a read request: either a valid service-auth JWT (which also - * identifies the caller for member checks downstream) or a valid read-grant - * invite token bearer (`?inviteToken=...` or - * `Authorization: Bearer atmo-invite:`). */ - async function authorizeRead( - c: Context, - spaceUri: string - ): Promise<{ via: "token" } | { via: "jwt"; sa: ServiceAuth } | Response> { - const rawToken = extractInviteToken(c.req.raw); - if (rawToken) { - const ok = await checkInviteReadGrant(adapter, rawToken, spaceUri, hashInviteToken); - if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); - return { via: "token" }; - } - const sa = c.get("serviceAuth") as ServiceAuth | undefined; - if (sa) return { via: "jwt", sa }; - return c.json( - { error: "AuthRequired", message: "JWT or read-grant invite token required" }, - 401 - ); + if (spacesConfig.recordHost) { + registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth); } +} +/** Register the **space authority** XRPC surface — space lifecycle, member + * list, app policy, whoami. Does NOT touch records or blobs. */ +export function registerAuthorityRoutes( + app: Hono, + authority: SpaceAuthority, + authorityConfig: AuthorityConfig, + config: ContrailConfig, + auth: MiddlewareHandler, + community: import("../community/adapter").CommunityAdapter | null +): void { /** Space endpoints are emitted per-deployment under the configured namespace; * the deployment owns and publishes its own lexicons. The library ships * templates at `lexicon-templates/spaces/*` that the generator instantiates @@ -89,7 +80,8 @@ export function registerSpacesRoutes( const SPACE = `${config.namespace}.space`; const SPACE_EXT = `${config.namespace}.spaceExt`; - // Read endpoints + // ---- Read endpoints ---- + app.get(`/xrpc/${SPACE}.listSpaces`, auth, async (c) => { const sa = getAuth(c); const scope = c.req.query("scope") ?? "member"; // "member" | "owner" @@ -98,14 +90,14 @@ export function registerSpacesRoutes( const cursor = c.req.query("cursor") ?? undefined; const limit = c.req.query("limit") ? Number(c.req.query("limit")) : undefined; - const opts: Parameters[0] = { type, cursor, limit }; + const opts: Parameters[0] = { type, cursor, limit }; if (scope === "owner") opts.ownerDid = sa.issuer; else { opts.memberDid = sa.issuer; if (owner) opts.ownerDid = owner; // narrow to spaces owned by this DID } - const result = await adapter.listSpaces(opts); + const result = await authority.listSpaces(opts); return c.json({ spaces: result.spaces.map((s) => publicSpaceView(s, s.ownerDid === sa.issuer)), cursor: result.cursor, @@ -116,25 +108,36 @@ export function registerSpacesRoutes( const sa = getAuth(c); const spaceUri = c.req.query("spaceUri"); if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); const isOwner = space.ownerDid === sa.issuer; - const member = isOwner ? null : await adapter.getMember(spaceUri, sa.issuer); + const member = isOwner ? null : await authority.getMember(spaceUri, sa.issuer); if (!isOwner && !member) { return c.json({ error: "Forbidden", reason: "not-member" }, 403); } - const members = await adapter.listMembers(spaceUri); + const members = await authority.listMembers(spaceUri); return c.json({ members }); }); + /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is + * present so anonymous bearer reads don't 401 before the route handler can + * validate the token. */ + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return auth(c, next); + }; + app.get(`/xrpc/${SPACE}.getSpace`, readAuth, async (c) => { const uri = c.req.query("uri"); if (!uri) return c.json({ error: "InvalidRequest", message: "uri required" }, 400); - const space = await adapter.getSpace(uri); + const space = await authority.getSpace(uri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, uri); + const authz = await authorizeRead(c, authority, uri); if (authz instanceof Response) return authz; if (authz.via === "token") { @@ -144,28 +147,173 @@ export function registerSpacesRoutes( const sa = authz.sa; const isOwner = sa.issuer === space.ownerDid; - const member = isOwner ? null : await adapter.getMember(uri, sa.issuer); + const member = isOwner ? null : await authority.getMember(uri, sa.issuer); if (!isOwner && !member) { return c.json({ error: "Forbidden", reason: "not-member" }, 403); } return c.json({ space: publicSpaceView(space, isOwner) }); }); + // ---- Space management (owner-gated) ---- + + app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => ({}))) as { + type?: string; + key?: string; + appPolicy?: SpaceRow["appPolicy"]; + appPolicyRef?: string; + }; + + const type = body.type ?? authorityConfig.type; + const key = body.key ?? nextTid(); + const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); + + const existing = await authority.getSpace(uri); + if (existing) return c.json({ error: "AlreadyExists", uri }, 409); + + const space = await authority.createSpace({ + uri, + ownerDid: sa.issuer, + type, + key, + serviceDid: authorityConfig.serviceDid, + appPolicyRef: body.appPolicyRef ?? null, + appPolicy: body.appPolicy ?? authorityConfig.defaultAppPolicy ?? null, + }); + // Owner is implicit; we still write a row so membership queries are uniform. + await authority.addMember(uri, sa.issuer, sa.issuer); + + return c.json({ space: publicSpaceView(space, true) }); + }); + + app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + await authority.addMember(body.spaceUri, body.did, sa.issuer); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + if (body.did === space.ownerDid) { + return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); + } + await authority.removeMember(body.spaceUri, body.did); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid === sa.issuer) { + return c.json( + { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, + 400 + ); + } + await authority.removeMember(body.spaceUri, sa.issuer); + return c.json({ ok: true }); + }); + + // Unified whoami — `.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, + // accessLevel? }. `accessLevel` is present only when the target space is + // community-owned; for user-owned spaces membership is binary. + app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + + // Community-owned space: resolve through the access-level ladder. The + // reconciler keeps spaces_members in sync, so isMember derives from the + // effective level directly. + const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; + if (isCommunity) { + const level = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); + return c.json({ + isOwner, + isMember: isOwner || !!level, + accessLevel: level, + }); + } + + // User-owned space: binary membership. + if (isOwner) return c.json({ isOwner: true, isMember: true }); + const member = await authority.getMember(spaceUri, sa.issuer); + return c.json({ isOwner: false, isMember: !!member }); + }); +} + +/** Register the **record host** XRPC surface — record + blob CRUD. Today + * consults the authority for membership / app-policy checks at write time; + * phase 3 adds a credential verifier that replaces those calls in + * split-deployment configurations. */ +export function registerRecordHostRoutes( + app: Hono, + recordHost: RecordHost, + authority: SpaceAuthority, + recordHostConfig: RecordHostConfig, + config: ContrailConfig, + auth: MiddlewareHandler +): void { + const SPACE = `${config.namespace}.space`; + + /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is + * present so anonymous bearer reads don't 401 before the route handler can + * validate the token. */ + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return auth(c, next); + }; + app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { const spaceUri = c.req.query("spaceUri"); const collection = c.req.query("collection"); if (!spaceUri || !collection) { return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, spaceUri); + const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", space, @@ -178,7 +326,7 @@ export function registerSpacesRoutes( } } - const list = await adapter.listRecords(spaceUri, collection, { + const list = await recordHost.listRecords(spaceUri, collection, { byUser: c.req.query("byUser") ?? undefined, cursor: c.req.query("cursor") ?? undefined, limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, @@ -194,15 +342,15 @@ export function registerSpacesRoutes( if (!spaceUri || !collection || !author || !rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, spaceUri); + const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", space, @@ -214,7 +362,7 @@ export function registerSpacesRoutes( if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); } - const record = await adapter.getRecord(spaceUri, collection, author, rkey); + const record = await recordHost.getRecord(spaceUri, collection, author, rkey); if (!record) return c.json({ error: "NotFound" }, 404); return c.json({ record }); }); @@ -228,10 +376,10 @@ export function registerSpacesRoutes( if (!body?.spaceUri || !body.collection || !body.record) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); } - const space = await adapter.getSpace(body.spaceUri); + const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(body.spaceUri, sa.issuer); + const member = await authority.getMember(body.spaceUri, sa.issuer); const result = checkAccess({ op: "write", space, @@ -245,10 +393,10 @@ export function registerSpacesRoutes( // uploaded into this space. This mirrors how PDSes require uploadBlob // before putRecord, and prevents forging refs to blobs the caller never // actually claimed. - if (spacesConfig.blobs) { + if (recordHostConfig.blobs) { const cids = collectBlobCids(body.record); for (const cid of cids) { - const meta = await adapter.getBlobMeta(body.spaceUri, cid); + const meta = await recordHost.getBlobMeta(body.spaceUri, cid); if (!meta) { return c.json( { @@ -264,7 +412,7 @@ export function registerSpacesRoutes( const rkey = body.rkey ?? nextTid(); const now = Date.now(); - await adapter.putRecord({ + await recordHost.putRecord({ spaceUri: body.spaceUri, collection: body.collection, authorDid: sa.issuer, @@ -284,10 +432,10 @@ export function registerSpacesRoutes( if (!body?.spaceUri || !body.collection || !body.rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); } - const space = await adapter.getSpace(body.spaceUri); + const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(body.spaceUri, sa.issuer); + const member = await authority.getMember(body.spaceUri, sa.issuer); const result = checkAccess({ op: "delete", space, @@ -298,13 +446,13 @@ export function registerSpacesRoutes( }); if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - await adapter.deleteRecord(body.spaceUri, body.collection, sa.issuer, body.rkey); + await recordHost.deleteRecord(body.spaceUri, body.collection, sa.issuer, body.rkey); return c.json({ ok: true }); }); // Blobs (only registered when a blob adapter is configured) - if (spacesConfig.blobs) { - const blobsCfg = spacesConfig.blobs; + if (recordHostConfig.blobs) { + const blobsCfg = recordHostConfig.blobs; const blobAdapter = blobsCfg.adapter; const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; const accept = blobsCfg.accept; @@ -315,10 +463,10 @@ export function registerSpacesRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "write", space, @@ -360,7 +508,7 @@ export function registerSpacesRoutes( const key = await blobKey(spaceUri, cidString); await blobAdapter.put(key, bytes, { mimeType, size: bytes.byteLength }); - await adapter.putBlobMeta({ + await recordHost.putBlobMeta({ spaceUri, cid: cidString, mimeType, @@ -385,15 +533,15 @@ export function registerSpacesRoutes( if (!spaceUri || !cid) { return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const authz = await authorizeRead(c, spaceUri); + const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "read", space, @@ -406,7 +554,7 @@ export function registerSpacesRoutes( } } - const meta = await adapter.getBlobMeta(spaceUri, cid); + const meta = await recordHost.getBlobMeta(spaceUri, cid); if (!meta) return c.json({ error: "NotFound" }, 404); const key = await blobKey(spaceUri, cid); const bytes = await blobAdapter.get(key); @@ -426,10 +574,10 @@ export function registerSpacesRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await adapter.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await adapter.getMember(spaceUri, sa.issuer); + const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "read", space, @@ -441,7 +589,7 @@ export function registerSpacesRoutes( return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); } - const result = await adapter.listBlobMeta(spaceUri, { + const result = await recordHost.listBlobMeta(spaceUri, { byUser: c.req.query("byUser") ?? undefined, cursor: c.req.query("cursor") ?? undefined, limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, @@ -449,132 +597,28 @@ export function registerSpacesRoutes( return c.json(result); }); } - - // Space management (owner-gated) - app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => ({}))) as { - type?: string; - key?: string; - appPolicy?: SpaceRow["appPolicy"]; - appPolicyRef?: string; - }; - - const type = body.type ?? spacesConfig.type; - const key = body.key ?? nextTid(); - const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); - - const existing = await adapter.getSpace(uri); - if (existing) return c.json({ error: "AlreadyExists", uri }, 409); - - const space = await adapter.createSpace({ - uri, - ownerDid: sa.issuer, - type, - key, - serviceDid: spacesConfig.serviceDid, - appPolicyRef: body.appPolicyRef ?? null, - appPolicy: body.appPolicy ?? spacesConfig.defaultAppPolicy ?? null, - }); - // Owner is implicit; we still write a row so membership queries are uniform. - await adapter.addMember(uri, sa.issuer, sa.issuer); - - return c.json({ space: publicSpaceView(space, true) }); - }); - - // Invites live under `.invite.*` — see src/core/invite/router.ts. The - // unified surface dispatches on space ownership. - - app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await adapter.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - await adapter.addMember(body.spaceUri, body.did, sa.issuer); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await adapter.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - if (body.did === space.ownerDid) { - return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); - } - await adapter.removeMember(body.spaceUri, body.did); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const space = await adapter.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid === sa.issuer) { - return c.json( - { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, - 400 - ); - } - await adapter.removeMember(body.spaceUri, sa.issuer); - return c.json({ ok: true }); - }); - - // Unified whoami — `.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, - // accessLevel? }. `accessLevel` is present only when the target space is - // community-owned; for user-owned spaces membership is binary. - app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await adapter.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - - // Community-owned space: resolve through the access-level ladder. The - // reconciler keeps spaces_members in sync, so isMember derives from the - // effective level directly. - const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; - if (isCommunity) { - const level = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); - return c.json({ - isOwner, - isMember: isOwner || !!level, - accessLevel: level, - }); - } - - // User-owned space: binary membership. - if (isOwner) return c.json({ isOwner: true, isMember: true }); - const member = await adapter.getMember(spaceUri, sa.issuer); - return c.json({ isOwner: false, isMember: !!member }); - }); } -function buildAuthMiddleware(spaces: SpacesConfig): MiddlewareHandler { - const verifier = buildVerifier(spaces); - return createServiceAuthMiddleware(verifier); +/** Authorize a read request: either a valid service-auth JWT (which also + * identifies the caller for member checks downstream) or a valid read-grant + * invite token bearer. */ +async function authorizeRead( + c: Context, + authority: SpaceAuthority, + spaceUri: string +): Promise<{ via: "token" } | { via: "jwt"; sa: ServiceAuth } | Response> { + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); + if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); + return { via: "token" }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (sa) return { via: "jwt", sa }; + return c.json( + { error: "AuthRequired", message: "JWT or read-grant invite token required" }, + 401 + ); } function getAuth(c: Parameters[0]): ServiceAuth { @@ -583,7 +627,6 @@ function getAuth(c: Parameters[0]): ServiceAuth { return auth; } - function publicSpaceView(space: SpaceRow, forOwner: boolean) { return { uri: space.uri, @@ -596,4 +639,3 @@ function publicSpaceView(space: SpaceRow, forOwner: boolean) { ...(forOwner ? { appPolicy: space.appPolicy } : {}), }; } - diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 397e6a8..fb05d2a 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -25,8 +25,13 @@ export interface SpacesBlobsConfig { export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; -export interface SpacesConfig { - /** NSID that identifies the kind of space this service hosts, e.g. "tools.atmo.event.space". */ +/** Configuration for the **space authority** role: holds the member list, + * signs credentials (later phases), and gates space-management operations. + * In a fully-split deployment, the authority can run in a different process + * (or even a different operator) than the record host. */ +export interface AuthorityConfig { + /** NSID that identifies the kind of space this authority hosts, + * e.g. "tools.atmo.event.space". */ type: string; /** Service DID that service-auth tokens must target (aud claim). */ serviceDid: string; @@ -35,10 +40,29 @@ export interface SpacesConfig { /** DID document resolver for service-auth JWT verification. * Defaults to a composite PLC + did:web resolver if omitted. */ resolver?: DidDocumentResolver; +} + +/** Configuration for the **record host** role: stores per-space records and + * blobs and serves reads. Verifies space credentials (later phases) on + * incoming traffic. */ +export interface RecordHostConfig { /** Blob-upload backend. When omitted, blob XRPCs are not exposed. */ blobs?: SpacesBlobsConfig; } +/** Spaces config — host an authority, a record host, or both. + * Today both run in one process and most deployments will set both; the + * shape is split now so phase 5 can run them independently without churning + * every consumer's config. */ +export interface SpacesConfig { + /** Space-authority config — member list, credentials (later), space + * management. Required for any space to exist. */ + authority?: AuthorityConfig; + /** Record-host config — record + blob storage. Required for records to be + * written/read on this deployment. */ + recordHost?: RecordHostConfig; +} + export interface SpaceRow { uri: string; ownerDid: string; @@ -145,7 +169,13 @@ export interface ListBlobsResult { cursor?: string; } -export interface StorageAdapter { +/** **Space authority** interface — owner of the space's ACL state and + * (eventually) credential issuer. Holds the member list, manages invites, + * governs space lifecycle and app policy. Does NOT touch records or blobs. + * + * In a fully-split deployment this is a separate service; today the + * HostedAdapter implements both this and {@link RecordHost} against one DB. */ +export interface SpaceAuthority { // Space lifecycle createSpace(space: Omit): Promise; getSpace(spaceUri: string): Promise; @@ -167,7 +197,7 @@ export interface StorageAdapter { addedBy: string | null ): Promise; - // Invites + // Invites (token primitive — issued by the authority, scoped to a space) createInvite(input: CreateInviteInput): Promise; listInvites(spaceUri: string, options?: { includeRevoked?: boolean }): Promise; revokeInvite(tokenHash: string): Promise; @@ -176,7 +206,16 @@ export interface StorageAdapter { /** Atomically mark a join-capable invite as used. Returns the row if usable * (kind allows join, not expired/revoked/exhausted), null otherwise. */ redeemInvite(tokenHash: string, now: number): Promise; +} +/** **Record host** interface — stores records and blobs for a space. Trusts + * the authority's ACL: in later phases, validates incoming credentials + * rather than calling out for a member check. + * + * Read-side note: today the record host needs `getSpace` to know the space + * exists at all (and for app-policy checks before permissioned writes). When + * enrollment lands (phase 5), this becomes a local lookup instead. */ +export interface RecordHost { // Records putRecord(record: StoredRecord): Promise; getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise; @@ -194,6 +233,12 @@ export interface StorageAdapter { findOrphanBlobs(spaceUri: string, cutoff: number, limit: number): Promise; } +/** Combined adapter. Used internally where a single object satisfies both + * roles (today's HostedAdapter, the community reconciler, the realtime + * publishing wrapper). Phases 5+ replace consumers of this with two + * injected interfaces. */ +export type StorageAdapter = SpaceAuthority & RecordHost; + export interface AdapterContext { db: Database; } diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index 53239d0..b895141 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -390,9 +390,9 @@ export function validateConfig(config: ContrailConfig): void { } } - if (config.community && !config.spaces) { + if (config.community && !config.spaces?.authority) { throw new Error( - "Invalid config: `community` requires `spaces`. Community-owned spaces reuse the spaces storage adapter." + "Invalid config: `community` requires `spaces.authority`. Community-owned spaces reuse the spaces storage adapter." ); } } diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 1e0c0cf..8d678cb 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -39,6 +39,10 @@ export type { PersistentIngestOptions } from "./core/persistent"; // Spaces export type { SpacesConfig, + AuthorityConfig, + RecordHostConfig, + SpaceAuthority, + RecordHost, AppPolicy, AppPolicyMode, SpaceRow, diff --git a/packages/contrail/tests/community-delegation.test.ts b/packages/contrail/tests/community-delegation.test.ts index d602f86..1514964 100644 --- a/packages/contrail/tests/community-delegation.test.ts +++ b/packages/contrail/tests/community-delegation.test.ts @@ -20,8 +20,11 @@ const CONFIG: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -68,7 +71,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/community-e2e.test.ts b/packages/contrail/tests/community-e2e.test.ts index 79e0a64..db008e6 100644 --- a/packages/contrail/tests/community-e2e.test.ts +++ b/packages/contrail/tests/community-e2e.test.ts @@ -22,8 +22,11 @@ const CONFIG: ContrailConfig = { message: { collection: "app.event.message" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -77,7 +80,7 @@ function fakeAuth(): MiddlewareHandler { if (!did) return c.json({ error: "AuthRequired" }, 401); c.set("serviceAuth", { issuer: did, - audience: CONFIG.spaces!.serviceDid, + audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined, }); await next(); diff --git a/packages/contrail/tests/community-mint.test.ts b/packages/contrail/tests/community-mint.test.ts index 1f1b6a7..c83491f 100644 --- a/packages/contrail/tests/community-mint.test.ts +++ b/packages/contrail/tests/community-mint.test.ts @@ -27,8 +27,11 @@ const CONFIG: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -52,7 +55,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/community-publishing.test.ts b/packages/contrail/tests/community-publishing.test.ts index 5ab42c3..10a3d76 100644 --- a/packages/contrail/tests/community-publishing.test.ts +++ b/packages/contrail/tests/community-publishing.test.ts @@ -22,8 +22,11 @@ const CONFIG: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -82,7 +85,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/invite-unified.test.ts b/packages/contrail/tests/invite-unified.test.ts index a3892ef..b56d81d 100644 --- a/packages/contrail/tests/invite-unified.test.ts +++ b/packages/contrail/tests/invite-unified.test.ts @@ -23,7 +23,10 @@ const MASTER_KEY = new Uint8Array(32).fill(11); const CONFIG: ContrailConfig = { namespace: "test.inv", collections: { message: { collection: "app.event.message" } }, - spaces: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + recordHost: {}, + }, community: { masterKey: MASTER_KEY, fetch: mockFetch, @@ -55,7 +58,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/realtime-e2e.test.ts b/packages/contrail/tests/realtime-e2e.test.ts index 7a3d1ec..59e9338 100644 --- a/packages/contrail/tests/realtime-e2e.test.ts +++ b/packages/contrail/tests/realtime-e2e.test.ts @@ -19,8 +19,11 @@ const CONFIG: ContrailConfig = { namespace: "test.rt", collections: { message: { collection: "app.event.message" } }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, community: { masterKey: MASTER_KEY, @@ -61,7 +64,7 @@ function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); if (!did) return c.json({ error: "AuthRequired" }, 401); - c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.serviceDid, lxm: undefined }); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); await next(); }; } diff --git a/packages/contrail/tests/spaces-blobs.test.ts b/packages/contrail/tests/spaces-blobs.test.ts index 63af7a5..f4c4dee 100644 --- a/packages/contrail/tests/spaces-blobs.test.ts +++ b/packages/contrail/tests/spaces-blobs.test.ts @@ -21,9 +21,13 @@ function makeConfig(blobs: MemoryBlobAdapter, maxSize = 2 * 1024 * 1024): Contra photo: { collection: "app.event.photo" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", - blobs: { adapter: blobs, maxSize }, + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: { + blobs: { adapter: blobs, maxSize }, + }, }, }; } @@ -51,7 +55,7 @@ async function makeApp( const resolved = resolveConfig(cfg); await initSchema(db, resolved); const app = createApp(db, resolved, { - spaces: { authMiddleware: fakeAuth(cfg.spaces!.serviceDid) }, + spaces: { authMiddleware: fakeAuth(cfg.spaces!.authority!.serviceDid) }, }); return { app, db, config: resolved }; } diff --git a/packages/contrail/tests/spaces-e2e.test.ts b/packages/contrail/tests/spaces-e2e.test.ts index e69c29b..e70f9ec 100644 --- a/packages/contrail/tests/spaces-e2e.test.ts +++ b/packages/contrail/tests/spaces-e2e.test.ts @@ -19,8 +19,11 @@ const CONFIG: ContrailConfig = { ticket: { collection: "app.event.ticket" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, }; @@ -31,7 +34,7 @@ function fakeAuth(): MiddlewareHandler { if (!did) return c.json({ error: "AuthRequired" }, 401); c.set("serviceAuth", { issuer: did, - audience: CONFIG.spaces!.serviceDid, + audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined, clientId: c.req.header("X-Test-App") ?? undefined, }); diff --git a/packages/contrail/tests/spaces-invites.test.ts b/packages/contrail/tests/spaces-invites.test.ts index eb3fc1c..d4e0406 100644 --- a/packages/contrail/tests/spaces-invites.test.ts +++ b/packages/contrail/tests/spaces-invites.test.ts @@ -18,8 +18,11 @@ const CONFIG: ContrailConfig = { message: { collection: "app.event.message" }, }, spaces: { - type: "tools.atmo.event.space", - serviceDid: "did:web:test.example#svc", + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, }, }; @@ -29,7 +32,7 @@ function fakeAuth(): MiddlewareHandler { if (!did) return c.json({ error: "AuthRequired" }, 401); c.set("serviceAuth", { issuer: did, - audience: CONFIG.spaces!.serviceDid, + audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined, }); await next(); -- 2.51.2 From 44194b2497ac28a6a514fe0d074648cb813ceb37 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:03:18 +0200 Subject: [PATCH 02/19] phase two --- packages/contrail/src/core/community/index.ts | 2 + .../src/core/community/invite-handler.ts | 155 ++++++++++++++++ .../contrail/src/core/community/reconcile.ts | 11 +- .../contrail/src/core/community/whoami.ts | 26 +++ .../src/core/invite/community-handler.ts | 67 +++++++ packages/contrail/src/core/invite/router.ts | 175 ++++++------------ packages/contrail/src/core/router/index.ts | 44 +++-- packages/contrail/src/core/spaces/router.ts | 45 +++-- 8 files changed, 371 insertions(+), 154 deletions(-) create mode 100644 packages/contrail/src/core/community/invite-handler.ts create mode 100644 packages/contrail/src/core/community/whoami.ts create mode 100644 packages/contrail/src/core/invite/community-handler.ts diff --git a/packages/contrail/src/core/community/index.ts b/packages/contrail/src/core/community/index.ts index 3fbe1ad..fd0cc53 100644 --- a/packages/contrail/src/core/community/index.ts +++ b/packages/contrail/src/core/community/index.ts @@ -21,6 +21,8 @@ export { export { CredentialCipher } from "./credentials"; export { resolveEffectiveLevel, flattenEffectiveMembers, wouldCycle } from "./acl"; export { reconcile } from "./reconcile"; +export { createCommunityInviteHandler } from "./invite-handler"; +export { createCommunityWhoamiExtension } from "./whoami"; export { initCommunitySchema, buildCommunitySchema } from "./schema"; export { resolveIdentity, createPdsSession } from "./pds"; export { diff --git a/packages/contrail/src/core/community/invite-handler.ts b/packages/contrail/src/core/community/invite-handler.ts new file mode 100644 index 0000000..0b41f8a --- /dev/null +++ b/packages/contrail/src/core/community/invite-handler.ts @@ -0,0 +1,155 @@ +/** Implementation of {@link CommunityInviteHandler} for community-grant + * invites. Lives here (not in invite/) so the dependency edge points + * community → invite (downward), not the other way around. */ + +import type { + CommunityInviteHandler, + HandlerResponse, +} from "../invite/community-handler"; +import { mintInviteToken } from "../invite/token"; +import type { CommunityAdapter } from "./adapter"; +import type { SpaceAuthority } from "../spaces/types"; +import { resolveEffectiveLevel } from "./acl"; +import { reconcile } from "./reconcile"; +import type { AccessLevel, CommunityInviteRow } from "./types"; +import { isAccessLevel, rankOf } from "./types"; + +interface PublicInviteView { + tokenHash: string; + spaceUri: string; + accessLevel: AccessLevel; + createdBy: string; + createdAt: number; + expiresAt: number | null; + maxUses: number | null; + usedCount: number; + revokedAt: number | null; + note: string | null; +} + +function toView(row: CommunityInviteRow): PublicInviteView { + return { + tokenHash: row.tokenHash, + spaceUri: row.spaceUri, + accessLevel: row.accessLevel, + createdBy: row.createdBy, + createdAt: row.createdAt, + expiresAt: row.expiresAt, + maxUses: row.maxUses, + usedCount: row.usedCount, + revokedAt: row.revokedAt, + note: row.note, + }; +} + +const ok = (body: Record): HandlerResponse => ({ status: 200, body }); +const err = (status: number, body: Record): HandlerResponse => ({ status, body }); + +export function createCommunityInviteHandler(args: { + community: CommunityAdapter; + /** Space authority — used to look up space metadata after redemption (e.g. + * to return the community DID). */ + authority: SpaceAuthority; +}): CommunityInviteHandler { + const { community, authority } = args; + + return { + async isCommunityOwned(spaceUri) { + const space = await authority.getSpace(spaceUri); + if (!space) return false; + return !!(await community.getCommunity(space.ownerDid)); + }, + + async create(input) { + if (input.kind) { + return err(400, { + error: "InvalidRequest", + reason: "kind-on-community-space", + message: "community spaces take accessLevel, not kind", + }); + } + if (!input.accessLevel || !isAccessLevel(input.accessLevel)) { + return err(400, { error: "InvalidRequest", reason: "accessLevel-required" }); + } + const callerLevel = await resolveEffectiveLevel(community, input.spaceUri, input.callerDid); + if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { + return err(403, { error: "Forbidden", reason: "manager-required" }); + } + if (rankOf(input.accessLevel) > rankOf(callerLevel)) { + return err(403, { error: "Forbidden", reason: "cannot-grant-higher-than-self" }); + } + const { token, tokenHash } = await mintInviteToken(); + const row = await community.createInvite({ + spaceUri: input.spaceUri, + tokenHash, + accessLevel: input.accessLevel, + createdBy: input.callerDid, + expiresAt: input.expiresAt, + maxUses: input.maxUses, + note: input.note, + }); + return ok({ token, invite: toView(row) }); + }, + + async list(input) { + const callerLevel = await resolveEffectiveLevel(community, input.spaceUri, input.callerDid); + if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { + return err(403, { error: "Forbidden", reason: "manager-required" }); + } + const rows = await community.listInvites(input.spaceUri, { + includeRevoked: input.includeRevoked, + }); + return ok({ invites: rows.map(toView) }); + }, + + async revoke(input) { + const level = await resolveEffectiveLevel(community, input.spaceUri, input.callerDid); + const managerOrHigher = !!level && rankOf(level) >= rankOf("manager"); + if (!managerOrHigher) { + const crow = await community.getInvite(input.tokenHash); + if (!crow || crow.createdBy !== input.callerDid) { + return err(403, { error: "Forbidden", reason: "creator-or-manager-required" }); + } + } + const revoked = await community.revokeInvite(input.tokenHash); + return ok({ ok: revoked }); + }, + + async tryRevokeByToken(input) { + const crow = await community.getInvite(input.tokenHash); + if (!crow) return null; + let allowed = crow.createdBy === input.callerDid; + if (!allowed) { + const level = await resolveEffectiveLevel(community, crow.spaceUri, input.callerDid); + allowed = !!level && rankOf(level) >= rankOf("manager"); + } + if (!allowed) { + return err(403, { error: "Forbidden", reason: "creator-or-manager-required" }); + } + const revoked = await community.revokeInvite(input.tokenHash); + return ok({ ok: revoked }); + }, + + async tryRedeem(input) { + const cinvite = await community.redeemInvite(input.tokenHash, input.now); + if (!cinvite) return null; + const space = await authority.getSpace(cinvite.spaceUri); + if (!space) return err(404, { error: "NotFound", reason: "space-not-found" }); + // The token itself is the authorization: creator (manager+) pre-signed + // "anyone with this token gets level X". Grant directly, attributing + // to the creator so audit trails make sense. + await community.grant({ + spaceUri: cinvite.spaceUri, + subjectDid: input.callerDid, + accessLevel: cinvite.accessLevel, + grantedBy: cinvite.createdBy, + }); + await reconcile(community, authority, cinvite.spaceUri, cinvite.createdBy); + return ok({ + spaceUri: cinvite.spaceUri, + accessLevel: cinvite.accessLevel, + communityDid: space.ownerDid, + }); + }, + }; +} diff --git a/packages/contrail/src/core/community/reconcile.ts b/packages/contrail/src/core/community/reconcile.ts index b9d7e35..5fde1d3 100644 --- a/packages/contrail/src/core/community/reconcile.ts +++ b/packages/contrail/src/core/community/reconcile.ts @@ -1,13 +1,18 @@ -import type { StorageAdapter as SpacesAdapter } from "../spaces/types"; +import type { SpaceAuthority } from "../spaces/types"; import type { CommunityAdapter } from "./adapter"; import { flattenEffectiveMembers } from "./acl"; /** Reconcile `spaces_members` for `spaceUri` to match the flattened effective * member set derived from `community_access_levels`. Also re-reconciles any - * spaces that delegate to this one (reverse-graph). */ + * spaces that delegate to this one (reverse-graph). + * + * Takes a {@link SpaceAuthority} (not a full `StorageAdapter`) — the + * reconciler only needs member-level operations, so depending on the + * narrower interface keeps the dependency direction clean and proves we + * could swap in a non-Contrail authority. */ export async function reconcile( community: CommunityAdapter, - spaces: SpacesAdapter, + spaces: SpaceAuthority, spaceUri: string, byDid: string, opts: { depth?: number; maxReverseDepth?: number } = {} diff --git a/packages/contrail/src/core/community/whoami.ts b/packages/contrail/src/core/community/whoami.ts new file mode 100644 index 0000000..4a75a34 --- /dev/null +++ b/packages/contrail/src/core/community/whoami.ts @@ -0,0 +1,26 @@ +/** Whoami extension that adds `accessLevel` (and the corrected `isMember`) + * for community-owned spaces. Returns null for non-community spaces so the + * spaces module's default binary-membership logic runs. */ + +import type { WhoamiExtension } from "../spaces/router"; +import type { CommunityAdapter } from "./adapter"; +import { resolveEffectiveLevel } from "./acl"; + +export function createCommunityWhoamiExtension(args: { + community: CommunityAdapter; +}): WhoamiExtension { + const { community } = args; + return async ({ spaceUri, callerDid, isOwner, ownerDid }) => { + const isCommunity = !!(await community.getCommunity(ownerDid)); + if (!isCommunity) return null; + + // The reconciler keeps spaces_members in sync with the access-level + // ladder, so isMember derives from the effective level directly. + const level = await resolveEffectiveLevel(community, spaceUri, callerDid); + return { + isOwner, + isMember: isOwner || !!level, + accessLevel: level, + }; + }; +} diff --git a/packages/contrail/src/core/invite/community-handler.ts b/packages/contrail/src/core/invite/community-handler.ts new file mode 100644 index 0000000..1043525 --- /dev/null +++ b/packages/contrail/src/core/invite/community-handler.ts @@ -0,0 +1,67 @@ +/** Pluggable handler for community-grant invites within the unified invite + * surface. The invite router calls into this when the target space is + * community-owned, or "tries" it on the redeem / revoke-without-spaceUri + * paths. Community module provides the impl; invite/router doesn't import + * from community at all. + * + * Each method returns a `HandlerResponse`: a `{status, body}` envelope that + * the router relays as JSON, or `null` (only on the "try" methods) meaning + * "not applicable, fall through to the user-owned path." */ + +export type HandlerResponse = { + status: number; + body: Record; +}; + +export interface CommunityInviteHandler { + /** True iff this space is owned by a community (vs. a regular user DID). + * Used by the invite router to choose the dispatch path on + * create / list / revoke-with-spaceUri. */ + isCommunityOwned(spaceUri: string): Promise; + + /** Create a community-grant invite. Caller is validated upstream for + * having a JWT; this method handles the access-level checks. */ + create(input: { + spaceUri: string; + callerDid: string; + /** Raw caller-supplied access level — implementation validates. */ + accessLevel?: string; + /** Caller-supplied `kind` field — community spaces don't accept this; the + * handler returns an InvalidRequest if set. */ + kind?: string; + expiresAt: number | null; + maxUses: number | null; + note: string | null; + }): Promise; + + /** List invites for a community-owned space. */ + list(input: { + spaceUri: string; + callerDid: string; + includeRevoked: boolean; + }): Promise; + + /** Revoke a known community-owned invite (caller already passed spaceUri + * and the router classified it as community-owned). */ + revoke(input: { + spaceUri: string; + tokenHash: string; + callerDid: string; + }): Promise; + + /** Revoke without a spaceUri — try to find the invite in the community + * table; return null if not a community invite (router falls through). */ + tryRevokeByToken(input: { + tokenHash: string; + callerDid: string; + }): Promise; + + /** Try to redeem a token as a community invite. Returns null if the token + * is not a community invite, in which case the router falls through to + * the user-owned redeem path. */ + tryRedeem(input: { + tokenHash: string; + callerDid: string; + now: number; + }): Promise; +} diff --git a/packages/contrail/src/core/invite/router.ts b/packages/contrail/src/core/invite/router.ts index 440bc16..941af9b 100644 --- a/packages/contrail/src/core/invite/router.ts +++ b/packages/contrail/src/core/invite/router.ts @@ -2,36 +2,31 @@ * user-owned and community-owned spaces. Dispatches on space ownership. * * - User-owned space → `kind` in create, `addMember` on redeem, owner-only. - * - Community-owned → `accessLevel` in create, `grant` on redeem, - * manager+ with "cannot grant higher than self". + * - Community-owned → routed to a {@link CommunityInviteHandler} provided + * by the community module (or null when community is + * not configured). * * Storage stays separate (`spaces_invites` vs `community_invites` tables) — * schemas differ enough that unifying them would be net-negative. The token - * primitive and HTTP dance are shared. */ + * primitive and HTTP dance are shared. invite/router has zero imports from + * community/ — coupling is via the {@link CommunityInviteHandler} interface. */ import type { Context, Hono, MiddlewareHandler } from "hono"; import type { ContrailConfig } from "../types"; import type { ServiceAuth } from "../spaces/auth"; -import type { StorageAdapter } from "../spaces/types"; +import type { SpaceAuthority } from "../spaces/types"; import type { InviteKind, InviteRow } from "../spaces/types"; -import type { CommunityAdapter } from "../community/adapter"; -import type { CommunityInviteRow, AccessLevel } from "../community/types"; -import { isAccessLevel, rankOf } from "../community/types"; import { hashInviteToken, mintInviteToken } from "./token"; -import { resolveEffectiveLevel } from "../community/acl"; -import { reconcile } from "../community/reconcile"; +import type { CommunityInviteHandler, HandlerResponse } from "./community-handler"; export interface InviteRoutesOptions { authMiddleware: MiddlewareHandler; } -/** Shape returned to clients — `kind` (user-owned space) or `accessLevel` - * (community-owned space) is set, never both. */ interface PublicInviteView { tokenHash: string; spaceUri: string; kind?: InviteKind; - accessLevel?: AccessLevel; createdBy: string; createdAt: number; expiresAt: number | null; @@ -56,26 +51,11 @@ function toSpacesView(row: InviteRow): PublicInviteView { }; } -function toCommunityView(row: CommunityInviteRow): PublicInviteView { - return { - tokenHash: row.tokenHash, - spaceUri: row.spaceUri, - accessLevel: row.accessLevel, - createdBy: row.createdBy, - createdAt: row.createdAt, - expiresAt: row.expiresAt, - maxUses: row.maxUses, - usedCount: row.usedCount, - revokedAt: row.revokedAt, - note: row.note, - }; -} - export function registerInviteRoutes( app: Hono, config: ContrailConfig, - spaces: StorageAdapter, - community: CommunityAdapter | null, + authority: SpaceAuthority, + community: CommunityInviteHandler | null, options: InviteRoutesOptions ): void { if (!config.spaces?.authority) return; @@ -86,9 +66,9 @@ export function registerInviteRoutes( /** Resolve whether a space is community-owned. Returns null if the space * doesn't exist. */ const classifySpace = async (spaceUri: string) => { - const space = await spaces.getSpace(spaceUri); + const space = await authority.getSpace(spaceUri); if (!space) return null; - const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; + const isCommunity = community ? await community.isCommunityOwned(spaceUri) : false; return { space, isCommunity }; }; @@ -120,35 +100,15 @@ export function registerInviteRoutes( if (isCommunity) { if (!community) return c.json({ error: "InvalidState" }, 500); - if (body.kind) { - return c.json( - { error: "InvalidRequest", reason: "kind-on-community-space", message: "community spaces take accessLevel, not kind" }, - 400 - ); - } - if (!body.accessLevel || !isAccessLevel(body.accessLevel)) { - return c.json({ error: "InvalidRequest", reason: "accessLevel-required" }, 400); - } - // Caller must have manager+ on the target space and cannot create an - // invite that confers a higher level than their own. - const callerLevel = await resolveEffectiveLevel(community, body.spaceUri, sa.issuer); - if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { - return c.json({ error: "Forbidden", reason: "manager-required" }, 403); - } - if (rankOf(body.accessLevel) > rankOf(callerLevel)) { - return c.json({ error: "Forbidden", reason: "cannot-grant-higher-than-self" }, 403); - } - const { token, tokenHash } = await mintInviteToken(); - const row = await community.createInvite({ + return relay(c, await community.create({ spaceUri: body.spaceUri, - tokenHash, + callerDid: sa.issuer, accessLevel: body.accessLevel, - createdBy: sa.issuer, + kind: body.kind, expiresAt: body.expiresAt ?? null, maxUses: body.maxUses ?? null, note: body.note ?? null, - }); - return c.json({ token, invite: toCommunityView(row) }); + })); } // User-owned space. @@ -166,7 +126,7 @@ export function registerInviteRoutes( return c.json({ error: "InvalidRequest", message: "kind must be 'join', 'read', or 'read-join'" }, 400); } const { token, tokenHash } = await mintInviteToken(); - const invite = await spaces.createInvite({ + const invite = await authority.createInvite({ spaceUri: body.spaceUri, tokenHash, kind, @@ -189,18 +149,17 @@ export function registerInviteRoutes( const { space, isCommunity } = classified; if (isCommunity) { - const callerLevel = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); - if (!callerLevel || rankOf(callerLevel) < rankOf("manager")) { - return c.json({ error: "Forbidden", reason: "manager-required" }, 403); - } - const rows = await community!.listInvites(spaceUri, { includeRevoked }); - return c.json({ invites: rows.map(toCommunityView) }); + return relay(c, await community!.list({ + spaceUri, + callerDid: sa.issuer, + includeRevoked, + })); } if (space.ownerDid !== sa.issuer) { return c.json({ error: "Forbidden", reason: "not-owner" }, 403); } - const rows = await spaces.listInvites(spaceUri, { includeRevoked }); + const rows = await authority.listInvites(spaceUri, { includeRevoked }); return c.json({ invites: rows.map(toSpacesView) }); }); @@ -213,54 +172,39 @@ export function registerInviteRoutes( return c.json({ error: "InvalidRequest", message: "tokenHash required" }, 400); } - // When the caller passes spaceUri we do an auth check up front so the - // response doesn't leak token existence. Community revokers may also be - // the invite creator (even without manager+), which is resolved after. if (body.spaceUri) { const classified = await classifySpace(body.spaceUri); if (!classified) return c.json({ error: "NotFound" }, 404); if (classified.isCommunity) { - const level = await resolveEffectiveLevel(community!, body.spaceUri, sa.issuer); - const managerOrHigher = !!level && rankOf(level) >= rankOf("manager"); - if (!managerOrHigher) { - const crow = await community!.getInvite(body.tokenHash); - if (!crow || crow.createdBy !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "creator-or-manager-required" }, 403); - } - } - const ok = await community!.revokeInvite(body.tokenHash); - return c.json({ ok }); + return relay(c, await community!.revoke({ + spaceUri: body.spaceUri, + tokenHash: body.tokenHash, + callerDid: sa.issuer, + })); } if (classified.space.ownerDid !== sa.issuer) { return c.json({ error: "Forbidden", reason: "not-owner" }, 403); } - const ok = await spaces.revokeInvite(body.tokenHash); + const ok = await authority.revokeInvite(body.tokenHash); return c.json({ ok }); } - // No spaceUri provided — infer from the invite row. + // No spaceUri — try the community handler first (it returns null if the + // token isn't a community invite), then fall back to the user-owned path. if (community) { - const crow = await community.getInvite(body.tokenHash); - if (crow) { - let allowed = crow.createdBy === sa.issuer; - if (!allowed) { - const level = await resolveEffectiveLevel(community, crow.spaceUri, sa.issuer); - allowed = !!level && rankOf(level) >= rankOf("manager"); - } - if (!allowed) { - return c.json({ error: "Forbidden", reason: "creator-or-manager-required" }, 403); - } - const ok = await community.revokeInvite(body.tokenHash); - return c.json({ ok }); - } + const r = await community.tryRevokeByToken({ + tokenHash: body.tokenHash, + callerDid: sa.issuer, + }); + if (r) return relay(c, r); } - const srow = await spaces.getInvite(body.tokenHash); + const srow = await authority.getInvite(body.tokenHash); if (!srow) return c.json({ error: "NotFound" }, 404); - const space = await spaces.getSpace(srow.spaceUri); + const space = await authority.getSpace(srow.spaceUri); if (space && space.ownerDid !== sa.issuer) { return c.json({ error: "Forbidden", reason: "not-owner" }, 403); } - const ok = await spaces.revokeInvite(body.tokenHash); + const ok = await authority.revokeInvite(body.tokenHash); return c.json({ ok }); }); @@ -273,43 +217,32 @@ export function registerInviteRoutes( const tokenHash = await hashInviteToken(body.token); const now = Date.now(); - // Try community first (it's atomic — returns null if not consumable). + // Try community first (atomic — null if not a community invite). if (community) { - const cinvite = await community.redeemInvite(tokenHash, now); - if (cinvite) { - const space = await spaces.getSpace(cinvite.spaceUri); - if (!space) { - return c.json({ error: "NotFound", reason: "space-not-found" }, 404); - } - // The token itself is the authorization: creator (manager+) pre-signed - // "anyone with this token gets level X". Grant directly, attributing - // to the creator so audit trails make sense. - await community.grant({ - spaceUri: cinvite.spaceUri, - subjectDid: sa.issuer, - accessLevel: cinvite.accessLevel, - grantedBy: cinvite.createdBy, - }); - await reconcile(community, spaces, cinvite.spaceUri, cinvite.createdBy); - return c.json({ - spaceUri: cinvite.spaceUri, - accessLevel: cinvite.accessLevel, - communityDid: space.ownerDid, - }); - } + const r = await community.tryRedeem({ + tokenHash, + callerDid: sa.issuer, + now, + }); + if (r) return relay(c, r); } - // Fall back to the spaces (user-owned) path. The spaces redeem filter - // already restricts to `kind IN ('join','read-join')` at the SQL level. - const sinvite = await spaces.redeemInvite(tokenHash, now); + // Fall back to the user-owned spaces path. The redeem filter at the SQL + // level already restricts to `kind IN ('join','read-join')`. + const sinvite = await authority.redeemInvite(tokenHash, now); if (!sinvite) { return c.json({ error: "InvalidInvite", reason: "expired-revoked-or-exhausted" }, 400); } - await spaces.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); + await authority.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); return c.json({ spaceUri: sinvite.spaceUri, kind: sinvite.kind }); }); } +/** Forward a community-handler response to the wire. */ +function relay(c: Context, r: HandlerResponse) { + return c.json(r.body, r.status as Parameters[1]); +} + function getAuth(c: Context): ServiceAuth { const a = c.get("serviceAuth") as ServiceAuth | undefined; if (!a) throw new Error("service auth not set"); diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 8c10854..535c15f 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -15,6 +15,8 @@ import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; import { registerCommunityRoutes } from "../community/router"; import type { CommunityRoutesOptions } from "../community/router"; import { CommunityAdapter } from "../community/adapter"; +import { createCommunityInviteHandler } from "../community/invite-handler"; +import { createCommunityWhoamiExtension } from "../community/whoami"; import { registerRealtimeRoutes } from "../realtime/router"; import type { RealtimeRoutesOptions } from "../realtime/router"; import { registerInviteRoutes } from "../invite/router"; @@ -129,6 +131,12 @@ export function createApp( } : null; + // Community is wired up at this layer, not from inside spaces / invite — + // those modules consume injected hooks, not community internals. The + // adapter is shared across every call site that needs it (publishing + // wrapper, collection routes, whoami extension, invite handler, realtime). + const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; + // Realtime pubsub is built whenever realtime is configured — independent of // spaces. With spaces, the spaces adapter is wrapped so private record/member // events publish to space:/community: topics. Without spaces, only public @@ -141,7 +149,6 @@ export function createApp( queueBound: config.realtime.queueBound, }); if (spacesCtx) { - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; const isCommunityDid = communityAdapter ? cachedIsCommunityDid(communityAdapter) : undefined; @@ -153,19 +160,25 @@ export function createApp( } registerAdminRoutes(app, db, config); - const communityAdapterForCollection = config.community - ? new CommunityAdapter(spacesDb) - : null; + registerCollectionRoutes(app, db, config, spacesCtx, { pubsub: realtimePubsub, - community: communityAdapterForCollection, + community: communityAdapter, }); registerFeedRoutes(app, db, config); registerNotifyRoute(app, db, config); - const communityAdapterForSpaces = config.community && spacesCtx - ? new CommunityAdapter(spacesDb) - : null; - registerSpacesRoutes(app, spacesDb, config, options.spaces, spacesCtx, communityAdapterForSpaces); + + // Spaces routes — get a whoami extension when community is configured so + // community-owned spaces get an `accessLevel` field. + const spacesOptions = { + ...options.spaces, + whoamiExtension: + options.spaces?.whoamiExtension ?? + (communityAdapter + ? createCommunityWhoamiExtension({ community: communityAdapter }) + : undefined), + }; + registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); if (config.community && spacesCtx) { // Community routes reuse the spaces service-auth middleware (same JWT verifier). @@ -184,12 +197,18 @@ export function createApp( if (config.spaces?.authority && spacesCtx) { // Unified invite surface: one `.invite.*` family that dispatches on - // space ownership (user-owned → addMember; community-owned → grant). + // space ownership (user-owned → addMember; community-owned → grant via + // an injected community-invite handler). const authMiddleware = options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier); - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; - registerInviteRoutes(app, config, spacesCtx.adapter, communityAdapter, { authMiddleware }); + const inviteHandler = communityAdapter + ? createCommunityInviteHandler({ + community: communityAdapter, + authority: spacesCtx.adapter, + }) + : null; + registerInviteRoutes(app, config, spacesCtx.adapter, inviteHandler, { authMiddleware }); } if (config.realtime && realtimePubsub) { @@ -202,7 +221,6 @@ export function createApp( options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier) : null; - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; registerRealtimeRoutes(app, config, spacesCtx?.adapter ?? null, communityAdapter, { authMiddleware, pubsub: realtimePubsub, diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index f3e3fd5..4e71c05 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -11,7 +11,6 @@ import { } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; -import { resolveEffectiveLevel } from "../community/acl"; import { buildSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, @@ -26,11 +25,27 @@ import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; import { create as createCid, toString as cidToString } from "@atcute/cid"; +/** Optional hook to extend `.spaceExt.whoami` with extra fields when a + * module above spaces (e.g. community) wants to override the default + * binary-membership response. If the hook returns a non-null object, that + * object is the entire response body. If null, falls through to the + * default behavior (just `isOwner`/`isMember`). + * + * Spaces stays community-agnostic: any consumer can plug in here. */ +export type WhoamiExtension = (input: { + spaceUri: string; + callerDid: string; + isOwner: boolean; + ownerDid: string; +}) => Promise | null>; + export interface SpacesRoutesOptions { /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ authMiddleware?: MiddlewareHandler; /** Storage adapter override. Defaults to HostedAdapter(db). */ adapter?: StorageAdapter; + /** Optional whoami extension; see {@link WhoamiExtension}. */ + whoamiExtension?: WhoamiExtension; } /** Umbrella registration: wires both the authority and the record-host @@ -43,8 +58,7 @@ export function registerSpacesRoutes( db: Database, config: ContrailConfig, options: SpacesRoutesOptions = {}, - ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null, - community?: import("../community/adapter").CommunityAdapter | null + ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null ): void { const spacesConfig = config.spaces; if (!spacesConfig) return; @@ -55,7 +69,7 @@ export function registerSpacesRoutes( const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, community ?? null); + registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); if (spacesConfig.recordHost) { registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth); @@ -70,7 +84,7 @@ export function registerAuthorityRoutes( authorityConfig: AuthorityConfig, config: ContrailConfig, auth: MiddlewareHandler, - community: import("../community/adapter").CommunityAdapter | null + whoamiExtension?: WhoamiExtension ): void { /** Space endpoints are emitted per-deployment under the configured namespace; * the deployment owns and publishes its own lexicons. The library ships @@ -243,8 +257,8 @@ export function registerAuthorityRoutes( }); // Unified whoami — `.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, - // accessLevel? }. `accessLevel` is present only when the target space is - // community-owned; for user-owned spaces membership is binary. + // ... }. Extra fields (e.g. accessLevel for community-owned spaces) come + // from the optional whoamiExtension hook; without one, response is binary. app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { const sa = getAuth(c); const spaceUri = c.req.query("spaceUri"); @@ -254,20 +268,17 @@ export function registerAuthorityRoutes( const isOwner = space.ownerDid === sa.issuer; - // Community-owned space: resolve through the access-level ladder. The - // reconciler keeps spaces_members in sync, so isMember derives from the - // effective level directly. - const isCommunity = community ? !!(await community.getCommunity(space.ownerDid)) : false; - if (isCommunity) { - const level = await resolveEffectiveLevel(community!, spaceUri, sa.issuer); - return c.json({ + if (whoamiExtension) { + const ext = await whoamiExtension({ + spaceUri, + callerDid: sa.issuer, isOwner, - isMember: isOwner || !!level, - accessLevel: level, + ownerDid: space.ownerDid, }); + if (ext) return c.json(ext); } - // User-owned space: binary membership. + // Default: binary membership. if (isOwner) return c.json({ isOwner: true, isMember: true }); const member = await authority.getMember(spaceUri, sa.issuer); return c.json({ isOwner: false, isMember: !!member }); -- 2.51.2 From 4a47874caae0df4ca86b7b4d6dba00bbbeb13bca Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:31:53 +0200 Subject: [PATCH 03/19] phase tres --- packages/contrail/src/core/spaces/auth.ts | 7 + .../contrail/src/core/spaces/credentials.ts | 253 +++++++++++ packages/contrail/src/core/spaces/router.ts | 377 ++++++++++++++--- packages/contrail/src/core/spaces/types.ts | 20 +- packages/contrail/src/index.ts | 19 + .../contrail/tests/spaces-credentials.test.ts | 398 ++++++++++++++++++ .../spaces/getCredential.json | 42 ++ .../spaces/refreshCredential.json | 38 ++ 8 files changed, 1084 insertions(+), 70 deletions(-) create mode 100644 packages/contrail/src/core/spaces/credentials.ts create mode 100644 packages/contrail/tests/spaces-credentials.test.ts create mode 100644 packages/lexicons/lexicon-templates/spaces/getCredential.json create mode 100644 packages/lexicons/lexicon-templates/spaces/refreshCredential.json diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 85ba814..4c1e817 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -136,6 +136,13 @@ export async function verifyServiceAuthRequest( }; } +/** Pull a space credential off the request — `X-Space-Credential: ` + * header. Returns the raw token or null. */ +export function extractSpaceCredential(request: Request): string | null { + const header = request.headers.get("X-Space-Credential"); + return header ? header.trim() : null; +} + /** Pull a read-grant invite token off the request — query string `?inviteToken=` * or `Authorization: Bearer atmo-invite:`. Returns the raw token (not * hashed) or null. Routes hash + look up via the adapter. */ diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts new file mode 100644 index 0000000..8c4a843 --- /dev/null +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -0,0 +1,253 @@ +/** Space-credential primitives: ES256 (P-256) JWTs minted by the authority, + * verified by the record host (or any third party that can resolve the + * authority's DID document). + * + * Format is a compact JWS: + * header = { alg: "ES256", typ: "JWT", kid: "#" } + * payload = { iss, sub, space, scope, iat, exp } + * + * - `iss` is the authority DID (the signer; for phase 3 this is the local + * authority's serviceDid; phase 4 adds a binding-resolution layer that + * lets the issuer be a *different* DID from the space owner). + * - `sub` is the caller DID — the credential bearer. + * - `space` is the full `ats:////` URI. + * - `scope` is "rw" or "read". + * + * We don't use a JWT library — Web Crypto's subtle covers everything (P-256 + * generate, sign, verify, JWK import/export) and saves a runtime dep. */ + +const ALG = "ES256"; +const TYP = "JWT"; +const DEFAULT_KEY_ID = "atproto_space_authority"; + +export type CredentialScope = "rw" | "read"; + +export interface CredentialClaims { + iss: string; + sub: string; + space: string; + scope: CredentialScope; + iat: number; // seconds since epoch + exp: number; // seconds since epoch +} + +export interface CredentialKeyMaterial { + /** Private key in JWK form. P-256 / ES256. */ + privateKey: JsonWebKey; + /** Public key in JWK form. Must match privateKey. */ + publicKey: JsonWebKey; + /** DID-doc verification method id. The full JWT `kid` becomes + * `#`. Defaults to "atproto_space_authority". */ + keyId?: string; +} + +/** Generate a fresh P-256 keypair as JWKs. Useful for local dev / tests; in + * production the operator generates once and stores out-of-band. */ +export async function generateAuthoritySigningKey(): Promise { + const pair = (await crypto.subtle.generateKey( + { name: "ECDSA", namedCurve: "P-256" }, + true, + ["sign", "verify"] + )) as CryptoKeyPair; + const privateKey = (await crypto.subtle.exportKey("jwk", pair.privateKey)) as JsonWebKey; + const publicKey = (await crypto.subtle.exportKey("jwk", pair.publicKey)) as JsonWebKey; + return { privateKey, publicKey }; +} + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function base64urlEncode(bytes: Uint8Array): string { + let s = btoa(String.fromCharCode(...bytes)); + return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +function base64urlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function jsonEncode(value: unknown): string { + return base64urlEncode(enc.encode(JSON.stringify(value))); +} + +function jsonDecode(seg: string): T { + return JSON.parse(dec.decode(base64urlDecode(seg))) as T; +} + +async function importPrivate(jwk: JsonWebKey): Promise { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["sign"] + ); +} + +async function importPublic(jwk: JsonWebKey): Promise { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"] + ); +} + +/** Sign a credential payload with the authority's private key. + * `iat` and `exp` are filled in by the caller (so tests can mint expired + * tokens deterministically). */ +export async function signCredential( + payload: CredentialClaims, + key: CredentialKeyMaterial +): Promise { + const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; + const header = { alg: ALG, typ: TYP, kid }; + const head = jsonEncode(header); + const body = jsonEncode(payload); + const signingInput = `${head}.${body}`; + const privateKey = await importPrivate(key.privateKey); + const sig = await crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + privateKey, + enc.encode(signingInput) + ); + return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; +} + +/** Issue a credential using the current wall-clock for iat/exp. */ +export async function issueCredential( + args: Omit & { ttlMs: number }, + key: CredentialKeyMaterial +): Promise<{ credential: string; expiresAt: number }> { + const now = Math.floor(Date.now() / 1000); + const expSec = now + Math.floor(args.ttlMs / 1000); + const claims: CredentialClaims = { + iss: args.iss, + sub: args.sub, + space: args.space, + scope: args.scope, + iat: now, + exp: expSec, + }; + const credential = await signCredential(claims, key); + return { credential, expiresAt: expSec * 1000 }; +} + +export type VerifyOk = { ok: true; claims: CredentialClaims }; +export type VerifyErr = { + ok: false; + reason: + | "malformed" + | "bad-alg" + | "bad-signature" + | "expired" + | "not-yet-valid" + | "wrong-space" + | "wrong-scope" + | "unknown-issuer"; +}; + +export interface VerifyOptions { + /** Optional: when set, rejects credentials whose `space` claim differs. + * Omit when verifying in middleware where the target space isn't known + * yet — handlers can do the match themselves against the verified + * claims. */ + expectedSpace?: string; + /** Optional: required scope (e.g. "rw" rejects read-only credentials on writes). */ + requiredScope?: CredentialScope; + /** Resolve a verification key for `iss`. If null, verification fails with + * unknown-issuer. */ + resolveKey: (iss: string, kid: string | undefined) => Promise; + /** Time provider for tests. Returns ms since epoch. */ + now?: () => number; +} + +export async function verifyCredential( + jwt: string, + opts: VerifyOptions +): Promise { + const parts = jwt.split("."); + if (parts.length !== 3) return { ok: false, reason: "malformed" }; + const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; + + let header: { alg?: string; typ?: string; kid?: string }; + let claims: CredentialClaims; + try { + header = jsonDecode(headSeg); + claims = jsonDecode(bodySeg); + } catch { + return { ok: false, reason: "malformed" }; + } + if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; + if (opts.expectedSpace !== undefined && claims.space !== opts.expectedSpace) { + return { ok: false, reason: "wrong-space" }; + } + if (opts.requiredScope === "rw" && claims.scope !== "rw") { + return { ok: false, reason: "wrong-scope" }; + } + + const nowMs = (opts.now ?? Date.now)(); + const nowSec = Math.floor(nowMs / 1000); + if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; + if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; + + const jwk = await opts.resolveKey(claims.iss, header.kid); + if (!jwk) return { ok: false, reason: "unknown-issuer" }; + + const publicKey = await importPublic(jwk); + const sigBytes = base64urlDecode(sigSeg); + const signingInput = `${headSeg}.${bodySeg}`; + const valid = await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + sigBytes, + enc.encode(signingInput) + ); + if (!valid) return { ok: false, reason: "bad-signature" }; + return { ok: true, claims }; +} + +/** Header reader for handlers that want to peek at `iss` before resolving the + * key (e.g. to short-circuit DID-doc fetches for the local authority). */ +export function decodeUnverifiedClaims(jwt: string): CredentialClaims | null { + const parts = jwt.split("."); + if (parts.length !== 3) return null; + try { + return jsonDecode(parts[1]!); + } catch { + return null; + } +} + +/** Verifier interface consumed by the record host. The record host doesn't + * care HOW credentials get verified — it only cares whether a given JWT is + * valid. Phase 3 ships an in-process verifier that knows the local + * authority's public key; phase 4 adds a binding-resolving verifier that + * consults PDS records / DID docs. */ +export interface CredentialVerifier { + /** Verify a credential's signature, expiry, and `not-before` window. Does + * NOT enforce a space match — handlers do that against the request URI. */ + verify(jwt: string): Promise; +} + +/** In-process verifier for the simple deployment: the authority and record + * host run in one process and the record host has direct access to the + * authority's public key. Rejects any credential whose `iss` isn't the + * configured authority. Phase 4 generalizes to multi-authority. */ +export function createInProcessVerifier(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): CredentialVerifier { + return { + verify(jwt) { + return verifyCredential(jwt, { + resolveKey: async (iss) => (iss === args.authorityDid ? args.publicKey : null), + }); + }, + }; +} diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index 4e71c05..fa9aeb5 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -8,12 +8,14 @@ import { checkInviteReadGrant, createServiceAuthMiddleware, extractInviteToken, + extractSpaceCredential, } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; import { buildSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, + DEFAULT_CREDENTIAL_TTL_MS, type AuthorityConfig, type RecordHostConfig, type RecordHost, @@ -23,6 +25,15 @@ import { } from "./types"; import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; +import { + createInProcessVerifier, + decodeUnverifiedClaims, + issueCredential, + verifyCredential, + type CredentialClaims, + type CredentialScope, + type CredentialVerifier, +} from "./credentials"; import { create as createCid, toString as cidToString } from "@atcute/cid"; /** Optional hook to extend `.spaceExt.whoami` with extra fields when a @@ -72,7 +83,16 @@ export function registerSpacesRoutes( registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); if (spacesConfig.recordHost) { - registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth); + // In-process verifier: when authority and record host are colocated, + // the host has direct access to the authority's public key. Phase 4 + // adds a binding-resolving verifier for split deployments. + const verifier = authorityConfig.signing + ? createInProcessVerifier({ + authorityDid: authorityConfig.serviceDid, + publicKey: authorityConfig.signing.publicKey, + }) + : undefined; + registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, verifier); } } @@ -159,6 +179,12 @@ export function registerAuthorityRoutes( return c.json({ space: publicSpaceView(space, false) }); } + if (authz.via === "credential") { + // Credential proves membership; derive isOwner from sub vs ownerDid. + const isOwner = authz.claims.sub === space.ownerDid; + return c.json({ space: publicSpaceView(space, isOwner) }); + } + const sa = authz.sa; const isOwner = sa.issuer === space.ownerDid; const member = isOwner ? null : await authority.getMember(uri, sa.issuer); @@ -283,31 +309,189 @@ export function registerAuthorityRoutes( const member = await authority.getMember(spaceUri, sa.issuer); return c.json({ isOwner: false, isMember: !!member }); }); + + // ---- Credential endpoints ---- + + /** Mint a space credential for a member of `spaceUri`. Caller is identified + * by the JWT issuer; the credential's `sub` is set to that DID. */ + app.post(`/xrpc/${SPACE}.getCredential`, auth, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + const member = isOwner ? null : await authority.getMember(body.spaceUri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + // App policy is checked at credential-issuance time. Existing credentials + // remain valid until expiry — that's the spec contract (revocation + // bounded by TTL, not synchronous). + if (space.appPolicy) { + const allowed = checkClientId(space.appPolicy, sa.clientId); + if (!allowed) return c.json({ error: "Forbidden", reason: "app-not-allowed" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: sa.issuer, + space: body.spaceUri, + scope: "rw", + ttlMs: ttl, + }, + authorityConfig.signing + ); + return c.json({ credential, expiresAt }); + }); + + /** Refresh an unexpired credential. Used by long-running clients to extend + * their access without going back through the JWT mint dance. The current + * credential must verify; the bearer must still be a member. */ + app.post(`/xrpc/${SPACE}.refreshCredential`, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const body = (await c.req.json().catch(() => null)) as { credential?: string } | null; + if (!body?.credential) { + return c.json({ error: "InvalidRequest", message: "credential required" }, 400); + } + const signing = authorityConfig.signing; + const claims = await verifyAndAuthorizeRefresh(body.credential, authorityConfig); + if ("error" in claims) return c.json(claims, claims.status); + + const space = await authority.getSpace(claims.space); + if (!space) return c.json({ error: "NotFound" }, 404); + const isOwner = space.ownerDid === claims.sub; + const member = isOwner ? null : await authority.getMember(claims.space, claims.sub); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: claims.sub, + space: claims.space, + scope: claims.scope, + ttlMs: ttl, + }, + signing + ); + return c.json({ credential, expiresAt }); + }); } -/** Register the **record host** XRPC surface — record + blob CRUD. Today - * consults the authority for membership / app-policy checks at write time; - * phase 3 adds a credential verifier that replaces those calls in - * split-deployment configurations. */ +/** Verify a credential presented at refreshCredential. Returns the claims, or + * an error envelope ready to relay. Different from the record-host verifier + * in two ways: (a) we don't have the expectedSpace yet — we read it from the + * credential itself; (b) we don't enforce a scope. */ +async function verifyAndAuthorizeRefresh( + credential: string, + authorityConfig: AuthorityConfig +): Promise { + const peek = decodeUnverifiedClaims(credential); + if (!peek) return { error: "InvalidRequest", reason: "malformed", status: 400 }; + if (peek.iss !== authorityConfig.serviceDid) { + return { error: "Forbidden", reason: "wrong-issuer", status: 401 }; + } + if (!authorityConfig.signing) { + return { error: "InvalidState", status: 401 }; + } + const signing = authorityConfig.signing; + const result = await verifyCredential(credential, { + expectedSpace: peek.space, + resolveKey: async (iss) => (iss === authorityConfig.serviceDid ? signing.publicKey : null), + }); + if (!result.ok) { + return { error: "InvalidCredential", reason: result.reason, status: 401 }; + } + return result.claims; +} + +/** App-policy check using just `clientId`. Mirrors `acl.ts:checkAppPolicy` + * but inlined here so the credential-issuance path doesn't need to construct + * a full AclInput. */ +function checkClientId( + appPolicy: NonNullable, + clientId: string | undefined +): boolean { + const listed = clientId ? appPolicy.apps.includes(clientId) : false; + if (appPolicy.mode === "allow") return !listed; + return listed; +} + +/** Register the **record host** XRPC surface — record + blob CRUD. + * + * Auth precedence on every route: + * 1. `X-Space-Credential` header (if a verifier is wired and the credential + * is valid) — caller DID = credential `sub`, no clientId. + * 2. Read-route invite token (`?inviteToken=` or `Bearer atmo-invite:...`). + * 3. Service-auth JWT (existing behavior) — caller DID = JWT issuer. + * + * When a credential is presented, the record host trusts it: no member + * check, no app-policy check (those happen at issuance time on the + * authority side). Service-auth requests still consult the authority — that + * bridge is what phase 5 cuts when the host/authority split goes runtime. */ export function registerRecordHostRoutes( app: Hono, recordHost: RecordHost, authority: SpaceAuthority, recordHostConfig: RecordHostConfig, config: ContrailConfig, - auth: MiddlewareHandler + auth: MiddlewareHandler, + /** Optional credential verifier. When present, the record host accepts + * `X-Space-Credential` as an alternative to a service-auth JWT. */ + credentialVerifier?: CredentialVerifier ): void { const SPACE = `${config.namespace}.space`; - /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is - * present so anonymous bearer reads don't 401 before the route handler can - * validate the token. */ + /** Auth wrapper: tries credential first, then delegates to JWT auth. */ + const authWithCredential: MiddlewareHandler = async (c, next) => { + const credToken = extractSpaceCredential(c.req.raw); + if (credToken) { + if (!credentialVerifier) { + return c.json( + { error: "AuthRequired", reason: "credential-verifier-not-configured" }, + 401 + ); + } + const result = await credentialVerifier.verify(credToken); + if (!result.ok) { + return c.json({ error: "AuthRequired", reason: result.reason }, 401); + } + c.set("spaceCredential", result.claims); + await next(); + return; + } + return auth(c, next); + }; + + /** Read-route auth: like {@link authWithCredential} but also short-circuits + * on a read-grant invite token. Token presence skips both credential and + * JWT middlewares; the route handler validates the token via authorizeRead. */ const readAuth: MiddlewareHandler = async (c, next) => { if (extractInviteToken(c.req.raw)) { await next(); return; } - return auth(c, next); + return authWithCredential(c, next); }; app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { @@ -336,6 +520,8 @@ export function registerRecordHostRoutes( return c.json({ error: "Forbidden", reason: result.reason }, 403); } } + // Credential and token paths are pre-authorized — credential's signature + // proves the authority granted access; token validation already happened. const list = await recordHost.listRecords(spaceUri, collection, { byUser: c.req.query("byUser") ?? undefined, @@ -379,8 +565,7 @@ export function registerRecordHostRoutes( }); // Write endpoints - app.post(`/xrpc/${SPACE}.putRecord`, auth, async (c) => { - const sa = getAuth(c); + app.post(`/xrpc/${SPACE}.putRecord`, authWithCredential, async (c) => { const body = (await c.req.json().catch(() => null)) as | { spaceUri?: string; collection?: string; rkey?: string; record?: Record } | null; @@ -390,15 +575,20 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, sa.issuer); - const result = checkAccess({ - op: "write", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } // Validate that every blob referenced by this record has already been // uploaded into this space. This mirrors how PDSes require uploadBlob @@ -426,17 +616,16 @@ export function registerRecordHostRoutes( await recordHost.putRecord({ spaceUri: body.spaceUri, collection: body.collection, - authorDid: sa.issuer, + authorDid: caller.callerDid, rkey, cid: null, record: body.record, createdAt: now, }); - return c.json({ rkey, authorDid: sa.issuer, createdAt: now }); + return c.json({ rkey, authorDid: caller.callerDid, createdAt: now }); }); - app.post(`/xrpc/${SPACE}.deleteRecord`, auth, async (c) => { - const sa = getAuth(c); + app.post(`/xrpc/${SPACE}.deleteRecord`, authWithCredential, async (c) => { const body = (await c.req.json().catch(() => null)) as | { spaceUri?: string; collection?: string; rkey?: string } | null; @@ -446,18 +635,26 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(body.spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, sa.issuer); - const result = checkAccess({ - op: "delete", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - targetAuthorDid: sa.issuer, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "delete", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + targetAuthorDid: caller.callerDid, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + // Credential path: scope=rw is checked in resolveCaller. Delete remains + // author-scoped — the credential's `sub` is the caller, and we only + // delete records authored by that DID. - await recordHost.deleteRecord(body.spaceUri, body.collection, sa.issuer, body.rkey); + await recordHost.deleteRecord(body.spaceUri, body.collection, caller.callerDid, body.rkey); return c.json({ ok: true }); }); @@ -468,8 +665,7 @@ export function registerRecordHostRoutes( const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; const accept = blobsCfg.accept; - app.post(`/xrpc/${SPACE}.uploadBlob`, auth, async (c) => { - const sa = getAuth(c); + app.post(`/xrpc/${SPACE}.uploadBlob`, authWithCredential, async (c) => { const spaceUri = c.req.query("spaceUri"); if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); @@ -477,16 +673,21 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const aclResult = checkAccess({ - op: "write", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + const caller = resolveCaller(c, spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } } const mimeType = c.req.header("content-type") ?? "application/octet-stream"; @@ -524,7 +725,7 @@ export function registerRecordHostRoutes( cid: cidString, mimeType, size: bytes.byteLength, - authorDid: sa.issuer, + authorDid: caller.callerDid, createdAt: Date.now(), }); @@ -579,8 +780,7 @@ export function registerRecordHostRoutes( }); }); - app.get(`/xrpc/${SPACE}.listBlobs`, auth, async (c) => { - const sa = getAuth(c); + app.get(`/xrpc/${SPACE}.listBlobs`, authWithCredential, async (c) => { const spaceUri = c.req.query("spaceUri"); if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); @@ -588,16 +788,21 @@ export function registerRecordHostRoutes( const space = await authority.getSpace(spaceUri); if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const aclResult = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + const caller = resolveCaller(c, spaceUri, "read"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "read", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } } const result = await recordHost.listBlobMeta(spaceUri, { @@ -610,14 +815,30 @@ export function registerRecordHostRoutes( } } -/** Authorize a read request: either a valid service-auth JWT (which also - * identifies the caller for member checks downstream) or a valid read-grant - * invite token bearer. */ +/** Authorize a read request — three valid paths: a verified space credential + * (set by the credential middleware), a read-grant invite token, or a + * service-auth JWT. + * + * Credential and token paths skip the membership check downstream — the + * credential or token IS the proof. The JWT path requires a member check + * in the route handler. */ async function authorizeRead( c: Context, authority: SpaceAuthority, spaceUri: string -): Promise<{ via: "token" } | { via: "jwt"; sa: ServiceAuth } | Response> { +): Promise< + | { via: "credential"; claims: CredentialClaims } + | { via: "token" } + | { via: "jwt"; sa: ServiceAuth } + | Response +> { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== spaceUri) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + return { via: "credential", claims: cred }; + } const rawToken = extractInviteToken(c.req.raw); if (rawToken) { const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); @@ -627,11 +848,35 @@ async function authorizeRead( const sa = c.get("serviceAuth") as ServiceAuth | undefined; if (sa) return { via: "jwt", sa }; return c.json( - { error: "AuthRequired", message: "JWT or read-grant invite token required" }, + { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, 401 ); } +/** Unified caller resolution for write/manage paths on the record host. + * Either a verified credential (set by middleware) or a service-auth JWT. + * When a credential is present, also enforces space-match and the requested + * scope. Returns either a caller envelope or a Response to relay. */ +function resolveCaller( + c: Context, + requestSpace: string, + requiredScope: CredentialScope +): { callerDid: string; clientId: string | undefined; viaCredential: boolean } | Response { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== requestSpace) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + if (requiredScope === "rw" && cred.scope !== "rw") { + return c.json({ error: "Forbidden", reason: "credential-wrong-scope" }, 403); + } + return { callerDid: cred.sub, clientId: undefined, viaCredential: true }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (!sa) return c.json({ error: "AuthRequired", reason: "no-auth" }, 401); + return { callerDid: sa.issuer, clientId: sa.clientId, viaCredential: false }; +} + function getAuth(c: Parameters[0]): ServiceAuth { const auth = c.get("serviceAuth") as ServiceAuth | undefined; if (!auth) throw new Error("service auth not set"); diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index fb05d2a..5429c96 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -1,6 +1,7 @@ import type { Database } from "../types"; import type { DidDocumentResolver } from "@atcute/identity-resolver"; import type { BlobAdapter } from "./blob-adapter"; +import type { CredentialKeyMaterial } from "./credentials"; export type AppPolicyMode = "allow" | "deny"; @@ -25,21 +26,32 @@ export interface SpacesBlobsConfig { export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; +/** Default credential lifetime. The rough spec calls for 2–4h; we pick the + * lower bound so revocation (kicked-from-space) is observable within 2h. */ +export const DEFAULT_CREDENTIAL_TTL_MS = 2 * 60 * 60 * 1000; + /** Configuration for the **space authority** role: holds the member list, - * signs credentials (later phases), and gates space-management operations. - * In a fully-split deployment, the authority can run in a different process - * (or even a different operator) than the record host. */ + * signs credentials, and gates space-management operations. In a fully-split + * deployment, the authority can run in a different process (or even a + * different operator) than the record host. */ export interface AuthorityConfig { /** NSID that identifies the kind of space this authority hosts, * e.g. "tools.atmo.event.space". */ type: string; - /** Service DID that service-auth tokens must target (aud claim). */ + /** Service DID that service-auth tokens must target (aud claim) AND that + * signs credentials it issues (`iss` claim on emitted JWTs). */ serviceDid: string; /** Default app policy applied to new spaces. */ defaultAppPolicy?: AppPolicy; /** DID document resolver for service-auth JWT verification. * Defaults to a composite PLC + did:web resolver if omitted. */ resolver?: DidDocumentResolver; + /** Signing key material for issuing space credentials. When omitted, + * `.space.getCredential` returns 501 NotImplemented and the record + * host's credential-verifying middleware can't be wired up. */ + signing?: CredentialKeyMaterial; + /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ + credentialTtlMs?: number; } /** Configuration for the **record host** role: stores per-space records and diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 8d678cb..54e22bf 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -68,6 +68,25 @@ export { export type { BlobAdapter, BlobUploadMeta, R2BucketLike } from "./core/spaces/blob-adapter"; export type { SpacesBlobsConfig, BlobMetaRow } from "./core/spaces/types"; +// Space credentials +export { + generateAuthoritySigningKey, + signCredential, + issueCredential, + verifyCredential, + decodeUnverifiedClaims, + createInProcessVerifier, +} from "./core/spaces/credentials"; +export type { + CredentialClaims, + CredentialKeyMaterial, + CredentialScope, + CredentialVerifier, + VerifyOk, + VerifyErr, + VerifyOptions, +} from "./core/spaces/credentials"; + // Realtime export type { PubSub, diff --git a/packages/contrail/tests/spaces-credentials.test.ts b/packages/contrail/tests/spaces-credentials.test.ts new file mode 100644 index 0000000..ec3c428 --- /dev/null +++ b/packages/contrail/tests/spaces-credentials.test.ts @@ -0,0 +1,398 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + generateAuthoritySigningKey, + signCredential, + verifyCredential, + issueCredential, +} from "../src/core/spaces/credentials"; +import type { CredentialKeyMaterial } from "../src/core/spaces/credentials"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const CHARLIE = "did:plc:charlie"; + +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function makeConfig(): ContrailConfig { + return { + namespace: "test.cred", + collections: { + message: { collection: "app.event.message" }, + }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + credentialTtlMs: 60_000, + }, + recordHost: {}, + }, + }; +} + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; +} + +async function makeApp(): Promise { + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); +} + +function call( + app: Hono, + method: string, + path: string, + did: string | null, + body?: any, + extraHeaders?: Record +) { + const headers: Record = { ...(extraHeaders ?? {}) }; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +async function createSpace(app: Hono, owner: string): Promise { + const res = await call(app, "POST", "/xrpc/test.cred.space.createSpace", owner, {}); + expect(res.status).toBe(200); + return ((await res.json()) as any).space.uri; +} + +async function getCredential(app: Hono, did: string, spaceUri: string): Promise { + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", did, { spaceUri }); + expect(res.status).toBe(200); + return ((await res.json()) as any).credential; +} + +describe("space credentials — sign/verify primitives", () => { + it("round-trips via verifyCredential", async () => { + const { credential } = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: "ats://did:plc:alice/test/main", + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://did:plc:alice/test/main", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.claims.iss).toBe(SERVICE_DID); + expect(result.claims.sub).toBe(ALICE); + expect(result.claims.scope).toBe("rw"); + } + }); + + it("rejects expired credentials", async () => { + const past = Math.floor(Date.now() / 1000) - 10; + const credential = await signCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: "ats://x/y/z", + scope: "rw", + iat: past - 60, + exp: past, + }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("expired"); + }); + + it("rejects wrong-space credentials", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://a/b/c", scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://different/space/here", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("wrong-space"); + }); + + it("rejects credentials signed by a different key", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://x/y/z", scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const otherKey = await generateAuthoritySigningKey(); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + resolveKey: async () => otherKey.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("bad-signature"); + }); + + it("rejects unknown issuer", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://x/y/z", scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + resolveKey: async () => null, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); + + it("scope=rw rejects read-only credentials when verifier requires rw", async () => { + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: "ats://x/y/z", scope: "read", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyCredential(credential, { + expectedSpace: "ats://x/y/z", + requiredScope: "rw", + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("wrong-scope"); + }); +}); + +describe("space credentials — getCredential / refreshCredential endpoints", () => { + it("issues a credential for a space owner", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", ALICE, { + spaceUri: uri, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.credential).toBeTypeOf("string"); + expect(body.expiresAt).toBeTypeOf("number"); + expect(body.expiresAt).toBeGreaterThan(Date.now()); + }); + + it("issues a credential for a member who isn't the owner", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + await call(app, "POST", "/xrpc/test.cred.space.addMember", ALICE, { + spaceUri: uri, + did: BOB, + }); + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", BOB, { + spaceUri: uri, + }); + expect(res.status).toBe(200); + }); + + it("denies non-members", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const res = await call(app, "POST", "/xrpc/test.cred.space.getCredential", CHARLIE, { + spaceUri: uri, + }); + expect(res.status).toBe(403); + expect((await res.json()).reason).toBe("not-member"); + }); + + it("refreshes an unexpired credential", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const cred = await getCredential(app, ALICE, uri); + + const res = await call(app, "POST", "/xrpc/test.cred.space.refreshCredential", null, { + credential: cred, + }); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.credential).toBeTypeOf("string"); + expect(body.credential).not.toBe(cred); // fresh iat/exp → different signature + }); + + it("refresh rejects when the holder is no longer a member", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + await call(app, "POST", "/xrpc/test.cred.space.addMember", ALICE, { + spaceUri: uri, + did: BOB, + }); + const cred = await getCredential(app, BOB, uri); + // Owner kicks Bob out. + const removeRes = await call(app, "POST", "/xrpc/test.cred.space.removeMember", ALICE, { + spaceUri: uri, + did: BOB, + }); + expect(removeRes.status).toBe(200); + + const res = await call(app, "POST", "/xrpc/test.cred.space.refreshCredential", null, { + credential: cred, + }); + expect(res.status).toBe(403); + }); +}); + +describe("space credentials — record host accepts X-Space-Credential", () => { + it("putRecord works with a credential and no service-auth JWT", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const cred = await getCredential(app, ALICE, uri); + + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, // no X-Test-Did header + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "hi from credential" }, + }, + { "X-Space-Credential": cred } + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.authorDid).toBe(ALICE); + expect(body.rkey).toBeTypeOf("string"); + }); + + it("listRecords works with a credential", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const cred = await getCredential(app, ALICE, uri); + // Plant a record via the JWT path so listRecords has something to return. + await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + ALICE, + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "one" }, + } + ); + + const res = await call( + app, + "GET", + `/xrpc/test.cred.space.listRecords?spaceUri=${encodeURIComponent(uri)}&collection=app.event.message`, + null, + undefined, + { "X-Space-Credential": cred } + ); + expect(res.status).toBe(200); + expect(((await res.json()) as any).records.length).toBe(1); + }); + + it("rejects credential issued for a different space", async () => { + const app = await makeApp(); + const uriA = await createSpace(app, ALICE); + // Alice can create a second one; key auto-generated, owner is implicit member. + const uriB = await createSpace(app, ALICE); + const credForA = await getCredential(app, ALICE, uriA); + + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, + { + spaceUri: uriB, + collection: "app.event.message", + record: { $type: "app.event.message", text: "wrong space" }, + }, + { "X-Space-Credential": credForA } + ); + expect(res.status).toBe(403); + expect((await res.json()).reason).toBe("credential-wrong-space"); + }); + + it("rejects malformed credentials", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "x" }, + }, + { "X-Space-Credential": "not-a-jwt" } + ); + expect(res.status).toBe(401); + expect((await res.json()).reason).toBe("malformed"); + }); + + it("rejects credentials forged with an unknown issuer", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + const otherKey = await generateAuthoritySigningKey(); + const { credential } = await issueCredential( + { + iss: "did:web:attacker.example", + sub: ALICE, + space: uri, + scope: "rw", + ttlMs: 60_000, + }, + otherKey + ); + const res = await call( + app, + "POST", + "/xrpc/test.cred.space.putRecord", + null, + { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "x" }, + }, + { "X-Space-Credential": credential } + ); + expect(res.status).toBe(401); + expect((await res.json()).reason).toBe("unknown-issuer"); + }); +}); diff --git a/packages/lexicons/lexicon-templates/spaces/getCredential.json b/packages/lexicons/lexicon-templates/spaces/getCredential.json new file mode 100644 index 0000000..e24e79b --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/getCredential.json @@ -0,0 +1,42 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.getCredential", + "defs": { + "main": { + "type": "procedure", + "description": "Mint a short-lived space credential for a member of `spaceUri`. The caller is identified by their service-auth JWT; the credential's `sub` is set to that DID, scoped 'rw'. Use the returned credential as the `X-Space-Credential` header on subsequent requests instead of minting a fresh service-auth JWT each time. Refresh via `refreshCredential` before expiry.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["spaceUri"], + "properties": { + "spaceUri": { "type": "string" } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["credential", "expiresAt"], + "properties": { + "credential": { + "type": "string", + "description": "Compact JWS (ES256) signed by the authority's key. Header `kid` references the verification method on the authority's DID document." + }, + "expiresAt": { + "type": "integer", + "description": "Expiry as ms since epoch." + } + } + } + }, + "errors": [ + { "name": "NotFound" }, + { "name": "Forbidden" }, + { "name": "NotImplemented", "description": "Authority is not configured to sign credentials." } + ] + } + } +} diff --git a/packages/lexicons/lexicon-templates/spaces/refreshCredential.json b/packages/lexicons/lexicon-templates/spaces/refreshCredential.json new file mode 100644 index 0000000..7e8bbfb --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/refreshCredential.json @@ -0,0 +1,38 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.refreshCredential", + "defs": { + "main": { + "type": "procedure", + "description": "Refresh an unexpired space credential. The current credential must verify and the bearer must still be a member. Returns a fresh credential (same scope, same sub, same space) with a new expiry. No service-auth JWT required — the credential itself is the authentication.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["credential"], + "properties": { + "credential": { "type": "string" } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["credential", "expiresAt"], + "properties": { + "credential": { "type": "string" }, + "expiresAt": { "type": "integer" } + } + } + }, + "errors": [ + { "name": "InvalidRequest" }, + { "name": "InvalidCredential" }, + { "name": "NotFound" }, + { "name": "Forbidden" }, + { "name": "NotImplemented" } + ] + } + } +} -- 2.51.2 From 774a833a9f832c6ddaca5a93b37d7c64a0db1cb2 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 17:40:08 +0200 Subject: [PATCH 04/19] phase 4 --- packages/contrail/src/core/spaces/binding.ts | 256 +++++++++++ .../contrail/src/core/spaces/credentials.ts | 33 +- packages/contrail/src/core/spaces/router.ts | 39 +- packages/contrail/src/index.ts | 14 + .../contrail/tests/spaces-binding.test.ts | 435 ++++++++++++++++++ .../lexicon-templates/spaces/declaration.json | 31 ++ 6 files changed, 797 insertions(+), 11 deletions(-) create mode 100644 packages/contrail/src/core/spaces/binding.ts create mode 100644 packages/contrail/tests/spaces-binding.test.ts create mode 100644 packages/lexicons/lexicon-templates/spaces/declaration.json diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts new file mode 100644 index 0000000..d349d55 --- /dev/null +++ b/packages/contrail/src/core/spaces/binding.ts @@ -0,0 +1,256 @@ +/** Binding resolution: given a space URI, which DID is authorized to sign + * credentials for it, and where do we find that DID's verification key? + * + * Two layers of pluggable resolvers compose into a credential verifier: + * + * BindingResolver — `ats:////` → authority DID + * KeyResolver — (DID, kid) → JsonWebKey + * + * The BindingResolver is what makes user-owned-DID-with-PDS-record work: + * given a space URI, we resolve the owner's PDS, fetch the declaration + * record, and read its `authority` field. For provisioned (no-PDS) DIDs we + * fall back to the owner DID's `#atproto_space_authority` service entry. + * And finally for the trivial case (HappyView-style "owner self-issues"), + * we return the owner DID itself. + * + * See conversation history (phase 4 design) for the rationale on why these + * three sources, in this order. */ + +import type { DidDocumentResolver } from "@atcute/identity-resolver"; +import type { Did } from "@atcute/lexicons"; +import { parseSpaceUri } from "./uri"; + +export interface BindingResolver { + /** Resolve the DID authorized to sign credentials for this space. Returns + * null if no binding could be found via this resolver — the composite + * walks down its list looking for a non-null. */ + resolveAuthority(spaceUri: string): Promise; +} + +export interface KeyResolver { + /** Resolve `did`'s verification key for credential signing. `kid` is the + * full header `kid` value (e.g. "did:web:x.com#atproto_space_authority"), + * used to disambiguate when a DID doc lists multiple methods. */ + resolveKey(did: string, kid: string | undefined): Promise; +} + +// --------------------------------------------------------------------------- +// Binding resolvers +// --------------------------------------------------------------------------- + +/** Always returns the configured authority DID. Used in-process when the + * authority and record host run in one deployment — no need to walk DID + * docs or PDSes; we know what we are. */ +export function createLocalBindingResolver(args: { + authorityDid: string; +}): BindingResolver { + const { authorityDid } = args; + return { + async resolveAuthority() { + return authorityDid; + }, + }; +} + +/** Returns the space owner DID as the authority. This is the implicit + * fallback ("HappyView path") — when no PDS record and no DID-doc service + * entry declare an issuer, the owner is taken to be its own. Whether the + * resulting credential actually verifies depends on whether the owner's DID + * doc publishes a usable signing key. */ +export function createOwnerSelfBindingResolver(): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + return parts ? parts.ownerDid : null; + }, + }; +} + +/** Walks the resolver list in order, returns the first non-null. Use this + * to compose [pdsRecord, didDocService, ownerSelf] etc. */ +export function createCompositeBindingResolver( + resolvers: BindingResolver[] +): BindingResolver { + return { + async resolveAuthority(spaceUri) { + for (const r of resolvers) { + const did = await r.resolveAuthority(spaceUri); + if (did) return did; + } + return null; + }, + }; +} + +/** Reads a space-declaration record from the owner's PDS at + * `at:////` and returns its `authority` field if present. + * + * This is the user-owned-DID path: the user writes a record to their PDS + * authorizing some service as the space's authority, no DID-doc edits + * required. */ +export function createPdsBindingResolver(args: { + /** DID resolver, used to look up the owner's PDS endpoint. */ + resolver: DidDocumentResolver; + /** Fetch impl. Defaults to `globalThis.fetch`. */ + fetch?: typeof fetch; + /** Per-request timeout in ms. Defaults to 5000. */ + timeoutMs?: number; +}): BindingResolver { + const fetchImpl = args.fetch ?? globalThis.fetch; + const timeoutMs = args.timeoutMs ?? 5000; + + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + const pds = await pdsEndpointFor(args.resolver, parts.ownerDid); + if (!pds) return null; + + const url = new URL(`${pds}/xrpc/com.atproto.repo.getRecord`); + url.searchParams.set("repo", parts.ownerDid); + url.searchParams.set("collection", parts.type); + url.searchParams.set("rkey", parts.key); + + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + let res: Response; + try { + res = await fetchImpl(url.toString(), { signal: ctrl.signal }); + } catch { + return null; + } finally { + clearTimeout(timer); + } + if (!res.ok) return null; + const body = (await res.json().catch(() => null)) as + | { value?: { authority?: unknown } } + | null; + const authority = body?.value?.authority; + return typeof authority === "string" && authority.startsWith("did:") ? authority : null; + }, + }; +} + +/** Reads `service[id="#atproto_space_authority"].serviceEndpoint` from the + * owner's DID doc. This is the no-PDS path — useful for provisioned space + * DIDs that exist as DID docs only. + * + * Note the service endpoint here is a *DID*, not a URL. The DID names the + * authority; the key resolver's job is to then fetch its verification key. + * For DID docs that declare a URL endpoint, we treat the URL as a + * did:web hint — caller can normalize. */ +export function createDidDocBindingResolver(args: { + resolver: DidDocumentResolver; + /** Service id to look up. Defaults to "#atproto_space_authority". */ + serviceId?: string; +}): BindingResolver { + const serviceId = args.serviceId ?? "#atproto_space_authority"; + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + let doc; + try { + doc = await args.resolver.resolve(parts.ownerDid as Did); + } catch { + return null; + } + const entry = doc.service?.find((s: { id?: string }) => s.id === serviceId); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + if (typeof endpoint !== "string") return null; + // Endpoint may be a DID (preferred) or a URL hint. Only DIDs are + // verifiable downstream; URLs require the caller to map URL → DID. + return endpoint.startsWith("did:") ? endpoint : null; + }, + }; +} + +// --------------------------------------------------------------------------- +// Key resolvers +// --------------------------------------------------------------------------- + +/** Knows the local authority's public key directly. Returns null for any + * other DID — composite with a DID-doc resolver if you also accept + * external authorities. */ +export function createLocalKeyResolver(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): KeyResolver { + return { + async resolveKey(did) { + return did === args.authorityDid ? args.publicKey : null; + }, + }; +} + +/** Resolves a DID, finds the verification method matching `kid`, returns + * its `publicKeyJwk`. */ +export function createDidDocKeyResolver(args: { + resolver: DidDocumentResolver; +}): KeyResolver { + return { + async resolveKey(did, kid) { + let doc; + try { + doc = await args.resolver.resolve(did as Did); + } catch { + return null; + } + const methods = (doc as { verificationMethod?: VerificationMethod[] }).verificationMethod; + if (!methods) return null; + // kid is "#" — we match against the method.id which DID + // docs spell as "#" too. + const method = kid + ? methods.find((m) => m.id === kid) + : methods[0]; + if (!method?.publicKeyJwk) return null; + return method.publicKeyJwk as JsonWebKey; + }, + }; +} + +/** Walks resolvers in order; returns the first non-null. */ +export function createCompositeKeyResolver( + resolvers: KeyResolver[] +): KeyResolver { + return { + async resolveKey(did, kid) { + for (const r of resolvers) { + const k = await r.resolveKey(did, kid); + if (k) return k; + } + return null; + }, + }; +} + +interface VerificationMethod { + id: string; + type?: string; + controller?: string; + publicKeyJwk?: unknown; + publicKeyMultibase?: string; +} + +// --------------------------------------------------------------------------- +// Internal: PDS endpoint lookup +// --------------------------------------------------------------------------- + +async function pdsEndpointFor( + resolver: DidDocumentResolver, + did: string +): Promise { + let doc; + try { + doc = await resolver.resolve(did as Did); + } catch { + return null; + } + const entry = doc.service?.find( + (s: { id?: string }) => s.id === "#atproto_pds" + ); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + return typeof endpoint === "string" ? endpoint : null; +} diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts index 8c4a843..d5c3029 100644 --- a/packages/contrail/src/core/spaces/credentials.ts +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -238,7 +238,8 @@ export interface CredentialVerifier { /** In-process verifier for the simple deployment: the authority and record * host run in one process and the record host has direct access to the * authority's public key. Rejects any credential whose `iss` isn't the - * configured authority. Phase 4 generalizes to multi-authority. */ + * configured authority. Phase 4 has a more general + * {@link createBindingCredentialVerifier} that does proper binding lookup. */ export function createInProcessVerifier(args: { authorityDid: string; publicKey: JsonWebKey; @@ -251,3 +252,33 @@ export function createInProcessVerifier(args: { }, }; } + +/** Verifier composed of a {@link BindingResolver} (which DID is authorized + * to issue for this space?) and a {@link KeyResolver} (what's that DID's + * public key?). This is the production-shape verifier — phase 4's main + * contribution. + * + * Verification flow: + * 1. Decode the JWT's claims (no signature check yet). + * 2. Ask the binding resolver: who's authorized for `claims.space`? + * 3. Confirm `claims.iss === authorizedDid`. + * 4. Ask the key resolver for that DID's verification key. + * 5. Verify signature + expiry + scope match. + */ +export function createBindingCredentialVerifier(args: { + bindings: import("./binding").BindingResolver; + keys: import("./binding").KeyResolver; +}): CredentialVerifier { + return { + async verify(jwt) { + const peek = decodeUnverifiedClaims(jwt); + if (!peek) return { ok: false, reason: "malformed" }; + const authorizedDid = await args.bindings.resolveAuthority(peek.space); + if (!authorizedDid) return { ok: false, reason: "unknown-issuer" }; + if (peek.iss !== authorizedDid) return { ok: false, reason: "unknown-issuer" }; + return verifyCredential(jwt, { + resolveKey: (iss, kid) => args.keys.resolveKey(iss, kid), + }); + }, + }; +} diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index fa9aeb5..c0c9e92 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -26,7 +26,7 @@ import { import { blobKey } from "./blob-adapter"; import { collectBlobCids } from "./blob-refs"; import { - createInProcessVerifier, + createBindingCredentialVerifier, decodeUnverifiedClaims, issueCredential, verifyCredential, @@ -34,6 +34,10 @@ import { type CredentialScope, type CredentialVerifier, } from "./credentials"; +import { + createLocalBindingResolver, + createLocalKeyResolver, +} from "./binding"; import { create as createCid, toString as cidToString } from "@atcute/cid"; /** Optional hook to extend `.spaceExt.whoami` with extra fields when a @@ -57,6 +61,12 @@ export interface SpacesRoutesOptions { adapter?: StorageAdapter; /** Optional whoami extension; see {@link WhoamiExtension}. */ whoamiExtension?: WhoamiExtension; + /** Optional credential verifier for the record host. When omitted, a + * default in-process binding verifier is built from the authority's + * signing config (Local binding + Local key resolvers). Override to + * accept credentials from external authorities — wire in PDS-record / + * DID-doc binding resolvers and a DID-doc key resolver. */ + credentialVerifier?: CredentialVerifier; } /** Umbrella registration: wires both the authority and the record-host @@ -83,15 +93,24 @@ export function registerSpacesRoutes( registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); if (spacesConfig.recordHost) { - // In-process verifier: when authority and record host are colocated, - // the host has direct access to the authority's public key. Phase 4 - // adds a binding-resolving verifier for split deployments. - const verifier = authorityConfig.signing - ? createInProcessVerifier({ - authorityDid: authorityConfig.serviceDid, - publicKey: authorityConfig.signing.publicKey, - }) - : undefined; + // Build the default in-process verifier when the authority can sign: + // Local binding (always points at the configured authority) + Local + // key resolver (knows the authority's public key directly). Caller + // can override via `options.credentialVerifier` to accept external + // authorities — wire in PDS-record / DID-doc resolvers there. + const verifier = + options.credentialVerifier ?? + (authorityConfig.signing + ? createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ + authorityDid: authorityConfig.serviceDid, + }), + keys: createLocalKeyResolver({ + authorityDid: authorityConfig.serviceDid, + publicKey: authorityConfig.signing.publicKey, + }), + }) + : undefined); registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, verifier); } } diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 54e22bf..5e59dac 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -76,6 +76,7 @@ export { verifyCredential, decodeUnverifiedClaims, createInProcessVerifier, + createBindingCredentialVerifier, } from "./core/spaces/credentials"; export type { CredentialClaims, @@ -87,6 +88,19 @@ export type { VerifyOptions, } from "./core/spaces/credentials"; +// Binding + key resolution +export { + createLocalBindingResolver, + createOwnerSelfBindingResolver, + createCompositeBindingResolver, + createPdsBindingResolver, + createDidDocBindingResolver, + createLocalKeyResolver, + createDidDocKeyResolver, + createCompositeKeyResolver, +} from "./core/spaces/binding"; +export type { BindingResolver, KeyResolver } from "./core/spaces/binding"; + // Realtime export type { PubSub, diff --git a/packages/contrail/tests/spaces-binding.test.ts b/packages/contrail/tests/spaces-binding.test.ts new file mode 100644 index 0000000..427b6f2 --- /dev/null +++ b/packages/contrail/tests/spaces-binding.test.ts @@ -0,0 +1,435 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + createLocalBindingResolver, + createOwnerSelfBindingResolver, + createCompositeBindingResolver, + createPdsBindingResolver, + createDidDocBindingResolver, + createLocalKeyResolver, + createDidDocKeyResolver, + createCompositeKeyResolver, +} from "../src/core/spaces/binding"; +import { + generateAuthoritySigningKey, + issueCredential, + createBindingCredentialVerifier, +} from "../src/core/spaces/credentials"; +import type { CredentialKeyMaterial } from "../src/core/spaces/credentials"; + +const ALICE = "did:plc:alice"; +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +const SPACE_URI = "ats://did:plc:alice/com.example.event.space/main"; + +describe("BindingResolver — basic resolvers", () => { + it("Local always returns the configured DID", async () => { + const r = createLocalBindingResolver({ authorityDid: SERVICE_DID }); + expect(await r.resolveAuthority(SPACE_URI)).toBe(SERVICE_DID); + expect(await r.resolveAuthority("ats://did:plc:bob/x/y")).toBe(SERVICE_DID); + }); + + it("OwnerSelf parses the owner from the URI", async () => { + const r = createOwnerSelfBindingResolver(); + expect(await r.resolveAuthority(SPACE_URI)).toBe(ALICE); + expect(await r.resolveAuthority("not a space uri")).toBeNull(); + }); + + it("Composite returns the first non-null result", async () => { + const r = createCompositeBindingResolver([ + { resolveAuthority: async () => null }, + { resolveAuthority: async () => "did:web:second" }, + { resolveAuthority: async () => "did:web:third" }, + ]); + expect(await r.resolveAuthority(SPACE_URI)).toBe("did:web:second"); + }); + + it("Composite returns null if every resolver returns null", async () => { + const r = createCompositeBindingResolver([ + { resolveAuthority: async () => null }, + { resolveAuthority: async () => null }, + ]); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); +}); + +describe("BindingResolver — PDS record", () => { + function mockResolver(opts: { + pdsEndpoint?: string; + fail?: boolean; + }): any { + return { + resolve: async (did: string) => { + if (opts.fail) throw new Error("nope"); + return { + id: did, + service: opts.pdsEndpoint + ? [ + { + id: "#atproto_pds", + type: "AtprotoPersonalDataServer", + serviceEndpoint: opts.pdsEndpoint, + }, + ] + : [], + }; + }, + }; + } + + function mockFetch(map: Map): typeof fetch { + return (async (url: string) => { + const u = String(url); + const found = [...map.entries()].find(([k]) => u.startsWith(k)); + if (!found) return new Response("not found", { status: 404 }); + return new Response(JSON.stringify(found[1]), { + status: 200, + headers: { "content-type": "application/json" }, + }); + }) as typeof fetch; + } + + it("reads `authority` from a declaration record", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + uri: "at://did:plc:alice/com.example.event.space/main", + value: { + $type: "com.example.event.space", + authority: "did:web:custom-authority.example", + recordHost: "did:web:host.example", + createdAt: "2026-04-30T00:00:00Z", + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBe("did:web:custom-authority.example"); + }); + + it("returns null when the PDS record is missing", async () => { + const fetch = mockFetch(new Map()); // 404 for everything + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when the record has no authority field", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { value: { $type: "x", createdAt: "..." } }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when the owner DID doc has no PDS endpoint", async () => { + const fetch = mockFetch(new Map()); + const r = createPdsBindingResolver({ + resolver: mockResolver({}), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); +}); + +describe("BindingResolver — DID-doc service entry", () => { + function mockResolver(serviceDid: string | null): any { + return { + resolve: async (did: string) => ({ + id: did, + service: serviceDid + ? [ + { + id: "#atproto_space_authority", + type: "AtprotoSpaceAuthority", + serviceEndpoint: serviceDid, + }, + ] + : [], + }), + }; + } + + it("reads the #atproto_space_authority service entry", async () => { + const r = createDidDocBindingResolver({ + resolver: mockResolver("did:web:authority.example"), + }); + expect(await r.resolveAuthority(SPACE_URI)).toBe("did:web:authority.example"); + }); + + it("returns null when the service entry is absent", async () => { + const r = createDidDocBindingResolver({ resolver: mockResolver(null) }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("rejects URL-shaped service endpoints (must be a DID)", async () => { + const r = createDidDocBindingResolver({ + resolver: mockResolver("https://authority.example.com"), + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); +}); + +describe("KeyResolver", () => { + it("Local matches by DID", async () => { + const r = createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }); + expect(await r.resolveKey(SERVICE_DID, undefined)).toEqual(SIGNING.publicKey); + expect(await r.resolveKey("did:web:other", undefined)).toBeNull(); + }); + + it("DidDoc finds the verification method matching kid", async () => { + const otherKey = await generateAuthoritySigningKey(); + const resolver = { + resolve: async (did: string) => ({ + id: did, + verificationMethod: [ + { + id: `${did}#atproto_space_authority`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: SIGNING.publicKey, + }, + { + id: `${did}#another-key`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: otherKey.publicKey, + }, + ], + }), + }; + const r = createDidDocKeyResolver({ resolver: resolver as any }); + const key = await r.resolveKey( + "did:web:authority.example", + "did:web:authority.example#atproto_space_authority" + ); + expect(key).toEqual(SIGNING.publicKey); + }); + + it("DidDoc returns the second key when kid points there", async () => { + const otherKey = await generateAuthoritySigningKey(); + const resolver = { + resolve: async (did: string) => ({ + id: did, + verificationMethod: [ + { + id: `${did}#atproto_space_authority`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: SIGNING.publicKey, + }, + { + id: `${did}#another-key`, + type: "JsonWebKey2020", + controller: did, + publicKeyJwk: otherKey.publicKey, + }, + ], + }), + }; + const r = createDidDocKeyResolver({ resolver: resolver as any }); + const key = await r.resolveKey( + "did:web:authority.example", + "did:web:authority.example#another-key" + ); + expect(key).toEqual(otherKey.publicKey); + }); + + it("Composite walks resolvers in order", async () => { + const fallback = await generateAuthoritySigningKey(); + const r = createCompositeKeyResolver([ + { resolveKey: async () => null }, + { resolveKey: async () => fallback.publicKey }, + ]); + expect(await r.resolveKey("did:any", undefined)).toEqual(fallback.publicKey); + }); +}); + +describe("createBindingCredentialVerifier — composes binding + key resolvers", () => { + it("verifies a credential whose iss matches the binding-resolved authority", async () => { + const verifier = createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ authorityDid: SERVICE_DID }), + keys: createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }), + }); + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: SPACE_URI, scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifier.verify(credential); + expect(result.ok).toBe(true); + }); + + it("rejects when credential iss disagrees with the binding", async () => { + const verifier = createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ authorityDid: SERVICE_DID }), + keys: createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }), + }); + const { credential } = await issueCredential( + { + iss: "did:web:imposter.example", + sub: ALICE, + space: SPACE_URI, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const result = await verifier.verify(credential); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); + + it("rejects when no resolver knows the authority", async () => { + const verifier = createBindingCredentialVerifier({ + bindings: { resolveAuthority: async () => null }, + keys: { resolveKey: async () => null }, + }); + const { credential } = await issueCredential( + { iss: SERVICE_DID, sub: ALICE, space: SPACE_URI, scope: "rw", ttlMs: 60_000 }, + SIGNING + ); + const result = await verifier.verify(credential); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); +}); + +describe("end-to-end — record host accepts credential from external authority", () => { + function makeConfig(): ContrailConfig { + return { + namespace: "test.binding", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + // No `signing` — this deployment is record-host only, accepting + // credentials issued by an external authority. + }, + recordHost: {}, + }, + }; + } + + function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; + } + + it("verifies a credential issued by a separate authority via injected verifier", async () => { + // Set up: the record-host's verifier knows about an external authority + // (did:web:external-authority.example) and where to find its public key. + // No PDS / DID-doc fetches — the verifier is configured directly. + const externalAuthority = "did:web:external-authority.example"; + const externalKey = SIGNING; // simulate operator-provided key material + + const verifier = createBindingCredentialVerifier({ + bindings: createLocalBindingResolver({ authorityDid: externalAuthority }), + keys: createLocalKeyResolver({ + authorityDid: externalAuthority, + publicKey: externalKey.publicKey, + }), + }); + + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + const app = createApp(db, resolved, { + spaces: { + authMiddleware: fakeAuth(), + credentialVerifier: verifier, + }, + }); + + // Create a space directly in the local authority's tables. (In a real + // split deployment, the authority would do this; the record host would + // just enroll. Phase 5 introduces enrollment — for now, we use the + // local authority routes as a stand-in.) + const create = await app.fetch( + new Request(`http://localhost/xrpc/test.binding.space.createSpace`, { + method: "POST", + headers: { "X-Test-Did": ALICE, "Content-Type": "application/json" }, + body: "{}", + }) + ); + expect(create.status).toBe(200); + const uri = ((await create.json()) as any).space.uri; + + // External authority signs a credential. + const { credential } = await issueCredential( + { + iss: externalAuthority, + sub: ALICE, + space: uri, + scope: "rw", + ttlMs: 60_000, + }, + externalKey + ); + + // Record host accepts it on putRecord with no service-auth JWT. + const put = await app.fetch( + new Request(`http://localhost/xrpc/test.binding.space.putRecord`, { + method: "POST", + headers: { + "Content-Type": "application/json", + "X-Space-Credential": credential, + }, + body: JSON.stringify({ + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "hello" }, + }), + }) + ); + expect(put.status).toBe(200); + expect(((await put.json()) as any).authorDid).toBe(ALICE); + }); +}); diff --git a/packages/lexicons/lexicon-templates/spaces/declaration.json b/packages/lexicons/lexicon-templates/spaces/declaration.json new file mode 100644 index 0000000..0a5572b --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/declaration.json @@ -0,0 +1,31 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.declaration", + "description": "PDS-record-as-discovery: when present at `at:////` (i.e. paired with the space URI by NSID + rkey), this record declares which DID is authorized to issue credentials for the space and which DID hosts its records. The space owner's PDS write is the cryptographic proof of authorization — the binding resolver fetches this record to verify a credential's `iss` against the user's wishes, no DID-doc edits required. App-defined space-type lexicons MAY embed these fields directly instead of writing a separate record under this NSID.", + "defs": { + "main": { + "type": "record", + "key": "any", + "record": { + "type": "object", + "required": ["authority", "createdAt"], + "properties": { + "authority": { + "type": "string", + "format": "did", + "description": "DID authorized to sign credentials for this space (`iss` on emitted JWTs). May equal the space owner DID for the self-issuing case." + }, + "recordHost": { + "type": "string", + "format": "did", + "description": "DID of the record host where records for this space live. Clients use this to know where to send writes; verifiers ignore it." + }, + "createdAt": { + "type": "string", + "format": "datetime" + } + } + } + } + } +} -- 2.51.2 From dc4d2bcaf15a440bc3a254044ad6f0e0760e491c Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:05:29 +0200 Subject: [PATCH 05/19] phase V --- .../src/core/realtime/publishing-adapter.ts | 4 + packages/contrail/src/core/spaces/adapter.ts | 59 +++ packages/contrail/src/core/spaces/binding.ts | 18 + packages/contrail/src/core/spaces/router.ts | 163 +++++-- packages/contrail/src/core/spaces/schema.ts | 12 + packages/contrail/src/core/spaces/types.ts | 29 +- packages/contrail/src/index.ts | 10 + .../contrail/tests/spaces-enrollment.test.ts | 402 ++++++++++++++++++ .../lexicon-templates/recordHost/enroll.json | 39 ++ 9 files changed, 704 insertions(+), 32 deletions(-) create mode 100644 packages/contrail/tests/spaces-enrollment.test.ts create mode 100644 packages/lexicons/lexicon-templates/recordHost/enroll.json diff --git a/packages/contrail/src/core/realtime/publishing-adapter.ts b/packages/contrail/src/core/realtime/publishing-adapter.ts index 07cbc00..c5dc61f 100644 --- a/packages/contrail/src/core/realtime/publishing-adapter.ts +++ b/packages/contrail/src/core/realtime/publishing-adapter.ts @@ -52,6 +52,10 @@ export function wrapWithPublishing( revokeInvite: inner.revokeInvite.bind(inner), getInvite: inner.getInvite.bind(inner), redeemInvite: inner.redeemInvite.bind(inner), + enroll: inner.enroll.bind(inner), + getEnrollment: inner.getEnrollment.bind(inner), + listEnrollments: inner.listEnrollments.bind(inner), + removeEnrollment: inner.removeEnrollment.bind(inner), getRecord: inner.getRecord.bind(inner), listRecords: inner.listRecords.bind(inner), listCollections: inner.listCollections.bind(inner), diff --git a/packages/contrail/src/core/spaces/adapter.ts b/packages/contrail/src/core/spaces/adapter.ts index b5f6b3c..f80ba5e 100644 --- a/packages/contrail/src/core/spaces/adapter.ts +++ b/packages/contrail/src/core/spaces/adapter.ts @@ -13,6 +13,7 @@ import type { BlobMetaRow, CollectionCount, CreateInviteInput, + EnrollmentRow, InviteKind, InviteRow, ListBlobsOptions, @@ -77,6 +78,15 @@ function mapBlobMetaRow(row: any): BlobMetaRow { }; } +function mapEnrollmentRow(row: any): EnrollmentRow { + return { + spaceUri: row.space_uri, + authorityDid: row.authority_did, + enrolledAt: toNum(row.enrolled_at), + enrolledBy: row.enrolled_by, + }; +} + function mapInviteRow(row: any): InviteRow { return { tokenHash: row.token_hash, @@ -362,6 +372,55 @@ export class HostedAdapter implements StorageAdapter { return row ? mapInviteRow(row) : null; } + async enroll(input: EnrollmentRow): Promise { + await this.db + .prepare( + `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri) DO UPDATE SET + authority_did = excluded.authority_did, + enrolled_at = excluded.enrolled_at, + enrolled_by = excluded.enrolled_by` + ) + .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) + .run(); + } + + async getEnrollment(spaceUri: string): Promise { + const row = await this.db + .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .first(); + return row ? mapEnrollmentRow(row) : null; + } + + async listEnrollments( + options: { authorityDid?: string; limit?: number } = {} + ): Promise { + const limit = Math.min(options.limit ?? 200, 1000); + if (options.authorityDid) { + const { results } = await this.db + .prepare( + `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` + ) + .bind(options.authorityDid, limit) + .all(); + return results.map(mapEnrollmentRow); + } + const { results } = await this.db + .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) + .bind(limit) + .all(); + return results.map(mapEnrollmentRow); + } + + async removeEnrollment(spaceUri: string): Promise { + await this.db + .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .run(); + } + async putRecord(record: StoredRecord): Promise { const table = this.tableFor(record.collection); const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts index d349d55..a81b95f 100644 --- a/packages/contrail/src/core/spaces/binding.ts +++ b/packages/contrail/src/core/spaces/binding.ts @@ -19,6 +19,7 @@ import type { DidDocumentResolver } from "@atcute/identity-resolver"; import type { Did } from "@atcute/lexicons"; import { parseSpaceUri } from "./uri"; +import type { RecordHost } from "./types"; export interface BindingResolver { /** Resolve the DID authorized to sign credentials for this space. Returns @@ -52,6 +53,23 @@ export function createLocalBindingResolver(args: { }; } +/** Reads the record host's local enrollment table. This is the *canonical* + * binding source on a record host: the host owner explicitly consented to a + * given authority for a given space (via the `recordHost.enroll` endpoint + * or auto-enrollment from the authority's createSpace). PDS-record / + * DID-doc resolvers are out-of-band discovery aids; the enrollment is what + * actually gates whether records get stored here. */ +export function createEnrollmentBindingResolver(args: { + recordHost: RecordHost; +}): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const e = await args.recordHost.getEnrollment(spaceUri); + return e?.authorityDid ?? null; + }, + }; +} + /** Returns the space owner DID as the authority. This is the implicit * fallback ("HappyView path") — when no PDS record and no DID-doc service * entry declare an issuer, the owner is taken to be its own. Whether the diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index c0c9e92..dadb7c9 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -12,7 +12,7 @@ import { } from "./auth"; import { nextTid } from "./tid"; import { hashInviteToken } from "../invite/token"; -import { buildSpaceUri } from "./uri"; +import { buildSpaceUri, parseSpaceUri } from "./uri"; import { DEFAULT_BLOB_MAX_SIZE, DEFAULT_CREDENTIAL_TTL_MS, @@ -35,6 +35,8 @@ import { type CredentialVerifier, } from "./credentials"; import { + createCompositeBindingResolver, + createEnrollmentBindingResolver, createLocalBindingResolver, createLocalKeyResolver, } from "./binding"; @@ -90,21 +92,39 @@ export function registerSpacesRoutes( const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - registerAuthorityRoutes(app, adapter, authorityConfig, config, auth, options.whoamiExtension); + // When the record host is colocated, hand it to the authority so + // createSpace can auto-enroll. This is the in-process default — + // single-call createSpace gives you a usable space without requiring an + // explicit `recordHost.enroll` afterward. Split deployments don't get + // this convenience; their createSpace caller (or operator) has to call + // enroll on the remote host explicitly. + const localRecordHost = spacesConfig.recordHost ? adapter : null; + registerAuthorityRoutes( + app, + adapter, + authorityConfig, + config, + auth, + options.whoamiExtension, + localRecordHost + ); if (spacesConfig.recordHost) { - // Build the default in-process verifier when the authority can sign: - // Local binding (always points at the configured authority) + Local - // key resolver (knows the authority's public key directly). Caller - // can override via `options.credentialVerifier` to accept external - // authorities — wire in PDS-record / DID-doc resolvers there. + // Default in-process verifier: enrollment is the canonical binding + // source (matches the host's local consent), with Local-binding as a + // fallback for the case where the authority's createSpace ran but + // auto-enroll was bypassed. Local key resolver knows the local + // authority's public key. Caller overrides via + // `options.credentialVerifier` to accept external authorities — wire + // in DID-doc key resolvers there. const verifier = options.credentialVerifier ?? (authorityConfig.signing ? createBindingCredentialVerifier({ - bindings: createLocalBindingResolver({ - authorityDid: authorityConfig.serviceDid, - }), + bindings: createCompositeBindingResolver([ + createEnrollmentBindingResolver({ recordHost: adapter }), + createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), + ]), keys: createLocalKeyResolver({ authorityDid: authorityConfig.serviceDid, publicKey: authorityConfig.signing.publicKey, @@ -116,14 +136,20 @@ export function registerSpacesRoutes( } /** Register the **space authority** XRPC surface — space lifecycle, member - * list, app policy, whoami. Does NOT touch records or blobs. */ + * list, app policy, whoami. Does NOT touch records or blobs. + * + * When a `localRecordHost` is passed, the authority's `createSpace` handler + * also enrolls the new space on that host — convenient for in-process + * deployments where the same operator runs both roles. Split deployments + * pass `null` and arrange enrollment explicitly via `recordHost.enroll`. */ export function registerAuthorityRoutes( app: Hono, authority: SpaceAuthority, authorityConfig: AuthorityConfig, config: ContrailConfig, auth: MiddlewareHandler, - whoamiExtension?: WhoamiExtension + whoamiExtension?: WhoamiExtension, + localRecordHost?: RecordHost | null ): void { /** Space endpoints are emitted per-deployment under the configured namespace; * the deployment owns and publishes its own lexicons. The library ships @@ -243,6 +269,19 @@ export function registerAuthorityRoutes( // Owner is implicit; we still write a row so membership queries are uniform. await authority.addMember(uri, sa.issuer, sa.issuer); + // Auto-enroll the space on the colocated record host (if any). Without + // this, putRecord/listRecords would 404 with "not-enrolled" until the + // caller explicitly hit recordHost.enroll. Idempotent (ON CONFLICT + // UPDATE), so calling enroll again later is harmless. + if (localRecordHost) { + await localRecordHost.enroll({ + spaceUri: uri, + authorityDid: authorityConfig.serviceDid, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + } + return c.json({ space: publicSpaceView(space, true) }); }); @@ -513,20 +552,79 @@ export function registerRecordHostRoutes( return authWithCredential(c, next); }; + /** Hard gate on every record-host operation: the space must be enrolled. + * Returns the enrollment row, or a Response to relay. */ + async function requireEnrollment( + c: Context, + spaceUri: string + ): Promise<{ authorityDid: string } | Response> { + const enrollment = await recordHost.getEnrollment(spaceUri); + if (!enrollment) { + return c.json( + { error: "NotFound", reason: "not-enrolled", message: "space is not enrolled on this record host" }, + 404 + ); + } + return enrollment; + } + + // ---- Enrollment endpoint ---- + // + // Authorize a space onto this record host. Caller must be either the space + // owner OR the declared authority — the host treats either as a sufficient + // signal of consent. Idempotent: re-enrolling the same space updates the + // authority binding. + const RECORD_HOST = `${config.namespace}.recordHost`; + + app.post(`/xrpc/${RECORD_HOST}.enroll`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; authority?: string } + | null; + if (!body?.spaceUri || !body.authority) { + return c.json({ error: "InvalidRequest", message: "spaceUri and authority required" }, 400); + } + const parts = parseSpaceUri(body.spaceUri); + if (!parts) { + return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); + } + const callerIsOwner = sa.issuer === parts.ownerDid; + const callerIsAuthority = sa.issuer === body.authority; + if (!callerIsOwner && !callerIsAuthority) { + return c.json( + { error: "Forbidden", reason: "not-owner-or-authority" }, + 403 + ); + } + await recordHost.enroll({ + spaceUri: body.spaceUri, + authorityDid: body.authority, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + return c.json({ ok: true }); + }); + app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { const spaceUri = c.req.query("spaceUri"); const collection = c.req.query("collection"); if (!spaceUri || !collection) { return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { + // JWT path consults the authority for member + app-policy checks. + // Split deployments where the host has no live authority will degrade + // here (getSpace returns null) and return 404, which is correct: a + // host that can't reach an authority can't validate JWT membership. const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", @@ -558,14 +656,16 @@ export function registerRecordHostRoutes( if (!spaceUri || !collection || !author || !rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, sa.issuer); const result = checkAccess({ op: "read", @@ -591,13 +691,16 @@ export function registerRecordHostRoutes( if (!body?.spaceUri || !body.collection || !body.record) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, body.spaceUri, "rw"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + // JWT path needs authority access for member + app-policy checks. + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(body.spaceUri, caller.callerDid); const result = checkAccess({ op: "write", @@ -651,13 +754,15 @@ export function registerRecordHostRoutes( if (!body?.spaceUri || !body.collection || !body.rkey) { return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, body.spaceUri, "rw"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(body.spaceUri, caller.callerDid); const result = checkAccess({ op: "delete", @@ -689,13 +794,15 @@ export function registerRecordHostRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, spaceUri, "rw"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, caller.callerDid); const aclResult = checkAccess({ op: "write", @@ -764,14 +871,16 @@ export function registerRecordHostRoutes( if (!spaceUri || !cid) { return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const authz = await authorizeRead(c, authority, spaceUri); if (authz instanceof Response) return authz; if (authz.via === "jwt") { const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, sa.issuer); const aclResult = checkAccess({ op: "read", @@ -804,13 +913,15 @@ export function registerRecordHostRoutes( if (!spaceUri) { return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); } - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; const caller = resolveCaller(c, spaceUri, "read"); if (caller instanceof Response) return caller; if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); const member = await authority.getMember(spaceUri, caller.callerDid); const aclResult = checkAccess({ op: "read", diff --git a/packages/contrail/src/core/spaces/schema.ts b/packages/contrail/src/core/spaces/schema.ts index a08ada6..ae33a7c 100644 --- a/packages/contrail/src/core/spaces/schema.ts +++ b/packages/contrail/src/core/spaces/schema.ts @@ -59,6 +59,18 @@ export function buildSpacesBaseSchema(dialect: SqlDialect): string[] { note TEXT )`, `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, + + // Record-host enrollment table — the host's local cache of "spaces I + // accept records for, and which authority signs credentials for each." + // Filled by the recordHost.enroll endpoint, or auto-populated by the + // authority's createSpace when both roles run in the same process. + `CREATE TABLE IF NOT EXISTS record_host_enrollments ( + space_uri TEXT PRIMARY KEY, + authority_did TEXT NOT NULL, + enrolled_at ${dialect.bigintType} NOT NULL, + enrolled_by TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, ]; } diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 5429c96..6561773 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -170,6 +170,17 @@ export interface BlobMetaRow { createdAt: number; } +/** Local cache on the record host: this space is accepted here, and `authority_did` + * is the DID authorized to sign credentials for it. Populated via the + * `recordHost.enroll` endpoint or auto-populated by the authority's + * createSpace when both roles run in one process. */ +export interface EnrollmentRow { + spaceUri: string; + authorityDid: string; + enrolledAt: number; + enrolledBy: string; +} + export interface ListBlobsOptions { byUser?: string; cursor?: string; @@ -220,14 +231,20 @@ export interface SpaceAuthority { redeemInvite(tokenHash: string, now: number): Promise; } -/** **Record host** interface — stores records and blobs for a space. Trusts - * the authority's ACL: in later phases, validates incoming credentials - * rather than calling out for a member check. +/** **Record host** interface — stores records and blobs for a space, plus + * the local enrollment table that decides which spaces this host accepts. * - * Read-side note: today the record host needs `getSpace` to know the space - * exists at all (and for app-policy checks before permissioned writes). When - * enrollment lands (phase 5), this becomes a local lookup instead. */ + * Trust model: the host trusts whatever credential the authority signs, so + * long as the authority is the one named in the local enrollment for this + * space. Enrollment is the consent step — the host owner agrees to spend + * storage on a given space, scoped to a specific authority. */ export interface RecordHost { + // Enrollment + enroll(input: EnrollmentRow): Promise; + getEnrollment(spaceUri: string): Promise; + listEnrollments(options?: { authorityDid?: string; limit?: number }): Promise; + removeEnrollment(spaceUri: string): Promise; + // Records putRecord(record: StoredRecord): Promise; getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise; diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index 5e59dac..bbd8efe 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -91,6 +91,7 @@ export type { // Binding + key resolution export { createLocalBindingResolver, + createEnrollmentBindingResolver, createOwnerSelfBindingResolver, createCompositeBindingResolver, createPdsBindingResolver, @@ -101,6 +102,15 @@ export { } from "./core/spaces/binding"; export type { BindingResolver, KeyResolver } from "./core/spaces/binding"; +// Route registration — exposed for split deployments where authority and +// record host run as separate Hono apps. Consumers wire them onto bare Honos +// individually instead of going through createApp's umbrella. +export { + registerAuthorityRoutes, + registerRecordHostRoutes, +} from "./core/spaces/router"; +export type { EnrollmentRow } from "./core/spaces/types"; + // Realtime export type { PubSub, diff --git a/packages/contrail/tests/spaces-enrollment.test.ts b/packages/contrail/tests/spaces-enrollment.test.ts new file mode 100644 index 0000000..3b7ef5b --- /dev/null +++ b/packages/contrail/tests/spaces-enrollment.test.ts @@ -0,0 +1,402 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + registerAuthorityRoutes, + registerRecordHostRoutes, +} from "../src/core/spaces/router"; +import { HostedAdapter } from "../src/core/spaces/adapter"; +import { + buildVerifier, + createServiceAuthMiddleware, +} from "../src/core/spaces/auth"; +import { + generateAuthoritySigningKey, + issueCredential, + createBindingCredentialVerifier, +} from "../src/core/spaces/credentials"; +import { + createEnrollmentBindingResolver, + createLocalKeyResolver, +} from "../src/core/spaces/binding"; +import type { CredentialKeyMaterial } from "../src/core/spaces/credentials"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; +} + +function call( + app: Hono, + method: string, + path: string, + did: string | null, + body?: any, + extraHeaders?: Record +) { + const headers: Record = { ...(extraHeaders ?? {}) }; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +// --------------------------------------------------------------------------- +// In-process: createSpace auto-enrolls; non-enrolled spaces 404 +// --------------------------------------------------------------------------- + +describe("auto-enrollment via createSpace", () => { + function makeConfig(): ContrailConfig { + return { + namespace: "test.enroll", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + }, + recordHost: {}, + }, + }; + } + + async function makeApp(): Promise { + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + } + + it("createSpace creates an enrollment row alongside the space row", async () => { + const app = await makeApp(); + const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); + expect(create.status).toBe(200); + const uri = ((await create.json()) as any).space.uri; + + // Subsequent putRecord should succeed because the space is enrolled. + const put = await call(app, "POST", "/xrpc/test.enroll.space.putRecord", ALICE, { + spaceUri: uri, + collection: "app.event.message", + record: { $type: "app.event.message", text: "hi" }, + }); + expect(put.status).toBe(200); + }); + + it("explicit enroll via the endpoint is idempotent", async () => { + const app = await makeApp(); + const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + // Re-enrolling the same space (e.g. to update the authority binding). + const reenroll = await call(app, "POST", "/xrpc/test.enroll.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(reenroll.status).toBe(200); + expect(((await reenroll.json()) as any).ok).toBe(true); + }); + + it("non-owner / non-authority callers cannot enroll", async () => { + const app = await makeApp(); + const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + const res = await call(app, "POST", "/xrpc/test.enroll.recordHost.enroll", BOB, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(res.status).toBe(403); + expect((await res.json()).reason).toBe("not-owner-or-authority"); + }); +}); + +// --------------------------------------------------------------------------- +// Split deployment: authority on machine A, record host on machine B +// --------------------------------------------------------------------------- + +describe("split deployment — authority and record host on separate apps", () => { + function buildAuthorityApp(db: any): Hono { + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + // Authority-only deployment: no recordHost configured. + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + }, + }, + }; + const resolved = resolveConfig(cfg); + const adapter = new HostedAdapter(db, resolved); + const app = new Hono(); + registerAuthorityRoutes( + app, + adapter, + cfg.spaces!.authority!, + resolved, + fakeAuth(), + undefined, + null // no local record host → no auto-enroll + ); + return app; + } + + function buildRecordHostApp(db: any): Hono { + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + // Authority config still required to provide the auth middleware + // and serviceDid context, but signing can be omitted (host doesn't + // sign — it verifies). + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + const adapter = new HostedAdapter(db, resolved); + const verifier = createBindingCredentialVerifier({ + bindings: createEnrollmentBindingResolver({ recordHost: adapter }), + keys: createLocalKeyResolver({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }), + }); + const app = new Hono(); + registerRecordHostRoutes( + app, + adapter, + adapter, + cfg.spaces!.recordHost!, + resolved, + fakeAuth(), + verifier + ); + return app; + } + + it("end-to-end: create on authority, enroll on host, write+read via credential", async () => { + // Two physically-separate DBs — proves the host doesn't peek into the + // authority's storage to know about spaces. + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + + // 1. Authority creates a space — auto-enroll did NOT happen because the + // authority has no local record host. + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + expect(create.status).toBe(200); + const uri = ((await create.json()) as any).space.uri; + + // 2. Host rejects writes for the not-yet-enrolled space. + // The credential's binding resolver (EnrollmentBindingResolver) + // finds no enrollment → returns null → verifier 401 "unknown-issuer". + // This is a different layer than the requireEnrollment 404 (which + // fires on JWT/non-credential paths), but both block writes. + const cred1 = await call(authorityApp, "POST", "/xrpc/test.split.space.getCredential", ALICE, { + spaceUri: uri, + }); + const credential = ((await cred1.json()) as any).credential; + const earlyPut = await call( + hostApp, + "POST", + "/xrpc/test.split.space.putRecord", + null, + { spaceUri: uri, collection: "app.event.message", record: { $type: "app.event.message", text: "x" } }, + { "X-Space-Credential": credential } + ); + expect(earlyPut.status).toBe(401); + expect((await earlyPut.json()).reason).toBe("unknown-issuer"); + + // 3. Owner enrolls the space on the host. + const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(enroll.status).toBe(200); + + // 4. Now the host accepts writes via credential. + const put = await call( + hostApp, + "POST", + "/xrpc/test.split.space.putRecord", + null, + { spaceUri: uri, collection: "app.event.message", record: { $type: "app.event.message", text: "hello" } }, + { "X-Space-Credential": credential } + ); + expect(put.status).toBe(200); + expect(((await put.json()) as any).authorDid).toBe(ALICE); + + // 5. Read back via credential. + const list = await call( + hostApp, + "GET", + `/xrpc/test.split.space.listRecords?spaceUri=${encodeURIComponent(uri)}&collection=app.event.message`, + null, + undefined, + { "X-Space-Credential": credential } + ); + expect(list.status).toBe(200); + expect(((await list.json()) as any).records.length).toBe(1); + }); + + it("host rejects credentials whose iss doesn't match the enrollment", async () => { + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); + + // A credential signed by a *different* DID with the same key, but iss + // doesn't match the enrolled authority. + const { credential } = await issueCredential( + { + iss: "did:web:imposter.example", + sub: ALICE, + space: uri, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const res = await call( + hostApp, + "POST", + "/xrpc/test.split.space.putRecord", + null, + { spaceUri: uri, collection: "app.event.message", record: { $type: "app.event.message", text: "x" } }, + { "X-Space-Credential": credential } + ); + expect(res.status).toBe(401); + expect((await res.json()).reason).toBe("unknown-issuer"); + }); + + it("non-enrolled spaces 404 with not-enrolled on JWT-path reads", async () => { + // Service-auth JWTs go through requireEnrollment in the route handler, + // so a non-enrolled space gives the explicit "not-enrolled" reason. + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + const list = await call( + hostApp, + "GET", + `/xrpc/test.split.space.listRecords?spaceUri=${encodeURIComponent(uri)}&collection=app.event.message`, + ALICE + ); + expect(list.status).toBe(404); + expect((await list.json()).reason).toBe("not-enrolled"); + }); + + it("the declared authority can enroll a space without the owner's involvement", async () => { + // Scenario: the authority service ('Contrail' in our naming) is a + // separate DID and acts on behalf of an owner. The authority can enroll + // because phase 5 accepts either owner-or-authority. + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + // Authority service identifies itself via the JWT issuer matching its DID. + const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", SERVICE_DID, { + spaceUri: uri, + authority: SERVICE_DID, + }); + expect(enroll.status).toBe(200); + }); +}); diff --git a/packages/lexicons/lexicon-templates/recordHost/enroll.json b/packages/lexicons/lexicon-templates/recordHost/enroll.json new file mode 100644 index 0000000..2e7bc66 --- /dev/null +++ b/packages/lexicons/lexicon-templates/recordHost/enroll.json @@ -0,0 +1,39 @@ +{ + "lexicon": 1, + "id": "tools.atmo.recordHost.enroll", + "defs": { + "main": { + "type": "procedure", + "description": "Enroll a space on this record host. The host stores `(spaceUri, authority)` in its local enrollment table; subsequent record / blob operations for this space go through, and the credential verifier uses the enrolled authority as the canonical binding source. Caller must be either the space owner OR the declared authority — the host treats either as a sufficient signal of consent. Idempotent: re-enrolling updates the authority binding.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["spaceUri", "authority"], + "properties": { + "spaceUri": { "type": "string" }, + "authority": { + "type": "string", + "format": "did", + "description": "DID authorized to sign credentials for this space." + } + } + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["ok"], + "properties": { + "ok": { "type": "boolean" } + } + } + }, + "errors": [ + { "name": "InvalidRequest" }, + { "name": "Forbidden", "description": "Caller is neither the space owner nor the declared authority." } + ] + } + } +} -- 2.51.2 From 81d0f370e61638965f541520a60b1cb23b6321a3 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 18:30:32 +0200 Subject: [PATCH 06/19] phase seis --- apps/group-chat/package.json | 1 + apps/group-chat/src/lib/contrail/index.ts | 6 +- packages/contrail-community/package.json | 52 +++++++++++ .../src}/acl.ts | 0 .../src}/adapter.ts | 2 +- .../src}/credentials.ts | 0 .../src}/index.ts | 6 ++ .../contrail-community/src/integration.ts | 80 +++++++++++++++++ .../src}/invite-handler.ts | 6 +- .../src}/pds.ts | 0 .../src}/plc.ts | 0 .../src}/reconcile.ts | 2 +- .../src}/router.ts | 14 +-- .../src}/schema.ts | 5 +- .../src}/types.ts | 2 +- .../src}/whoami.ts | 2 +- .../tests/community-delegation.test.ts | 19 ++-- .../tests/community-e2e.test.ts | 19 ++-- .../tests/community-mint.test.ts | 21 +++-- .../tests/community-publishing.test.ts | 19 ++-- .../tests/invite-unified.test.ts | 19 ++-- .../contrail-community/tsconfig.build.json | 7 ++ packages/contrail-community/tsconfig.json | 7 ++ packages/contrail-community/tsup.config.ts | 11 +++ packages/contrail-community/vitest.config.ts | 17 ++++ packages/contrail/package.json | 1 + packages/contrail/src/contrail.ts | 27 +++++- .../src/core/community-integration.ts | 54 ++++++++++++ packages/contrail/src/core/db/schema.ts | 21 +++-- .../contrail/src/core/realtime/resolve.ts | 7 +- packages/contrail/src/core/realtime/router.ts | 4 +- .../contrail/src/core/router/collection.ts | 4 +- packages/contrail/src/core/router/index.ts | 86 +++++++++---------- packages/contrail/src/core/types.ts | 8 +- packages/contrail/src/index.ts | 56 ++++++------ packages/contrail/tests/realtime-e2e.test.ts | 5 +- packages/contrail/vitest.config.ts | 12 +++ pnpm-lock.yaml | 43 ++++++++++ 38 files changed, 502 insertions(+), 143 deletions(-) create mode 100644 packages/contrail-community/package.json rename packages/{contrail/src/core/community => contrail-community/src}/acl.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/adapter.ts (99%) rename packages/{contrail/src/core/community => contrail-community/src}/credentials.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/index.ts (77%) create mode 100644 packages/contrail-community/src/integration.ts rename packages/{contrail/src/core/community => contrail-community/src}/invite-handler.ts (97%) rename packages/{contrail/src/core/community => contrail-community/src}/pds.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/plc.ts (100%) rename packages/{contrail/src/core/community => contrail-community/src}/reconcile.ts (96%) rename packages/{contrail/src/core/community => contrail-community/src}/router.ts (99%) rename packages/{contrail/src/core/community => contrail-community/src}/schema.ts (93%) rename packages/{contrail/src/core/community => contrail-community/src}/types.ts (98%) rename packages/{contrail/src/core/community => contrail-community/src}/whoami.ts (94%) rename packages/{contrail => contrail-community}/tests/community-delegation.test.ts (94%) rename packages/{contrail => contrail-community}/tests/community-e2e.test.ts (93%) rename packages/{contrail => contrail-community}/tests/community-mint.test.ts (90%) rename packages/{contrail => contrail-community}/tests/community-publishing.test.ts (93%) rename packages/{contrail => contrail-community}/tests/invite-unified.test.ts (92%) create mode 100644 packages/contrail-community/tsconfig.build.json create mode 100644 packages/contrail-community/tsconfig.json create mode 100644 packages/contrail-community/tsup.config.ts create mode 100644 packages/contrail-community/vitest.config.ts create mode 100644 packages/contrail/src/core/community-integration.ts diff --git a/apps/group-chat/package.json b/apps/group-chat/package.json index 54b950a..a773f5f 100644 --- a/apps/group-chat/package.json +++ b/apps/group-chat/package.json @@ -60,6 +60,7 @@ "dependencies": { "@atcute/jetstream": "^1.1.2", "@atmo-dev/contrail": "workspace:*", + "@atmo-dev/contrail-community": "workspace:*", "@atmo-dev/contrail-sync": "workspace:*", "@atmo-dev/contrail-lexicons": "workspace:*", "@foxui/core": "^0.9.1", diff --git a/apps/group-chat/src/lib/contrail/index.ts b/apps/group-chat/src/lib/contrail/index.ts index 85a6aa3..8baf918 100644 --- a/apps/group-chat/src/lib/contrail/index.ts +++ b/apps/group-chat/src/lib/contrail/index.ts @@ -4,12 +4,14 @@ import { InMemoryPubSub, MemoryBlobAdapter, R2BlobAdapter, + resolveConfig, type BlobAdapter, type ContrailConfig, type DurableObjectNamespace, type PubSub, type R2BucketLike } from '@atmo-dev/contrail'; +import { createCommunityIntegration } from '@atmo-dev/contrail-community'; import { createHandler, createServerClient } from '@atmo-dev/contrail/server'; import type { Client } from '@atcute/client'; import { dev } from '$app/environment'; @@ -65,7 +67,9 @@ function build(env: Env): Bundle { } }; - const contrail = new Contrail(config); + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ db: env.DB, config: resolved }); + const contrail = new Contrail({ ...config, db: env.DB, communityIntegration }); const handle = createHandler(contrail); const ready = contrail.init(env.DB); diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json new file mode 100644 index 0000000..2fae478 --- /dev/null +++ b/packages/contrail-community/package.json @@ -0,0 +1,52 @@ +{ + "name": "@atmo-dev/contrail-community", + "version": "0.1.0", + "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-community" + }, + "keywords": [ + "atproto", + "contrail", + "community" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit", + "test": "vitest run", + "test:watch": "vitest" + }, + "dependencies": { + "@atcute/atproto": "^3.1.10", + "@atcute/cbor": "^2.3.2", + "@atcute/identity": "^1.1.4", + "@atcute/identity-resolver": "^1.2.2", + "@atcute/lexicons": "^1.2.9", + "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "tsup": "^8.5.0", + "typescript": "^5.7.3", + "vitest": "^4.1.0" + }, + "license": "MIT" +} diff --git a/packages/contrail/src/core/community/acl.ts b/packages/contrail-community/src/acl.ts similarity index 100% rename from packages/contrail/src/core/community/acl.ts rename to packages/contrail-community/src/acl.ts diff --git a/packages/contrail/src/core/community/adapter.ts b/packages/contrail-community/src/adapter.ts similarity index 99% rename from packages/contrail/src/core/community/adapter.ts rename to packages/contrail-community/src/adapter.ts index 1587e63..586fa88 100644 --- a/packages/contrail/src/core/community/adapter.ts +++ b/packages/contrail-community/src/adapter.ts @@ -1,4 +1,4 @@ -import type { Database } from "../types"; +import type { Database } from "@atmo-dev/contrail"; import type { AccessLevel, AccessLevelRow, diff --git a/packages/contrail/src/core/community/credentials.ts b/packages/contrail-community/src/credentials.ts similarity index 100% rename from packages/contrail/src/core/community/credentials.ts rename to packages/contrail-community/src/credentials.ts diff --git a/packages/contrail/src/core/community/index.ts b/packages/contrail-community/src/index.ts similarity index 77% rename from packages/contrail/src/core/community/index.ts rename to packages/contrail-community/src/index.ts index fd0cc53..d349351 100644 --- a/packages/contrail/src/core/community/index.ts +++ b/packages/contrail-community/src/index.ts @@ -35,3 +35,9 @@ export { jwkToDidKey, } from "./plc"; export type { KeyPair, GenesisOpInput, UnsignedGenesisOp, SignedGenesisOp } from "./plc"; + +// The headline export — wire community into a contrail app via: +// const community = createCommunityIntegration({ db, config }); +// const app = createApp(db, config, { community }); +export { createCommunityIntegration } from "./integration"; +export type { CommunityIntegrationOptions } from "./integration"; diff --git a/packages/contrail-community/src/integration.ts b/packages/contrail-community/src/integration.ts new file mode 100644 index 0000000..9dca172 --- /dev/null +++ b/packages/contrail-community/src/integration.ts @@ -0,0 +1,80 @@ +/** Factory that builds a {@link CommunityIntegration} for contrail's + * `createApp({ community })` option. The integration is opaque to contrail + * core — it just exposes the hooks the umbrella router needs. */ + +import type { + CommunityIntegration, + CommunityProbe, + ContrailConfig, + Database, + StorageAdapter, +} from "@atmo-dev/contrail"; +import { HostedAdapter, buildVerifier, createServiceAuthMiddleware } from "@atmo-dev/contrail"; +import { CommunityAdapter } from "./adapter"; +import { createCommunityWhoamiExtension } from "./whoami"; +import { createCommunityInviteHandler } from "./invite-handler"; +import { resolveReachableSpaces } from "./acl"; +import { registerCommunityRoutes } from "./router"; +import { buildCommunitySchema } from "./schema"; +import { getDialect } from "@atmo-dev/contrail"; + +export interface CommunityIntegrationOptions { + /** Database the community tables live in. Should be the same DB the + * spaces module uses (community rows reference space_uri). */ + db: Database; + /** Resolved contrail config. */ + config: ContrailConfig; + /** Optional override of the community adapter (for tests). */ + communityAdapter?: CommunityAdapter; + /** Optional override of the spaces adapter (for tests). Otherwise built + * from the same db using HostedAdapter. */ + spacesAdapter?: StorageAdapter; +} + +export function createCommunityIntegration( + options: CommunityIntegrationOptions +): CommunityIntegration { + const { db, config } = options; + const community = options.communityAdapter ?? new CommunityAdapter(db); + const spaces = options.spacesAdapter ?? new HostedAdapter(db, config); + + const probe: CommunityProbe = { + async getCommunity(did) { + return community.getCommunity(did); + }, + async resolveReachableSpaces(callerDid) { + return resolveReachableSpaces(community, callerDid); + }, + }; + + const whoamiExtension = createCommunityWhoamiExtension({ community }); + const inviteHandler = createCommunityInviteHandler({ + community, + authority: spaces, + }); + + return { + probe, + whoamiExtension, + inviteHandler, + registerRoutes(app, opts) { + // Reuse the spaces JWT verifier — the auth model is identical. + if (!config.spaces?.authority) return; + const verifier = buildVerifier(config.spaces.authority); + const authMiddleware = + opts?.authMiddleware ?? createServiceAuthMiddleware(verifier); + registerCommunityRoutes( + app, + db, + config, + { authMiddleware, communityAdapter: community, spacesAdapter: spaces }, + { spacesAdapter: spaces, verifier } + ); + }, + async applySchema(target) { + const dialect = getDialect(target); + const stmts = buildCommunitySchema(dialect); + await target.batch(stmts.map((s) => target.prepare(s))); + }, + }; +} diff --git a/packages/contrail/src/core/community/invite-handler.ts b/packages/contrail-community/src/invite-handler.ts similarity index 97% rename from packages/contrail/src/core/community/invite-handler.ts rename to packages/contrail-community/src/invite-handler.ts index 0b41f8a..9f27f1d 100644 --- a/packages/contrail/src/core/community/invite-handler.ts +++ b/packages/contrail-community/src/invite-handler.ts @@ -5,10 +5,10 @@ import type { CommunityInviteHandler, HandlerResponse, -} from "../invite/community-handler"; -import { mintInviteToken } from "../invite/token"; + SpaceAuthority, +} from "@atmo-dev/contrail"; +import { mintInviteToken } from "@atmo-dev/contrail"; import type { CommunityAdapter } from "./adapter"; -import type { SpaceAuthority } from "../spaces/types"; import { resolveEffectiveLevel } from "./acl"; import { reconcile } from "./reconcile"; import type { AccessLevel, CommunityInviteRow } from "./types"; diff --git a/packages/contrail/src/core/community/pds.ts b/packages/contrail-community/src/pds.ts similarity index 100% rename from packages/contrail/src/core/community/pds.ts rename to packages/contrail-community/src/pds.ts diff --git a/packages/contrail/src/core/community/plc.ts b/packages/contrail-community/src/plc.ts similarity index 100% rename from packages/contrail/src/core/community/plc.ts rename to packages/contrail-community/src/plc.ts diff --git a/packages/contrail/src/core/community/reconcile.ts b/packages/contrail-community/src/reconcile.ts similarity index 96% rename from packages/contrail/src/core/community/reconcile.ts rename to packages/contrail-community/src/reconcile.ts index 5fde1d3..1b0bbb0 100644 --- a/packages/contrail/src/core/community/reconcile.ts +++ b/packages/contrail-community/src/reconcile.ts @@ -1,4 +1,4 @@ -import type { SpaceAuthority } from "../spaces/types"; +import type { SpaceAuthority } from "@atmo-dev/contrail"; import type { CommunityAdapter } from "./adapter"; import { flattenEffectiveMembers } from "./acl"; diff --git a/packages/contrail/src/core/community/router.ts b/packages/contrail-community/src/router.ts similarity index 99% rename from packages/contrail/src/core/community/router.ts rename to packages/contrail-community/src/router.ts index 84e86d1..23c3123 100644 --- a/packages/contrail/src/core/community/router.ts +++ b/packages/contrail-community/src/router.ts @@ -1,9 +1,11 @@ import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig, Database } from "../types"; -import type { ServiceAuth } from "../spaces/auth"; -import type { StorageAdapter as SpacesAdapter } from "../spaces/types"; -import { buildSpaceUri } from "../spaces/uri"; -import { HostedAdapter } from "../spaces/adapter"; +import type { + ContrailConfig, + Database, + ServiceAuth, + StorageAdapter as SpacesAdapter, +} from "@atmo-dev/contrail"; +import { buildSpaceUri, HostedAdapter } from "@atmo-dev/contrail"; import { CommunityAdapter } from "./adapter"; import { CredentialCipher } from "./credentials"; import { resolveIdentity, createPdsSession } from "./pds"; @@ -43,7 +45,7 @@ export function registerCommunityRoutes( verifier: ServiceJwtVerifier; } | null ): void { - const cfg = config.community; + const cfg = config.community as import("./types").CommunityConfig | undefined; if (!cfg) return; if (!config.spaces?.authority) { throw new Error("community module requires spaces.authority to be enabled in config"); diff --git a/packages/contrail/src/core/community/schema.ts b/packages/contrail-community/src/schema.ts similarity index 93% rename from packages/contrail/src/core/community/schema.ts rename to packages/contrail-community/src/schema.ts index b7d382c..cdcd56e 100644 --- a/packages/contrail/src/core/community/schema.ts +++ b/packages/contrail-community/src/schema.ts @@ -1,6 +1,5 @@ -import type { Database } from "../types"; -import { getDialect } from "../dialect"; -import type { SqlDialect } from "../dialect"; +import type { Database, SqlDialect } from "@atmo-dev/contrail"; +import { getDialect } from "@atmo-dev/contrail"; export function buildCommunitySchema(dialect: SqlDialect): string[] { return [ diff --git a/packages/contrail/src/core/community/types.ts b/packages/contrail-community/src/types.ts similarity index 98% rename from packages/contrail/src/core/community/types.ts rename to packages/contrail-community/src/types.ts index 7959e72..7f74698 100644 --- a/packages/contrail/src/core/community/types.ts +++ b/packages/contrail-community/src/types.ts @@ -1,4 +1,4 @@ -import type { Database } from "../types"; +import type { Database } from "@atmo-dev/contrail"; import type { DidDocumentResolver } from "@atcute/identity-resolver"; /** Access level a subject (did or group-space) has on a given space. diff --git a/packages/contrail/src/core/community/whoami.ts b/packages/contrail-community/src/whoami.ts similarity index 94% rename from packages/contrail/src/core/community/whoami.ts rename to packages/contrail-community/src/whoami.ts index 4a75a34..5f06123 100644 --- a/packages/contrail/src/core/community/whoami.ts +++ b/packages/contrail-community/src/whoami.ts @@ -2,7 +2,7 @@ * for community-owned spaces. Returns null for non-community spaces so the * spaces module's default binary-membership logic runs. */ -import type { WhoamiExtension } from "../spaces/router"; +import type { WhoamiExtension } from "@atmo-dev/contrail"; import type { CommunityAdapter } from "./adapter"; import { resolveEffectiveLevel } from "./acl"; diff --git a/packages/contrail/tests/community-delegation.test.ts b/packages/contrail-community/tests/community-delegation.test.ts similarity index 94% rename from packages/contrail/tests/community-delegation.test.ts rename to packages/contrail-community/tests/community-delegation.test.ts index 1514964..5449e7c 100644 --- a/packages/contrail/tests/community-delegation.test.ts +++ b/packages/contrail-community/tests/community-delegation.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll, beforeEach } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -79,8 +80,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/community-e2e.test.ts b/packages/contrail-community/tests/community-e2e.test.ts similarity index 93% rename from packages/contrail/tests/community-e2e.test.ts rename to packages/contrail-community/tests/community-e2e.test.ts index db008e6..95d6048 100644 --- a/packages/contrail/tests/community-e2e.test.ts +++ b/packages/contrail-community/tests/community-e2e.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -90,8 +91,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/community-mint.test.ts b/packages/contrail-community/tests/community-mint.test.ts similarity index 90% rename from packages/contrail/tests/community-mint.test.ts rename to packages/contrail-community/tests/community-mint.test.ts index c83491f..02ea73b 100644 --- a/packages/contrail/tests/community-mint.test.ts +++ b/packages/contrail-community/tests/community-mint.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; import { buildGenesisOp, computeDidPlc, @@ -13,7 +14,7 @@ import { generateKeyPair, jwkToDidKey, signGenesisOp, -} from "../src/core/community/plc"; +} from "../src/plc"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -63,8 +64,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/community-publishing.test.ts b/packages/contrail-community/tests/community-publishing.test.ts similarity index 93% rename from packages/contrail/tests/community-publishing.test.ts rename to packages/contrail-community/tests/community-publishing.test.ts index 10a3d76..a05ee0f 100644 --- a/packages/contrail/tests/community-publishing.test.ts +++ b/packages/contrail-community/tests/community-publishing.test.ts @@ -1,11 +1,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -93,8 +94,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call( diff --git a/packages/contrail/tests/invite-unified.test.ts b/packages/contrail-community/tests/invite-unified.test.ts similarity index 92% rename from packages/contrail/tests/invite-unified.test.ts rename to packages/contrail-community/tests/invite-unified.test.ts index b56d81d..32bd6c5 100644 --- a/packages/contrail/tests/invite-unified.test.ts +++ b/packages/contrail-community/tests/invite-unified.test.ts @@ -7,11 +7,12 @@ import { describe, it, expect, beforeAll } from "vitest"; import { Hono } from "hono"; import type { MiddlewareHandler } from "hono"; -import { createSqliteDatabase } from "../src/adapters/sqlite"; -import { initSchema } from "../src/core/db/schema"; -import { createApp } from "../src/core/router"; -import { resolveConfig } from "../src/core/types"; -import type { ContrailConfig } from "../src/core/types"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { initSchema } from "@atmo-dev/contrail"; +import { createApp } from "@atmo-dev/contrail"; +import { resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -66,8 +67,12 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); - return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); } function call(app: Hono, method: string, path: string, did: string, body?: any): Promise { diff --git a/packages/contrail-community/tsconfig.build.json b/packages/contrail-community/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-community/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-community/tsconfig.json b/packages/contrail-community/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-community/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-community/tsup.config.ts b/packages/contrail-community/tsup.config.ts new file mode 100644 index 0000000..b033415 --- /dev/null +++ b/packages/contrail-community/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["@atmo-dev/contrail"], +}); diff --git a/packages/contrail-community/vitest.config.ts b/packages/contrail-community/vitest.config.ts new file mode 100644 index 0000000..56720e0 --- /dev/null +++ b/packages/contrail-community/vitest.config.ts @@ -0,0 +1,17 @@ +import { defineConfig } from "vitest/config"; +import path from "node:path"; + +const contrailSrc = path.resolve(__dirname, "../contrail/src"); + +// Alias `@atmo-dev/contrail` and its subpaths to the source so tests don't +// run through the built dist. Mirrors the in-tree-source-resolution pattern +// the contrail package's own tests use (they import via ../src/...). +export default defineConfig({ + resolve: { + alias: { + "@atmo-dev/contrail/sqlite": path.join(contrailSrc, "adapters/sqlite.ts"), + "@atmo-dev/contrail/postgres": path.join(contrailSrc, "adapters/postgres.ts"), + "@atmo-dev/contrail": path.join(contrailSrc, "index.ts"), + }, + }, +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index a64484d..846e6e8 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -78,6 +78,7 @@ "jiti": "^2.4.0" }, "devDependencies": { + "@atmo-dev/contrail-community": "workspace:*", "@cloudflare/workers-types": "^4.20250124.0", "@types/node": "^25.5.0", "@types/pg": "^8.20.0", diff --git a/packages/contrail/src/contrail.ts b/packages/contrail/src/contrail.ts index 7c52aa0..dcefac7 100644 --- a/packages/contrail/src/contrail.ts +++ b/packages/contrail/src/contrail.ts @@ -21,27 +21,43 @@ import { import type { PubSub } from "./core/realtime/types"; import { InMemoryPubSub } from "./core/realtime/in-memory"; import { createApp, type CreateAppOptions } from "./core/router"; +import type { CommunityIntegration } from "./core/community-integration"; import type { Hono } from "hono"; -export interface ContrailOptions extends ContrailConfig { +/** Note: `community` is shadowed from ContrailConfig (where it's the + * user-supplied config blob, typed as `unknown`) to be the pre-built + * integration object the Contrail instance actually consumes. */ +export interface ContrailOptions extends Omit { db?: Database; /** Optional separate DB for permissioned spaces tables. Defaults to `db`. */ spacesDb?: Database; + /** Optional user-supplied community config blob. Same shape as + * ContrailConfig.community — the community integration reads this + * via `config.community`. */ + community?: unknown; + /** Optional pre-built community integration. When set, the Contrail + * instance applies its schema during `init()` and forwards it to + * `createApp` so community routes / hooks are wired automatically. + * Construct via `createCommunityIntegration(...)` from + * `@atmo-dev/contrail-community`. */ + communityIntegration?: CommunityIntegration; } export class Contrail { readonly config: ResolvedContrailConfig; private _db?: Database; private _spacesDb?: Database; + private _community?: CommunityIntegration; private _ingestState: IngestState = createIngestState(); private _pubsub: PubSub | null = null; constructor(options: ContrailOptions) { - const { db, spacesDb, ...configInput } = options; - this.config = resolveConfig(configInput); + const { db, spacesDb, communityIntegration, ...configInput } = options; + this.config = resolveConfig(configInput as ContrailConfig); validateConfig(this.config); this._db = db; this._spacesDb = spacesDb; + this._community = communityIntegration; // Build the pubsub instance up-front so ingestion and HTTP routes share // it. Caller overrides via `config.realtime.pubsub` (e.g. DurableObject). if (this.config.realtime) { @@ -72,7 +88,8 @@ export class Contrail { async init(db?: Database, spacesDb?: Database): Promise { const main = this.getDb(db); const spaces = spacesDb ?? this._spacesDb; - await initSchema(main, this.config, { spacesDb: spaces }); + const extraSchemas = this._community ? [this._community.applySchema] : []; + await initSchema(main, this.config, { spacesDb: spaces, extraSchemas }); } /** Query records from a collection. */ @@ -273,6 +290,8 @@ export class Contrail { return createApp(main, this.config, { ...appOpts, spacesDb: spaces, + // Per-call community override falls back to the constructor's. + community: appOpts.community ?? this._community ?? null, realtime: { ...appOpts.realtime, pubsub: this._pubsub ?? undefined }, }); } diff --git a/packages/contrail/src/core/community-integration.ts b/packages/contrail/src/core/community-integration.ts new file mode 100644 index 0000000..d91b3a4 --- /dev/null +++ b/packages/contrail/src/core/community-integration.ts @@ -0,0 +1,54 @@ +/** Pluggable integration surface for the community module. + * + * Phase 6 extracted community to its own package (`@atmo-dev/contrail-community`). + * The contrail core package never imports from it — couplings only flow + * through these interfaces. The community package's + * `createCommunityIntegration({ ... })` returns a {@link CommunityIntegration}, + * which the consumer hands to `createApp` via `options.community`. + * + * Two layers: + * - {@link CommunityProbe}: minimal "is this a community DID" / "what + * spaces does this caller reach" surface used by realtime + collection + * routes for community-aware dispatch. + * - {@link CommunityIntegration}: the umbrella bundle — probe, whoami + * extension, invite handler, plus route + schema wiring that the + * umbrella router calls during setup. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { Database } from "./types"; +import type { CommunityInviteHandler } from "./invite/community-handler"; +import type { WhoamiExtension } from "./spaces/router"; + +/** Narrow interface for the deep callers (realtime/resolve, router/collection) + * that just need to ask "is this a community DID?" or "what spaces does this + * caller reach via community membership?" */ +export interface CommunityProbe { + /** Look up a community row by DID. Returns null for non-community DIDs. + * Callers usually only check truthiness — community-specific fields stay + * inside the community package. */ + getCommunity(did: string): Promise<{ did: string } | null>; + + /** Resolve the set of space URIs reachable by `callerDid` through community + * membership (direct grants + delegations). Used by realtime to expand + * community: topics into the caller's concrete space: topics. */ + resolveReachableSpaces(callerDid: string): Promise>; +} + +/** Umbrella integration the consumer constructs once and hands to createApp. + * contrail core treats this as an opaque bundle — it doesn't introspect + * community state, just calls these methods at the right wiring points. */ +export interface CommunityIntegration { + /** Probe used by realtime + collection cross-cutting concerns. */ + probe: CommunityProbe; + /** Whoami extension that returns `accessLevel` for community-owned spaces. */ + whoamiExtension: WhoamiExtension; + /** Handler for the community-grant path of the unified invite surface. */ + inviteHandler: CommunityInviteHandler; + /** Register `.community.*` routes onto the Hono app. */ + registerRoutes( + app: Hono, + options?: { authMiddleware?: MiddlewareHandler } + ): void; + /** Apply community schema (DDL) to the database. Called by initSchema. */ + applySchema(db: Database): Promise; +} diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 5ef78e8..562a6ee 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -11,7 +11,6 @@ import { } from "../types"; import { getSearchableFields } from "../search"; import { buildSpacesBaseSchema } from "../spaces/schema"; -import { buildCommunitySchema } from "../community/schema"; import { buildLabelsSchema } from "../labels/schema"; function getResolved(config: ContrailConfig): ResolvedMaps { @@ -262,9 +261,17 @@ async function runMigrations(db: Database): Promise { } } +/** Pluggable schema applier — passed in by extension packages (community, + * third-party plugins) to install their own tables alongside contrail's. */ +export type SchemaModule = (db: Database) => Promise; + export interface InitSchemaOptions { /** Separate DB for the spaces tables. Defaults to the main `db`. */ spacesDb?: Database; + /** Extra schema modules to apply after contrail's own DDL. Used by the + * community package to install its tables — contrail core no longer + * imports community schema directly. */ + extraSchemas?: SchemaModule[]; } async function applySpacesSchema( @@ -311,11 +318,13 @@ export async function initSchema( await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); } - if (config.community) { - // Community tables live on the same DB as spaces (they reference space_uri). - const target = spacesSharesMainDb ? db : spacesDb!; - const communityStmts = buildCommunitySchema(dialect); - await target.batch(communityStmts.map((s) => target.prepare(s))); + // Extension schemas (e.g. community) — applied to the spacesDb when one's + // configured separately, since they typically reference space_uri. The + // caller is responsible for routing the schema to the right db; we just + // hand it the spaces-or-main DB as a sensible default. + const extensionTarget = spacesSharesMainDb ? db : spacesDb!; + for (const apply of options.extraSchemas ?? []) { + await apply(extensionTarget); } if (config.labels) { diff --git a/packages/contrail/src/core/realtime/resolve.ts b/packages/contrail/src/core/realtime/resolve.ts index f8a0ab4..c16abe5 100644 --- a/packages/contrail/src/core/realtime/resolve.ts +++ b/packages/contrail/src/core/realtime/resolve.ts @@ -12,8 +12,7 @@ * (not yet implemented). */ import type { StorageAdapter } from "../spaces/types"; -import type { CommunityAdapter } from "../community/adapter"; -import { resolveReachableSpaces } from "../community/acl"; +import type { CommunityProbe } from "../community-integration"; import { spaceTopic, parseCommunityTopic, parseSpaceTopic } from "./types"; export interface TopicResolutionContext { @@ -22,7 +21,7 @@ export interface TopicResolutionContext { * (`collection:`, `actor:`) still resolve. */ spaces: StorageAdapter | null; /** May be null if the community module is not enabled. */ - community: CommunityAdapter | null; + community: CommunityProbe | null; } export interface TopicResolution { @@ -63,7 +62,7 @@ export async function resolveTopicForCaller( } const row = await ctx.community.getCommunity(communityDid); if (!row) return { ok: false, error: "NotFound", reason: "community-not-found" }; - const reachable = await resolveReachableSpaces(ctx.community, callerDid); + const reachable = await ctx.community.resolveReachableSpaces(callerDid); // Filter to spaces owned by THIS community — reachable may include spaces // from other communities via cross-community delegation. const ownedList = await ctx.spaces.listSpaces({ ownerDid: communityDid, limit: 1000 }); diff --git a/packages/contrail/src/core/realtime/router.ts b/packages/contrail/src/core/realtime/router.ts index 2726226..d10e45e 100644 --- a/packages/contrail/src/core/realtime/router.ts +++ b/packages/contrail/src/core/realtime/router.ts @@ -4,7 +4,7 @@ import type { Context, Hono, MiddlewareHandler } from "hono"; import type { ContrailConfig } from "../types"; import type { ServiceAuth } from "../spaces/auth"; import type { StorageAdapter } from "../spaces/types"; -import type { CommunityAdapter } from "../community/adapter"; +import type { CommunityProbe } from "../community-integration"; import { InMemoryPubSub } from "./in-memory"; import { TicketSigner } from "./ticket"; import { sseResponse } from "./sse"; @@ -40,7 +40,7 @@ export function registerRealtimeRoutes( app: Hono, config: ContrailConfig, spaces: StorageAdapter | null, - community: CommunityAdapter | null, + community: CommunityProbe | null, options: RealtimeRoutesOptions ): void { const cfg = config.realtime; diff --git a/packages/contrail/src/core/router/collection.ts b/packages/contrail/src/core/router/collection.ts index e440522..6d43d00 100644 --- a/packages/contrail/src/core/router/collection.ts +++ b/packages/contrail/src/core/router/collection.ts @@ -31,7 +31,7 @@ import { DurableObjectPubSub } from "../realtime/durable-object"; import { TicketSigner, type TicketQuerySpec } from "../realtime/ticket"; import { resolveTopicForCaller } from "../realtime/resolve"; import { mergeAsyncIterables } from "../realtime/merge"; -import type { CommunityAdapter } from "../community/adapter"; +import type { CommunityProbe } from "../community-integration"; import { getRelationField, getNestedValue } from "../types"; /** Scope of a watch stream. @@ -497,7 +497,7 @@ export function registerCollectionRoutes( spacesCtx?: SpacesContext | null, options: { pubsub?: import("../realtime/types").PubSub | null; - community?: CommunityAdapter | null; + community?: CommunityProbe | null; } = {} ): void { const ns = config.namespace; diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 535c15f..0c5e058 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -12,11 +12,7 @@ import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; import { HostedAdapter } from "../spaces/adapter"; import type { StorageAdapter } from "../spaces/types"; import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; -import { registerCommunityRoutes } from "../community/router"; -import type { CommunityRoutesOptions } from "../community/router"; -import { CommunityAdapter } from "../community/adapter"; -import { createCommunityInviteHandler } from "../community/invite-handler"; -import { createCommunityWhoamiExtension } from "../community/whoami"; +import type { CommunityIntegration } from "../community-integration"; import { registerRealtimeRoutes } from "../realtime/router"; import type { RealtimeRoutesOptions } from "../realtime/router"; import { registerInviteRoutes } from "../invite/router"; @@ -28,6 +24,7 @@ import { resolveProfiles } from "./profiles"; import { backfillUser } from "../backfill"; import { selectAcceptedLabelers } from "../labels/select"; import { hydrateLabels } from "../labels/hydrate"; +import type { MiddlewareHandler } from "hono"; export interface SpacesContext { adapter: StorageAdapter; @@ -36,7 +33,13 @@ export interface SpacesContext { export interface CreateAppOptions { spaces?: SpacesRoutesOptions; - community?: CommunityRoutesOptions; + /** Pre-built community integration. Construct via the community package's + * `createCommunityIntegration({ ... })`. When set, contrail wires + * community whoami extension, invite handler, route registration, etc. + * When omitted, deployment runs without community features. */ + community?: CommunityIntegration | null; + /** Auth middleware override for community routes (rare — mostly for tests). */ + communityAuthMiddleware?: MiddlewareHandler; realtime?: Partial; /** Separate DB for the spaces tables. Defaults to `db`. */ spacesDb?: Database; @@ -131,11 +134,11 @@ export function createApp( } : null; - // Community is wired up at this layer, not from inside spaces / invite — - // those modules consume injected hooks, not community internals. The - // adapter is shared across every call site that needs it (publishing - // wrapper, collection routes, whoami extension, invite handler, realtime). - const communityAdapter = config.community ? new CommunityAdapter(spacesDb) : null; + // Community is provided as a pre-built integration — contrail core never + // imports from the community package. The integration object is opaque; + // we just pass through its probe / whoamiExtension / inviteHandler / + // registerRoutes hooks at the right wiring points. + const community = options.community ?? null; // Realtime pubsub is built whenever realtime is configured — independent of // spaces. With spaces, the spaces adapter is wrapped so private record/member @@ -149,8 +152,8 @@ export function createApp( queueBound: config.realtime.queueBound, }); if (spacesCtx) { - const isCommunityDid = communityAdapter - ? cachedIsCommunityDid(communityAdapter) + const isCommunityDid = community + ? cachedIsCommunityDid(community.probe) : undefined; spacesCtx = { ...spacesCtx, @@ -163,52 +166,43 @@ export function createApp( registerCollectionRoutes(app, db, config, spacesCtx, { pubsub: realtimePubsub, - community: communityAdapter, + community: community?.probe ?? null, }); registerFeedRoutes(app, db, config); registerNotifyRoute(app, db, config); - // Spaces routes — get a whoami extension when community is configured so - // community-owned spaces get an `accessLevel` field. + // Spaces routes — get a whoami extension from the community integration + // when one's wired so community-owned spaces get an `accessLevel` field. const spacesOptions = { ...options.spaces, whoamiExtension: - options.spaces?.whoamiExtension ?? - (communityAdapter - ? createCommunityWhoamiExtension({ community: communityAdapter }) - : undefined), + options.spaces?.whoamiExtension ?? community?.whoamiExtension, }; registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); - if (config.community && spacesCtx) { + if (community && spacesCtx) { // Community routes reuse the spaces service-auth middleware (same JWT verifier). const authMiddleware = - options.community?.authMiddleware ?? + options.communityAuthMiddleware ?? options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier); - registerCommunityRoutes( - app, - spacesDb, - config, - { ...options.community, authMiddleware }, - { spacesAdapter: spacesCtx.adapter, verifier: spacesCtx.verifier } - ); + community.registerRoutes(app, { authMiddleware }); } if (config.spaces?.authority && spacesCtx) { // Unified invite surface: one `.invite.*` family that dispatches on // space ownership (user-owned → addMember; community-owned → grant via - // an injected community-invite handler). + // the integration's invite handler). const authMiddleware = options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier); - const inviteHandler = communityAdapter - ? createCommunityInviteHandler({ - community: communityAdapter, - authority: spacesCtx.adapter, - }) - : null; - registerInviteRoutes(app, config, spacesCtx.adapter, inviteHandler, { authMiddleware }); + registerInviteRoutes( + app, + config, + spacesCtx.adapter, + community?.inviteHandler ?? null, + { authMiddleware } + ); } if (config.realtime && realtimePubsub) { @@ -221,17 +215,23 @@ export function createApp( options.spaces?.authMiddleware ?? createServiceAuthMiddleware(spacesCtx.verifier) : null; - registerRealtimeRoutes(app, config, spacesCtx?.adapter ?? null, communityAdapter, { - authMiddleware, - pubsub: realtimePubsub, - }); + registerRealtimeRoutes( + app, + config, + spacesCtx?.adapter ?? null, + community?.probe ?? null, + { + authMiddleware, + pubsub: realtimePubsub, + } + ); } return app; } function cachedIsCommunityDid( - community: CommunityAdapter + probe: import("../community-integration").CommunityProbe ): (did: string) => Promise { const TTL = 60_000; const cache = new Map(); @@ -239,7 +239,7 @@ function cachedIsCommunityDid( const now = Date.now(); const hit = cache.get(did); if (hit && hit.expires > now) return hit.value; - const row = await community.getCommunity(did); + const row = await probe.getCommunity(did); const value = row != null; cache.set(did, { value, expires: now + TTL }); return value; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index b895141..3b9dc48 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -155,9 +155,11 @@ export interface ContrailConfig { notify?: boolean | string; /** Permissioned spaces configuration. When set, the service exposes space XRPCs. */ spaces?: import("./spaces/types").SpacesConfig; - /** Community module configuration. When set, the service exposes community XRPCs - * for managing community-owned spaces and tiered access levels. Requires `spaces`. */ - community?: import("./community/types").CommunityConfig; + /** Community module configuration. Typed by the community package via + * declaration merging — contrail core only knows it's "something the + * community package consumes." Set when wiring community via + * `createCommunityIntegration({ ... })`. Requires `spaces.authority`. */ + community?: unknown; /** Realtime module configuration. When set, the service exposes ticket + SSE/WS * subscribe XRPCs, and wraps the spaces adapter to publish events after writes. */ realtime?: import("./realtime/types").RealtimeConfig; diff --git a/packages/contrail/src/index.ts b/packages/contrail/src/index.ts index bbd8efe..63d07ec 100644 --- a/packages/contrail/src/index.ts +++ b/packages/contrail/src/index.ts @@ -110,6 +110,34 @@ export { registerRecordHostRoutes, } from "./core/spaces/router"; export type { EnrollmentRow } from "./core/spaces/types"; +export type { WhoamiExtension } from "./core/spaces/router"; + +// Internal-but-exposed bits — extension packages (community, etc.) need +// these to wire themselves up. Consumer apps generally don't. +export { buildSpaceUri, parseSpaceUri } from "./core/spaces/uri"; +export type { ServiceAuth } from "./core/spaces/auth"; +export { buildVerifier, createServiceAuthMiddleware } from "./core/spaces/auth"; +export { getDialect } from "./core/dialect"; +export type { SqlDialect } from "./core/dialect"; + +// App + schema wiring — extension packages and tests use these. +export { createApp } from "./core/router"; +export type { CreateAppOptions, SpacesContext } from "./core/router"; +export { initSchema } from "./core/db/schema"; +export type { InitSchemaOptions } from "./core/db/schema"; + +// Community integration interfaces — contrail core defines the shapes +// extension packages implement. The @atmo-dev/contrail-community package +// provides the concrete implementations. +export type { + CommunityIntegration, + CommunityProbe, +} from "./core/community-integration"; +export type { + CommunityInviteHandler, + HandlerResponse, +} from "./core/invite/community-handler"; +export type { SchemaModule } from "./core/db/schema"; // Realtime export type { @@ -160,28 +188,6 @@ export { export type { PersistentLabelsOptions } from "./core/labels/subscribe"; export { resolveLabelerEndpoint } from "./core/labels/resolve"; -// Community -export { - CommunityAdapter, - CredentialCipher, - registerCommunityRoutes, - ACCESS_LEVELS, - RESERVED_KEYS, - isAccessLevel, - isReservedKey, - rankOf, - resolveEffectiveLevel, - flattenEffectiveMembers, - wouldCycle, - reconcile, -} from "./core/community"; -export type { - CommunityConfig, - CommunityMode, - CommunityRow, - CommunityInviteRow, - CreateCommunityInviteInput, - AccessLevel, - AccessLevelRow, - ReservedKey, -} from "./core/community"; +// Community has moved to @atmo-dev/contrail-community. Import from there: +// import { createCommunityIntegration, CommunityAdapter, ... } from "@atmo-dev/contrail-community"; +// const app = createApp(db, config, { community: createCommunityIntegration(...) }); diff --git a/packages/contrail/tests/realtime-e2e.test.ts b/packages/contrail/tests/realtime-e2e.test.ts index 59e9338..0ef8aa2 100644 --- a/packages/contrail/tests/realtime-e2e.test.ts +++ b/packages/contrail/tests/realtime-e2e.test.ts @@ -7,6 +7,7 @@ import { createApp } from "../src/core/router"; import { resolveConfig } from "../src/core/types"; import type { ContrailConfig } from "../src/core/types"; import type { RealtimeEvent } from "../src/core/realtime/types"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; @@ -72,9 +73,11 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - await initSchema(db, resolved); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, + community, }); } diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 04f1a15..70a22e6 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -1,4 +1,5 @@ import { defineConfig } from "vitest/config"; +import path from "node:path"; export default defineConfig({ test: { @@ -6,4 +7,15 @@ export default defineConfig({ // PostgreSQL tests share a single database and cannot run in parallel fileParallelism: false, }, + resolve: { + alias: { + // Point at contrail-community's source so tests don't require a built + // dist. Mirrors the contrail-community package's own vitest alias for + // `@atmo-dev/contrail` → contrail/src. + "@atmo-dev/contrail-community": path.resolve( + __dirname, + "../contrail-community/src/index.ts" + ), + }, + }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 8e648bf..11a1eb6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -82,6 +82,9 @@ importers: '@atmo-dev/contrail': specifier: workspace:* version: link:../../packages/contrail + '@atmo-dev/contrail-community': + specifier: workspace:* + version: link:../../packages/contrail-community '@atmo-dev/contrail-lexicons': specifier: workspace:* version: link:../../packages/lexicons @@ -383,6 +386,9 @@ importers: specifier: ^2.4.0 version: 2.6.1 devDependencies: + '@atmo-dev/contrail-community': + specifier: workspace:* + version: link:../contrail-community '@cloudflare/workers-types': specifier: ^4.20250124.0 version: 4.20260424.1 @@ -408,6 +414,43 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-community: + dependencies: + '@atcute/atproto': + specifier: ^3.1.10 + version: 3.1.11 + '@atcute/cbor': + specifier: ^2.3.2 + version: 2.3.2 + '@atcute/identity': + specifier: ^1.1.4 + version: 1.1.4 + '@atcute/identity-resolver': + specifier: ^1.2.2 + version: 1.2.2(@atcute/identity@1.1.4) + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atcute/xrpc-server': + specifier: ^0.1.12 + version: 0.1.12 + '@atmo-dev/contrail': + specifier: workspace:* + version: link:../contrail + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + vitest: + specifier: ^4.1.0 + version: 4.1.5(@types/node@25.6.0)(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.6.1)(tsx@4.21.0)) + packages/lexicons: dependencies: '@atcute/lex-cli': -- 2.51.2 From 7e3145b3950f161513798054f9535e8300451e9b Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 30 Apr 2026 19:14:39 +0200 Subject: [PATCH 07/19] fix deploy, update docs, update ci --- .changeset/config.json | 2 +- .changeset/spaces-host-authority-split.md | 101 +++++++++ README.md | 4 +- docs/05-auth.md | 102 +++++++-- docs/06-spaces.md | 115 ++++++++-- docs/07-communities.md | 77 +++++-- docs/10-deployment-shapes.md | 199 ++++++++++++++++++ packages/contrail-community/package.json | 2 +- .../tests/realtime-community.test.ts | 148 +++++++++++++ packages/contrail/package.json | 1 - packages/contrail/tests/realtime-e2e.test.ts | 69 +----- packages/contrail/vitest.config.ts | 12 -- packages/lexicons/tests/generate.test.ts | 5 +- pnpm-lock.yaml | 3 - refs/spaces-later.md | 127 ++++++++--- refs/spaces-spec-mapping.md | 195 +++++++++-------- 16 files changed, 908 insertions(+), 254 deletions(-) create mode 100644 .changeset/spaces-host-authority-split.md create mode 100644 docs/10-deployment-shapes.md create mode 100644 packages/contrail-community/tests/realtime-community.test.ts diff --git a/.changeset/config.json b/.changeset/config.json index 237216e..745d87f 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -3,7 +3,7 @@ "changelog": "@changesets/cli/changelog", "commit": false, "fixed": [], - "linked": [["@atmo-dev/contrail", "@atmo-dev/contrail-sync"]], + "linked": [["@atmo-dev/contrail", "@atmo-dev/contrail-sync", "@atmo-dev/contrail-community"]], "access": "public", "baseBranch": "main", "updateInternalDependencies": "patch", diff --git a/.changeset/spaces-host-authority-split.md b/.changeset/spaces-host-authority-split.md new file mode 100644 index 0000000..e14e46c --- /dev/null +++ b/.changeset/spaces-host-authority-split.md @@ -0,0 +1,101 @@ +--- +"@atmo-dev/contrail": minor +"@atmo-dev/contrail-community": minor +"@atmo-dev/contrail-sync": minor +--- + +Spaces refactor: split authority + record host into independently runnable +roles, add space credentials, extract community into its own package. + +**Breaking — config shape** + +`spaces` is no longer flat — split into `authority` and `recordHost`: + +```ts +// before +spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, +} + +// after +spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, +} +``` + +**Breaking — community moved to its own package** + +Community has been extracted to `@atmo-dev/contrail-community`. Wire it via +`createCommunityIntegration`: + +```ts +import { Contrail, resolveConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + +const resolved = resolveConfig(config); +const communityIntegration = createCommunityIntegration({ db, config: resolved }); +const contrail = new Contrail({ ...config, communityIntegration }); +``` + +The community config (`config.community`) stays the same; only the wiring +moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, +`reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of +`@atmo-dev/contrail`. + +**New — space credentials (`X-Space-Credential`)** + +The space authority issues short-lived ES256 JWTs (default 2h TTL) via +`.space.getCredential` and `refreshCredential`. The record host accepts +them on read/write paths in lieu of per-request service-auth JWTs. Skips +DID-doc fetches and member checks; the credential's signature is the proof. + +Generate a signing key once at deploy time: + +```ts +import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; +const signing = await generateAuthoritySigningKey(); +// Store the JWK; pass to spaces.authority.signing. +``` + +**New — binding resolution** + +Verifiers can resolve "which authority signs for this space?" from three +sources, in order: local enrollment table, PDS records at +`at:////`, DID-doc `#atproto_space_authority` service +entry, owner-self fallback. Lets user-owned DIDs authorize a third-party +authority via a normal PDS write — no DID-doc surgery. + +**New — independent deployments + enrollment** + +The authority and record host can run as separate processes/operators. +A new `.recordHost.enroll` endpoint lets owners (or authorities) +register a space onto a host. In-process deployments auto-enroll on +`createSpace`; nothing changes for single-instance setups. + +See `docs/10-deployment-shapes.md` for all-in-one / authority-only / +host-only configurations and when to choose each. + +**Migration** + +For most deployments running spaces today, the migration is: + +1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. +2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. +3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + +Existing service-auth JWT clients keep working as a fallback path. +Migrate to space credentials when convenient — exchange a JWT for a +credential once via `getCredential`, then reuse it. diff --git a/README.md b/README.md index 1f88ec9..c4203e0 100644 --- a/README.md +++ b/README.md @@ -89,13 +89,15 @@ returns every `community.lexicon.calendar.event` record published anywhere on at - [Communities](https://github.com/flo-bit/contrail/blob/main/docs/07-communities.md) — group-controlled atproto DIDs - [Sync](https://github.com/flo-bit/contrail/blob/main/docs/08-sync.md) — reactive client-side store over `watchRecords` - [Labels](https://github.com/flo-bit/contrail/blob/main/docs/09-labels.md) — atproto-native moderation hydration from external labelers +- [Deployment shapes](https://github.com/flo-bit/contrail/blob/main/docs/10-deployment-shapes.md) — all-in-one vs split-authority vs split-host configurations - Frameworks: [SvelteKit + Cloudflare](https://github.com/flo-bit/contrail/blob/main/docs/frameworks/sveltekit-cloudflare.md) ## Packages | Package | | |---|---| -| `@atmo-dev/contrail` | Core library — indexing, XRPC server, spaces, communities, realtime | +| `@atmo-dev/contrail` | Core library — indexing, XRPC server, spaces, realtime | +| `@atmo-dev/contrail-community` | Community module — group-controlled DIDs, access-level ladder. Plugs into core via an integration | | `@atmo-dev/contrail-sync` | Client-side reactive watch-store with optional IndexedDB cache | | `@atmo-dev/contrail-lexicons` | Codegen + `contrail-lex` CLI | diff --git a/docs/05-auth.md b/docs/05-auth.md index ad8f659..7cb7578 100644 --- a/docs/05-auth.md +++ b/docs/05-auth.md @@ -1,15 +1,18 @@ # Auth -Contrail has four auth mechanisms. Which one applies depends on who's calling and what they're asking for. +Contrail has six auth mechanisms. Which one applies depends on who's calling, where they're calling, and what they're asking for. | Mechanism | Used by | For | |---|---|---| | Anonymous | anyone | public reads | -| Service-auth JWT | third-party apps acting on behalf of a user | anything permissioned (spaces, communities) | +| Service-auth JWT | third-party apps acting on behalf of a user | authority-side ops + record-host fallback | +| **Space credential** (`X-Space-Credential`) | callers after exchange via `getCredential` | record-host reads/writes — primary path | | In-process server client | your own server code | loaders / actions that skip HTTP entirely | | Invite token | anonymous bearers | read-only access to a specific space | | Watch ticket | browsers | realtime subscriptions (`watchRecords`) | +The two "atproto-y" mechanisms (service-auth JWTs and space credentials) work in tandem on permissioned routes: a caller exchanges a JWT for a credential once via `space.getCredential`, then presents the credential on every subsequent record-host request until it expires. + ## Service-auth JWTs The standard atproto mechanism. When a third-party app wants to call your contrail service as a user, it: @@ -20,23 +23,87 @@ The standard atproto mechanism. When a third-party app wants to call your contra Contrail verifies every request against the public key in the issuer's DID doc (`@atcute/xrpc-server` does the heavy lifting). It checks: - Signature valid -- `aud` matches the `serviceDid` you configured +- `aud` matches the `serviceDid` you configured (under `spaces.authority.serviceDid`) - `lxm` covers the method being called - Token hasn't expired On pass, your handler sees a populated `serviceAuth = { issuer, audience, lxm, clientId }` context and can proceed. On fail, 401 or 403 with a structured reason. +### Where service-auth JWTs apply + +- **Authority routes** (`.space.createSpace`, `addMember`, `getCredential`, etc.) — JWT-only. Credentials are scoped to record-host operations; you can't use one to manage spaces. +- **Record-host routes** (`putRecord`, `listRecords`, `uploadBlob`, etc.) — accept JWTs as a fallback path. The credential path (below) is preferred. + ### The `serviceDid` gotcha Use the **plain DID** (no `#fragment`) when configuring contrail: ```ts -spaces: { serviceDid: "did:web:example.com" } // right -spaces: { serviceDid: "did:web:example.com#com_example_space" } // wrong +spaces: { authority: { serviceDid: "did:web:example.com" } } // right +spaces: { authority: { serviceDid: "did:web:example.com#com_example_x" } } // wrong ``` Many PDS implementations reject `aud` values containing `#fragment` in `com.atproto.server.getServiceAuth`, and contrail does strict string equality on `aud`. The fragment form belongs only in your DID doc's `service` entry, where PDSes use it to resolve the service endpoint URL for `Atproto-Proxy` routing — that's separate from JWT audience validation. +## Space credentials + +Short-lived (default 2h) ES256 JWTs minted by the space authority. Once a caller has one, they present it via `X-Space-Credential: ` on every record-host request and skip the per-request JWT mint dance. This matches the rough atproto permissioned-data spec. + +### Lifecycle + +``` +1. Caller mints a service-auth JWT { aud, lxm: ".space.getCredential" }. +2. POST .space.getCredential { spaceUri } Authorization: Bearer + → { credential: "", expiresAt: } +3. Caller stores the credential. For ~2 hours, every record-host request: + X-Space-Credential: + succeeds without going back through the user's PDS. +4. Before expiry, refresh: + POST .space.refreshCredential { credential } + → { credential: , expiresAt: } +``` + +### Claims + +```json +{ + "iss": "", + "sub": "", + "space": "ats:////", + "scope": "rw", + "iat": 1746000000, + "exp": 1746014400 +} +``` + +- Signed with the authority's ES256 key (kid = `#atproto_space_authority`). +- Stateless — verifiable by anyone who can resolve the authority DID's verification key. +- `scope` is `"rw"` or `"read"`. Today only `rw` is issued via `getCredential`; the read-only path is a future read-grant invite replacement. + +### How verification works + +When the record host receives a credential, it: + +1. Decodes the JWT, reads `iss` and `space`. +2. Asks its **binding resolver** "who's authorized to sign for this space?" — primary source is the local enrollment table; fallbacks include PDS records and DID-doc service entries. +3. Confirms `iss` matches the authorized DID. +4. Resolves the issuer's verification key (local in-process, or via DID doc). +5. Verifies signature, expiry, scope, space match. + +In an in-process deployment, this is one DB lookup + one signature verification. No DID-doc fetches per request. See [Spaces](./06-spaces.md#discovery--binding-resolution) for the binding details. + +### When the credential gets rejected + +| Reason | Response | +|---|---| +| `malformed` | 401 — JWT structure invalid | +| `bad-alg` | 401 — header alg ≠ ES256 | +| `bad-signature` | 401 — signature didn't verify against the resolved key | +| `expired` | 401 — past `exp`. Refresh, or re-mint. | +| `wrong-space` | 403 — credential's `space` ≠ request's space | +| `wrong-scope` | 403 — read-only credential on a write | +| `unknown-issuer` | 401 — `iss` doesn't match the binding for the space (most often: not enrolled here) | + ## In-process server client When your own server code wants to call contrail, the service-auth dance is pointless — it's your code talking to your code. `createServerClient` skips it: @@ -50,7 +117,7 @@ const client = createServerClient(async (req) => handle(req, env.DB), userDid); const res = await client.get("com.example.event.listRecords", { params: {...} }); ``` -Pass `did` to act as that user; omit it for anonymous calls against public endpoints. This is a trust boundary — anything that actually crosses a network needs a real service-auth JWT, not this shortcut. +Pass `did` to act as that user; omit it for anonymous calls against public endpoints. This is a trust boundary — anything that actually crosses a network needs a real service-auth JWT or space credential, not this shortcut. See [SvelteKit + Cloudflare](./frameworks/sveltekit-cloudflare.md) for the typical loader pattern. @@ -59,7 +126,7 @@ See [SvelteKit + Cloudflare](./frameworks/sveltekit-cloudflare.md) for the typic First-class auth for spaces. When a space owner creates an invite: ``` -com.example.space.invite.create { spaceUri, ttl?, maxUses? } +.invite.create { spaceUri, kind, ttl?, maxUses? } → { token: "...plaintext..." } // returned once, never again ``` @@ -67,7 +134,7 @@ The plaintext token is handed to the user out-of-band (link, QR, email). Contrai Three invite kinds, depending on what the token does: -- **`join`** — redeemed via `com.example.space.invite.redeem` with a service-auth JWT. Adds the caller's DID to the member list. Members have full read + write inside the space; there's no per-member permission axis beyond "is a member." +- **`join`** — redeemed via `.invite.redeem` with a service-auth JWT. Adds the caller's DID to the member list. Members have full read + write inside the space; there's no per-member permission axis beyond "is a member." - **`read`** — bearer-only. The token itself grants read access when passed as `?inviteToken=`, no DID, no redemption. Good for sharing a read-only link that doesn't add anyone to the member list. - **`read-join`** — both. Works anonymously as a read token; can also be redeemed with a JWT to promote the caller to member. @@ -79,7 +146,7 @@ Realtime subscriptions (`watchRecords`) can't use regular service-auth JWTs for Server-side minting comes in two flavours: -- `com.example.realtime.ticket` — POST `{ topic }` (e.g. `"space:ats://..."`) → `{ ticket, topics, expiresAt }`. Bare topic-list ticket, used with the generic `<ns>.realtime.subscribe` endpoint. +- `<ns>.realtime.ticket` — POST `{ topic }` (e.g. `"space:ats://..."`) → `{ ticket, topics, expiresAt }`. Bare topic-list ticket, used with the generic `<ns>.realtime.subscribe` endpoint. - `<collection>.watchRecords?mode=ws&spaceUri=…` (or `&actor=…`) handshake — returns `{ snapshot, ticket, wsUrl, sinceTs, ticketTtlMs, querySpec }`. The ticket is bound to `(did, topics, querySpec)` and is the one to use for the per-collection `watchRecords` stream — both for SSE (`?ticket=…`) and the subsequent WS upgrade. Both flavours are signed by `realtime.ticketSecret` (a 32-byte random, configured once). Clients hand the ticket off via `?ticket=...` on connect. @@ -129,9 +196,14 @@ A typical flow for a third-party app acting as a user in a space: 1. App registers OAuth client pointing at your permission set NSID. 2. User grants consent — PDS fetches your permission set lexicon via DNS, shows the user the methods, records the scope. -3. App calls `com.atproto.server.getServiceAuth` on the user's PDS: `{ aud: "did:web:example.com", lxm: "com.example.space.putRecord", exp: <60s> }`. PDS signs, returns JWT. -4. App sends `PUT /xrpc/com.example.space.putRecord` with `Authorization: Bearer <jwt>` and `Atproto-Proxy: did:web:example.com#com_example_space`. -5. User's PDS reads the `Atproto-Proxy` header, resolves the service endpoint from your DID doc, forwards the request. -6. Contrail verifies the JWT (signature, `aud`, `lxm`, expiry), runs ACL check (is this DID a member of that space?), dispatches the write. - -For your own loaders/actions, steps 3–6 collapse into a single `createServerClient({did}).post(...)` call. For a browser subscribing to a feed, steps 3–5 are replaced by a ticket mint from your server. Same auth model, different surface. +3. App calls `com.atproto.server.getServiceAuth` on the user's PDS: `{ aud: "did:web:example.com", lxm: "com.example.space.getCredential", exp: <60s> }`. PDS signs, returns JWT. +4. App POSTs `<ns>.space.getCredential { spaceUri }` with `Authorization: Bearer <jwt>` and `Atproto-Proxy: did:web:example.com#com_example_space`. Authority verifies the JWT, checks membership, mints a 2h credential. +5. App caches the credential. Every subsequent `putRecord` / `listRecords` / `uploadBlob`: + ``` + POST <ns>.space.putRecord + X-Space-Credential: <credential> + ``` + The record host verifies the credential against its enrolled authority's key — no PDS roundtrip, no DID-doc fetch. +6. Before expiry, app calls `refreshCredential` to get a fresh one. + +For your own loaders/actions, the credential dance disappears — `createServerClient({did}).post(...)` bypasses it entirely. For a browser subscribing to a feed, steps 3–5 are replaced by a ticket mint from your server. Same auth model, different surface. diff --git a/docs/06-spaces.md b/docs/06-spaces.md index 8bb9fb8..338abc7 100644 --- a/docs/06-spaces.md +++ b/docs/06-spaces.md @@ -12,17 +12,40 @@ Auth-gated store for records that can't live on public PDSes — private events, Every permission boundary is its own space. No nested ACLs. Richer roles = more spaces or app-layer checks. +### Two roles, one or two services + +A space has two operational roles: + +- **Space authority** — owns the member list, signs short-lived credentials. Identified by a service DID. +- **Record host** — stores records and blobs for spaces it has *enrolled*. + +In the default deployment both run in the same Contrail instance against the same DB; you don't notice the split. But the roles can also run separately — see [deployment shapes](./10-deployment-shapes.md) for ACL-on-arbiter / records-on-contrail patterns. + ## Enable ```ts import type { ContrailConfig } from "@atmo-dev/contrail"; +import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + +// One-time setup: generate a signing key and store it. The authority signs +// space credentials with this key; verifiers find the public key in the +// authority DID's DID document or via the binding-resolver chain. +const signing = await generateAuthoritySigningKey(); const config: ContrailConfig = { namespace: "com.example", collections: { /* ... */ }, spaces: { - type: "com.example.event.space", - serviceDid: "did:web:example.com", + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing, // omit to disable credential issuance + credentialTtlMs: 2 * 60 * 60 * 1000, // 2h, matches the rough spec + }, + recordHost: { + // blobs is optional; omit to disable blob endpoints + blobs: { adapter: blobsAdapter }, + }, }, }; ``` @@ -33,42 +56,98 @@ Each collection gets a parallel `spaces_records_<short>` table. Opt out per-coll public_only: { collection: "com.example.public", allowInSpaces: false } ``` -## Auth +## Auth — three paths + +The record host accepts three forms of auth on read/write paths, in this precedence order: -Spaces use the standard contrail auth surface — service-auth JWTs for third-party apps, in-process server clients for your own loaders, invite tokens for anonymous read-grant links. See [Auth](./05-auth.md) for the full picture. +1. **`X-Space-Credential` header** — a short-lived JWT minted by the space authority. The primary path: callers exchange a service-auth JWT once via `space.getCredential`, then present the credential on every request until it expires. Skips per-request DID-doc fetches and member checks; the credential's signature is the proof. +2. **`?inviteToken=...` query** (read-only) — bearer access for shareable links. See [Auth § Invite tokens](./05-auth.md#invite-tokens). +3. **`Authorization: Bearer <service-auth-jwt>`** — the standard atproto path. Useful for one-off calls (the credential exchange itself, space-management endpoints) or as a fallback when the caller doesn't want to manage credentials. + +Authority-side endpoints (`createSpace`, `addMember`, `getCredential`, etc.) only accept service-auth JWTs — credentials are scoped to record-host operations. + +See [Auth](./05-auth.md) for the full picture. + +## Credential flow + +```text + ┌──────────────┐ + │ user PDS │ mints service-auth JWT (lxm: getCredential) + └──────┬───────┘ + ▼ + ┌──────────────┐ + │ authority │ validates JWT, checks membership, + │ (Contrail) │ signs ES256 credential (2h TTL) + └──────┬───────┘ + │ { credential, expiresAt } + ▼ + ┌──────────────┐ + │ record host │ verifies credential signature against + │ (Contrail or │ authority DID's published key, + │ elsewhere) │ checks scope/space/expiry, serves request + └──────────────┘ +``` -Space-specific wiring: +`space.refreshCredential` re-issues a fresh credential from an unexpired one without going back through the JWT mint dance — useful for long-running clients. -- `serviceDid` in the config is the `aud` contrail expects on incoming JWTs. Plain DID, no `#fragment`. -- Apps acting in a space send `Atproto-Proxy: <serviceDid>#<service-id-from-your-did-doc>` so the user's PDS routes correctly. -- Invite redemption via service-auth JWT grants membership; via `?inviteToken=...` query param grants read-only bearer access to that space. +## Enrollment + +The record host maintains a local table of which spaces it accepts records for and which authority signs credentials for each. Two ways enrollment happens: + +- **Auto-enroll** (default for in-process deployments): the authority's `createSpace` automatically enrolls the new space on the colocated record host. New users see no enrollment surface; it just works. +- **Explicit `recordHost.enroll`**: for split deployments where the authority and record host run in different processes/operators, the owner (or the authority itself) calls `<ns>.recordHost.enroll { spaceUri, authority }` to consent. Idempotent — re-enrolling updates the binding. + +A non-enrolled space gets 404 "not-enrolled" on every record-host route. This is the host's consent layer — without it, anyone with a valid credential could create unbounded storage on your host. + +## Discovery — binding resolution + +When a record host receives a credential, it needs to know whether the credential's `iss` is authorized to sign for that space. Three sources, tried in order: + +1. **Local enrollment** — primary on the record host. `(spaceUri → authorityDid)` from the enrollment table. +2. **PDS record** at `at://<owner>/<type>/<key>` — for user-owned DIDs that declared a host via a normal PDS write. Lexicon: `tools.atmo.space.declaration` (or your namespaced variant). +3. **DID-doc service entry** — `#atproto_space_authority` on the owner's DID doc. For provisioned (no-PDS) DIDs. +4. **Owner self-issues** (fallback) — for the trivial case where the owner DID's own key signs credentials. + +For in-process deployments, step 1 is the only one that fires. The other resolvers are wired in by deployments that accept credentials from external authorities — see [deployment shapes](./10-deployment-shapes.md). ## Unified `listRecords` | Call | Returns | |---|---| | no auth, no `spaceUri` | public only | -| `?spaceUri=…` + JWT | one space (ACL-gated) | -| JWT, no `spaceUri` | public **unioned** with every space the caller is a member of | +| `?spaceUri=…` + credential or JWT | one space (ACL-gated) | +| credential / JWT, no `spaceUri` | public **unioned** with every space the caller is a member of | Filters, sorts, hydration, and references work across all three. Records from a space carry a `space: <spaceUri>` field — same on `listRecords`/`getRecord` responses and `watchRecords` stream events. ## Invites -First-class primitive — see [Auth § Invite tokens](./05-auth.md#invite-tokens) for the mechanism. Space-specific: create via `com.example.space.invite.create`, redeem via `.redeem` (membership grant) or `?inviteToken=...` query param (read-only bearer grant). +First-class primitive — see [Auth § Invite tokens](./05-auth.md#invite-tokens) for the mechanism. Space-specific: create via `<ns>.invite.create`, redeem via `.redeem` (membership grant) or `?inviteToken=...` query param (read-only bearer grant). ## XRPCs -- `com.example.space.create | get | list | delete` -- `com.example.space.putRecord | deleteRecord | listRecords | getRecord` -- `com.example.space.invite.create | redeem | revoke | list` -- `com.example.space.listMembers | removeMember` +### Authority routes (`<ns>.space.*` — spec-aligned) +- `createSpace` `getSpace` `listSpaces` `deleteSpace` +- `listMembers` `addMember` `removeMember` +- `getCredential` `refreshCredential` +- `leaveSpace` (contrail extra) + +### Record-host routes (`<ns>.space.*` for records, `<ns>.recordHost.*` for management) +- `putRecord` `deleteRecord` `getRecord` `listRecords` +- `uploadBlob` `getBlob` `listBlobs` (when `recordHost.blobs` is configured) +- `recordHost.enroll` + +### Contrail extras (`<ns>.spaceExt.*`) +- `whoami` — caller's relationship to a space (extensions plug in via the integration's whoami hook) + +### Invite (`<ns>.invite.*`) +- `create` `redeem` `revoke` `list` ## What's not here - No E2EE (data is operator-readable). - No FTS on `?spaceUri=…` yet. -- No per-space sharding — one DB, one operator. -- Not a long-term replacement for real atproto permissioned repos. +- Records still live in the operator's DB rather than user PDSes — federation is greenfield. (See `refs/spaces-spec-mapping.md` for the migration notes.) +- No managing-app routing (join requests, approval queues — see `refs/spaces-later.md`). -The design follows Daniel Holmgren's [permissioned data rough spec](https://dholms.leaflet.pub/3mhj6bcqats2o). The goal is that when real atproto permissioned repos ship, migration is mostly data movement — the API your app speaks doesn't change. +The design follows Daniel Holmgren's [permissioned data rough spec](https://dholms.leaflet.pub/3mhj6bcqats2o). When real atproto permissioned repos ship, migration is mostly data movement — the wire surface your app speaks doesn't change. diff --git a/docs/07-communities.md b/docs/07-communities.md index a7f2e26..0268c42 100644 --- a/docs/07-communities.md +++ b/docs/07-communities.md @@ -2,6 +2,56 @@ Group-controlled atproto DIDs. A community is a DID whose signing/rotation keys are held by the appview on behalf of multiple members, with tiered access levels. Built on top of [spaces](./06-spaces.md). +Communities live in a separate package — `@atmo-dev/contrail-community` — that plugs into Contrail via an integration object. The contrail core has no knowledge of community-specific concepts; the package wires itself in via injectable hooks (whoami extension, invite handler, route registration, schema). + +## Install + +```bash +pnpm add @atmo-dev/contrail @atmo-dev/contrail-community +``` + +## Wire it up + +Construct the integration once, hand it to `Contrail` (or directly to `createApp`): + +```ts +import { Contrail, resolveConfig, type ContrailConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + +const config: ContrailConfig = { + namespace: "com.example", + collections: { /* ... */ }, + spaces: { + authority: { type: "com.example.event.space", serviceDid: "did:web:example.com", signing }, + recordHost: {}, + }, + community: { + masterKey: env.COMMUNITY_MASTER_KEY, // 32-byte encryption key for stored credentials + serviceDid: "did:web:example.com", + levels: ["admin", "moderator"], // ranked, highest-first + }, +}; + +const resolved = resolveConfig(config); +const communityIntegration = createCommunityIntegration({ db, config: resolved }); + +const contrail = new Contrail({ ...config, db, communityIntegration }); +await contrail.init(); // applies community schema alongside contrail's own +``` + +Or with `createApp` directly: + +```ts +import { createApp, initSchema } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + +const community = createCommunityIntegration({ db, config }); +await initSchema(db, config, { extraSchemas: [community.applySchema] }); +const app = createApp(db, config, { community }); +``` + +Stored credentials (app passwords for adopted communities, signing keys for minted) are envelope-encrypted with `masterKey`. Never ship the placeholder. + ## When to use this When you want atproto records published under a *shared* identity — a team, a project, a channel — not a single user. Think: a group's published calendar events, a community's published posts. @@ -15,17 +65,7 @@ Either way, the result is the same: a DID that multiple members can act through, ## Access levels -Each member has a level (ranked). Levels map to write permissions. Owners can grant/revoke levels. Two reserved levels exist: `owner` and `member`. Your deployment defines the rest: - -```ts -community: { - masterKey: ENV.COMMUNITY_MASTER_KEY, // 32-byte encryption key for stored credentials - serviceDid: "did:web:example.com", - levels: ["admin", "moderator"], // ranked, highest-first -} -``` - -Stored credentials (app passwords for adopted communities, signing keys for minted) are envelope-encrypted with `masterKey`. Never ship the placeholder. +Each member has a level (ranked). Levels map to write permissions. Owners can grant/revoke levels. Two reserved levels exist: `owner` and `member`. Your deployment defines the rest via `config.community.levels`. ## How it composes with spaces @@ -37,13 +77,18 @@ community.space.grant { spaceUri, subject: { did: "did:plc:..." }, accessLevel: The spaces layer stays ignorant of access levels — it just sees "this DID is a member." The community layer projects member × level → membership in specific spaces. Once a DID is a member of a space (through a community grant or otherwise), they have full read + write inside it. +The integration plugs in to two contrail extension points: + +- **Whoami** — `<ns>.spaceExt.whoami` returns `accessLevel` for community-owned spaces (the community whoami extension overrides the default binary-membership response). +- **Invites** — the unified `<ns>.invite.*` family dispatches community-owned spaces through the community invite handler (which uses access levels) and user-owned spaces through the spaces module's binary-membership handler. + ## XRPCs -- `com.example.community.mint | adopt | list | delete` -- `com.example.community.invite.create | redeem | revoke | list` -- `com.example.community.setAccessLevel | revoke | listMembers` -- `com.example.community.space.create | grant | revoke | ...` — community-owned spaces -- `com.example.community.putRecord | deleteRecord` — publish records as the community DID +- `<ns>.community.mint | adopt | list | delete` +- `<ns>.community.invite.create | redeem | revoke | list` +- `<ns>.community.setAccessLevel | revoke | listMembers` +- `<ns>.community.space.create | grant | revoke | ...` — community-owned spaces +- `<ns>.community.putRecord | deleteRecord` — publish records as the community DID ## What's not here diff --git a/docs/10-deployment-shapes.md b/docs/10-deployment-shapes.md new file mode 100644 index 0000000..25464ed --- /dev/null +++ b/docs/10-deployment-shapes.md @@ -0,0 +1,199 @@ +# Deployment shapes + +Spaces split into two roles, run together by default. Three deployment shapes, in increasing order of complexity: + +1. **All-in-one** — authority + record host + (optional) community in one process. The default; what you get from `createApp` with both `spaces.authority` and `spaces.recordHost` configured. Most apps want this. +2. **Authority-only** — a service that controls ACL and signs credentials, but doesn't store records. Useful when records live on someone else's host (e.g. a community arbiter that delegates storage to a heavier appview). +3. **Record-host-only** — a service that stores records, accepting credentials signed by an external authority. Useful when storage lives separately from governance — e.g. Contrail-as-host backing spaces that an "Arbiter" or HappyView manages. + +The split is real at the wire level (different XRPCs, different auth shapes) but the same Contrail codebase handles all three. This doc walks through each. + +## Shape 1: all-in-one (default) + +``` +┌────────────────────────────────────┐ +│ Contrail │ +│ ┌────────┐ ┌────────────┐ │ +│ │authority│ │ record host│ │ +│ │ + signing│ │ + enrollment│ │ +│ └────────┘ └────────────┘ │ +│ shared DB, single process │ +└────────────────────────────────────┘ +``` + +Config: + +```ts +spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter: blobsAdapter }, // optional + }, +}, +``` + +What happens at startup: + +- `initSchema` creates both authority tables (`spaces`, `spaces_members`, `spaces_invites`) and record-host tables (`spaces_records_<short>`, `spaces_blobs`, `record_host_enrollments`). +- The umbrella router wires `registerAuthorityRoutes` + `registerRecordHostRoutes` against the same `HostedAdapter`. +- The credential verifier is built from `Local` binding + `Local` key — no DID-doc fetches; the host knows the authority's public key directly. + +What happens when a user creates a space: + +1. `createSpace` writes a row in `spaces` (authority) and immediately a row in `record_host_enrollments` (host). One round-trip, two DB writes. +2. From there, `getCredential` works, `putRecord` works, the world is in sync. + +This is the path most apps run. You don't notice the role split. + +## Shape 2: authority-only + +A lightweight service that holds ACL and signs credentials. Records live on someone else's host. + +``` + ┌────────────────┐ + │ this Contrail │ + │ authority │ + └────────────────┘ + ▲ + │ getCredential + │ + ┌─────────────┐ + │ client │ + └─────────────┘ + │ X-Space-Credential + ▼ + ┌─────────────┐ + │ external │ enrolled with this authority + │ record host │ (different operator, different DID) + └─────────────┘ +``` + +Config: + +```ts +spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:authority.example.com", + signing: await generateAuthoritySigningKey(), + }, + // recordHost omitted — this deployment doesn't store records +}, +``` + +`createSpace` here does NOT auto-enroll anywhere. The space owner (or the authority itself) calls `recordHost.enroll` on whichever host they want to use; the host then accepts credentials signed by this authority for that space. + +The authority's DID document needs to publish the verification key under `#atproto_space_authority` so external hosts can resolve it. + +## Shape 3: record-host-only + +A storage tier that accepts credentials signed by external authorities. + +``` +┌────────────┐ +│ external │ signs credentials +│ authority │ +└────────────┘ + │ + ▼ +┌─────────────────────────────────────┐ +│ this Contrail │ +│ record host (no authority) │ +│ │ +│ verifies credentials via: │ +│ - enrollment table │ +│ - DID-doc key resolver │ +│ (for external authorities) │ +└─────────────────────────────────────┘ +``` + +Config: + +```ts +spaces: { + // authority is still needed for the JWT verifier infrastructure (so the + // record host can validate JWTs on the recordHost.enroll endpoint), but + // no signing key is configured — this deployment doesn't issue creds. + authority: { + type: "com.example.event.space", + serviceDid: "did:web:host.example.com", + }, + recordHost: { + blobs: { adapter: blobsAdapter }, + }, +}, +``` + +To accept credentials from an external authority, wire a custom verifier: + +```ts +import { + createApp, + createBindingCredentialVerifier, + createEnrollmentBindingResolver, + createDidDocKeyResolver, +} from "@atmo-dev/contrail"; +import { CompositeDidDocumentResolver, PlcDidDocumentResolver, WebDidDocumentResolver } + from "@atcute/identity-resolver"; + +const didResolver = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, +}); + +const verifier = createBindingCredentialVerifier({ + // Local enrollment is the canonical binding source — only spaces this + // host has explicitly opted into are accepted. + bindings: createEnrollmentBindingResolver({ recordHost: hostAdapter }), + // For credential signature verification, walk DID docs of external + // authorities to find their published verification keys. + keys: createDidDocKeyResolver({ resolver: didResolver }), +}); + +const app = createApp(db, config, { + spaces: { credentialVerifier: verifier }, +}); +``` + +The flow when a request arrives: + +1. Caller presents `X-Space-Credential: <jwt>`. +2. Verifier reads `iss` from the JWT, looks up enrollment for `claims.space`. If the enrollment's `authorityDid` matches `iss` → continue. If not → 401 `unknown-issuer`. +3. Resolves the issuer DID, finds the verification method with id matching the JWT's `kid`, verifies the signature. +4. Checks expiry, scope, space match. +5. Serves the request. + +Enrollment is the host's consent layer: a credential can only be presented for spaces the host has agreed to store. Without enrollment, no records get written. + +## Mixing shapes + +You can run all three simultaneously in one Contrail instance. The umbrella router enables each set of routes based on what's configured: + +- `spaces.authority` → authority routes registered (`createSpace`, `getCredential`, etc.) +- `spaces.recordHost` → record-host routes registered (`putRecord`, `recordHost.enroll`, etc.) +- Both → today's default. + +A deployment can act as the authority for spaces it owns *and* a record host for spaces other authorities own. Auto-enroll fires only for spaces this deployment is the authority for; external authorities still enroll explicitly. + +## Choosing a shape + +| Need | Shape | +|---|---| +| One operator, one process, want it to work | All-in-one | +| You're running a "DAO governance / arbiter" service that decides ACL but not storage | Authority-only | +| You're running an appview / heavier storage tier and want to accept ACL decisions from external services | Record-host-only | +| You're an existing Contrail deployment that wants to also accept external authorities | All-in-one + custom verifier | + +When in doubt, all-in-one. Splitting is for when you have a real operational reason to separate the two — different teams running them, different latency profiles, different scaling targets, different governance. + +## What's not here + +- **Authority migration** — moving a space's authority from DID A to DID B. The architecture supports it (re-enroll on the host with the new authority binding) but no helper API yet. +- **Multi-authority per space** — could in principle allow several authorities to all sign for one space (replication scenarios). Not modeled today; spec is silent. +- **PDS-backed records** — when atproto's permissioned-repos protocol ships, records will federate from user PDSes. The host becomes an aggregator rather than a store. The role split here generalizes to that world without changes. diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index 2fae478..43fe2e1 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-community", - "version": "0.1.0", + "version": "0.4.2", "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-community/tests/realtime-community.test.ts b/packages/contrail-community/tests/realtime-community.test.ts new file mode 100644 index 0000000..c6c6b7a --- /dev/null +++ b/packages/contrail-community/tests/realtime-community.test.ts @@ -0,0 +1,148 @@ +/** Realtime + community integration test. Lives here (not in contrail) so + * contrail's package.json doesn't have to dev-depend on contrail-community + * (which would create a build-graph cycle in turbo). + * + * Tests that `community:<did>` topics expand to the caller's reachable + * community spaces — the cross-cutting concern that needs both modules. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "@atmo-dev/contrail/sqlite"; +import { + createApp, + initSchema, + resolveConfig, + type ContrailConfig, +} from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "../src/integration"; + +const ALICE = "did:plc:alice"; +const CHARLIE = "did:plc:charlie"; + +const MASTER_KEY = new Uint8Array(32).fill(5); +const REALTIME_SECRET = new Uint8Array(32).fill(9); + +const CONFIG: ContrailConfig = { + namespace: "test.rt", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: "did:web:test.example#svc", + }, + recordHost: {}, + }, + community: { + masterKey: MASTER_KEY, + plcDirectory: "https://plc.test", + fetch: mockFetch, + resolver: mockResolver(), + }, + realtime: { + ticketSecret: REALTIME_SECRET, + keepaliveMs: 60_000, + }, +}; + +function mockResolver(): any { + return { + resolve: async (_did: string) => ({ + id: _did, + service: [ + { id: "#atproto_pds", type: "AtprotoPersonalDataServer", serviceEndpoint: "https://pds.test" }, + ], + }), + }; +} + +async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> { + const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; + if (url.endsWith("/xrpc/com.atproto.server.createSession")) { + return new Response( + JSON.stringify({ accessJwt: "a.b.c", refreshJwt: "r.r.r", did: "did:plc:community" }), + { status: 200, headers: { "content-type": "application/json" } } + ); + } + if (url.startsWith("https://plc.test/")) return new Response("{}", { status: 200 }); + return new Response("not found", { status: 404 }); +} + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { issuer: did, audience: CONFIG.spaces!.authority!.serviceDid, lxm: undefined }); + await next(); + }; +} + +async function makeApp(): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const resolved = resolveConfig(CONFIG); + const community = createCommunityIntegration({ db, config: resolved }); + await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + return createApp(db, resolved, { + spaces: { authMiddleware: fakeAuth() }, + community, + }); +} + +function call( + app: Hono, + method: string, + path: string, + did: string | null, + body?: any +): Promise<Response> { + const headers: Record<string, string> = {}; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +describe("realtime + community", () => { + let app: Hono; + beforeAll(async () => { + app = await makeApp(); + }); + + it("community:<did> alias expands to reachable spaces", async () => { + // Adopt a community, create a child space, grant Charlie member. + const adoptRes = await call(app, "POST", "/xrpc/test.rt.community.adopt", ALICE, { + identifier: "did:plc:community", + appPassword: "anything", // mockFetch returns 200 for createSession + }); + expect(adoptRes.status).toBe(200); + const { communityDid } = (await adoptRes.json()) as any; + + const c1 = await call(app, "POST", "/xrpc/test.rt.community.space.create", ALICE, { + communityDid, + key: "general", + }); + expect(c1.status).toBe(200); + const general = ((await c1.json()) as any).space.uri as string; + + // Grant Charlie as member in general. + const g = await call(app, "POST", "/xrpc/test.rt.community.space.grant", ALICE, { + spaceUri: general, + subject: { did: CHARLIE }, + accessLevel: "member", + }); + expect(g.status).toBe(200); + + // Charlie mints a community-alias ticket; should expand to [space:general]. + const ticketRes = await call(app, "POST", "/xrpc/test.rt.realtime.ticket", CHARLIE, { + topic: `community:${communityDid}`, + }); + expect(ticketRes.status).toBe(200); + const body = (await ticketRes.json()) as any; + expect(body.topics).toContain(`space:${general}`); + }); +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 846e6e8..a64484d 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -78,7 +78,6 @@ "jiti": "^2.4.0" }, "devDependencies": { - "@atmo-dev/contrail-community": "workspace:*", "@cloudflare/workers-types": "^4.20250124.0", "@types/node": "^25.5.0", "@types/pg": "^8.20.0", diff --git a/packages/contrail/tests/realtime-e2e.test.ts b/packages/contrail/tests/realtime-e2e.test.ts index 0ef8aa2..01d1767 100644 --- a/packages/contrail/tests/realtime-e2e.test.ts +++ b/packages/contrail/tests/realtime-e2e.test.ts @@ -7,13 +7,10 @@ import { createApp } from "../src/core/router"; import { resolveConfig } from "../src/core/types"; import type { ContrailConfig } from "../src/core/types"; import type { RealtimeEvent } from "../src/core/realtime/types"; -import { createCommunityIntegration } from "@atmo-dev/contrail-community"; const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; -const CHARLIE = "did:plc:charlie"; -const MASTER_KEY = new Uint8Array(32).fill(5); const REALTIME_SECRET = new Uint8Array(32).fill(9); const CONFIG: ContrailConfig = { @@ -26,41 +23,12 @@ const CONFIG: ContrailConfig = { }, recordHost: {}, }, - community: { - masterKey: MASTER_KEY, - plcDirectory: "https://plc.test", - fetch: mockFetch, - resolver: mockResolver(), - }, realtime: { ticketSecret: REALTIME_SECRET, keepaliveMs: 60_000, }, }; -function mockResolver(): any { - return { - resolve: async (_did: string) => ({ - id: _did, - service: [ - { id: "#atproto_pds", type: "AtprotoPersonalDataServer", serviceEndpoint: "https://pds.test" }, - ], - }), - }; -} - -async function mockFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> { - const url = typeof input === "string" ? input : input instanceof URL ? input.href : input.url; - if (url.endsWith("/xrpc/com.atproto.server.createSession")) { - return new Response( - JSON.stringify({ accessJwt: "a.b.c", refreshJwt: "r.r.r", did: "did:plc:community" }), - { status: 200, headers: { "content-type": "application/json" } } - ); - } - if (url.startsWith("https://plc.test/")) return new Response("{}", { status: 200 }); - return new Response("not found", { status: 404 }); -} - function fakeAuth(): MiddlewareHandler { return async (c, next) => { const did = c.req.header("X-Test-Did"); @@ -73,11 +41,9 @@ function fakeAuth(): MiddlewareHandler { async function makeApp(): Promise<Hono> { const db = createSqliteDatabase(":memory:"); const resolved = resolveConfig(CONFIG); - const community = createCommunityIntegration({ db, config: resolved }); - await initSchema(db, resolved, { extraSchemas: [community.applySchema] }); + await initSchema(db, resolved); return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() }, - community, }); } @@ -280,37 +246,4 @@ describe("realtime e2e (in-memory pubsub, SSE transport)", () => { ); expect(res.status).toBe(401); }); - - it("community:<did> alias expands to reachable spaces", async () => { - // Adopt a community, create two child spaces, grant Charlie member on one. - const adoptRes = await call(app, "POST", "/xrpc/test.rt.community.adopt", ALICE, { - identifier: "did:plc:community", - appPassword: "anything", // mockFetch returns 200 for createSession - }); - expect(adoptRes.status).toBe(200); - const { communityDid } = (await adoptRes.json()) as any; - - const c1 = await call(app, "POST", "/xrpc/test.rt.community.space.create", ALICE, { - communityDid, - key: "general", - }); - expect(c1.status).toBe(200); - const general = ((await c1.json()) as any).space.uri as string; - - // Grant Charlie as member in general. - const g = await call(app, "POST", "/xrpc/test.rt.community.space.grant", ALICE, { - spaceUri: general, - subject: { did: CHARLIE }, - accessLevel: "member", - }); - expect(g.status).toBe(200); - - // Charlie mints a community-alias ticket; should expand to [space:general]. - const ticketRes = await call(app, "POST", "/xrpc/test.rt.realtime.ticket", CHARLIE, { - topic: `community:${communityDid}`, - }); - expect(ticketRes.status).toBe(200); - const body = (await ticketRes.json()) as any; - expect(body.topics).toContain(`space:${general}`); - }); }); diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 70a22e6..04f1a15 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -1,5 +1,4 @@ import { defineConfig } from "vitest/config"; -import path from "node:path"; export default defineConfig({ test: { @@ -7,15 +6,4 @@ export default defineConfig({ // PostgreSQL tests share a single database and cannot run in parallel fileParallelism: false, }, - resolve: { - alias: { - // Point at contrail-community's source so tests don't require a built - // dist. Mirrors the contrail-community package's own vitest alias for - // `@atmo-dev/contrail` → contrail/src. - "@atmo-dev/contrail-community": path.resolve( - __dirname, - "../contrail-community/src/index.ts" - ), - }, - }, }); diff --git a/packages/lexicons/tests/generate.test.ts b/packages/lexicons/tests/generate.test.ts index 44f45c5..eaf57bc 100644 --- a/packages/lexicons/tests/generate.test.ts +++ b/packages/lexicons/tests/generate.test.ts @@ -273,7 +273,10 @@ describe("extractXrpcMethods / listXrpcMethods", () => { const config: ContrailConfig = { namespace: "test.comm", collections: { message: { collection: "app.event.message" } }, - spaces: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: "did:web:test.example#svc" }, + recordHost: {}, + }, community: { masterKey: new Uint8Array(32).fill(1) }, realtime: { ticketSecret: new Uint8Array(32).fill(2) }, }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 11a1eb6..48ffc2e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -386,9 +386,6 @@ importers: specifier: ^2.4.0 version: 2.6.1 devDependencies: - '@atmo-dev/contrail-community': - specifier: workspace:* - version: link:../contrail-community '@cloudflare/workers-types': specifier: ^4.20250124.0 version: 4.20260424.1 diff --git a/refs/spaces-later.md b/refs/spaces-later.md index 270b55c..47a0714 100644 --- a/refs/spaces-later.md +++ b/refs/spaces-later.md @@ -4,26 +4,65 @@ Deferred items from the spaces design review. Not blocking shipping; keep an eye on these as usage grows or as the permissioned-data spec firms up. See also [spaces-spec-mapping.md](./spaces-spec-mapping.md). +Items resolved by the six-phase refactor (credential flow, host/authority +split, enrollment, community as separate package, etc.) have been removed +from this list — see the spec-mapping doc for the post-refactor state. + ## Hydrated members endpoint -`space.listMembers` today returns raw `{did, perms, addedAt, addedBy}` rows. + +`space.listMembers` today returns raw `{did, addedAt, addedBy}` rows. Every client ends up wanting profile hydration (handle, displayName, avatar). Add `space.getMembers` (or extend `listMembers`) with: - Cursor-based pagination (current endpoint is unbounded) - Optional `hydrate=true` that joins against the configured profile collection - Sort options (joined-at, alphabetical by handle) +## DID-doc publication helper + +The authority's signing key needs to be published in its DID document under +`#atproto_space_authority` (verification method) — that's how external +verifiers find it. Today the deployer does this manually: +- For `did:web:contrail.example.com`: edit `.well-known/did.json`. +- For `did:plc`: a PLC operation signed with the rotation key. + +A `contrail authority publish-key` CLI subcommand could: +- For `did:web`, emit the verification-method JSON to stdout for the deployer to drop into their DID doc. +- For `did:plc`, build and submit the PLC operation. + +Without this, external verifiers can't validate the authority's credentials. +The in-process default works either way (host has the key directly). + +## Auto-wiring discovery resolvers + +`createPdsBindingResolver` and `createDidDocBindingResolver` are exported but +not wired into the default verifier. Today the in-process verifier uses only +`Local` (configured authority) + `Enrollment` (locally consented spaces). + +For deployments that want to accept credentials from any authority that's +properly bound on a user's PDS or DID doc, the deployer composes the +resolvers manually (see [deployment-shapes](../docs/10-deployment-shapes.md)). + +A higher-level config knob — `spaces.recordHost.acceptExternalAuthorities: true` +or similar — could auto-wire the full discovery chain. Decide whether the +fast-path (Local+Enrollment only) or the universal-path (full chain) is the +right default once we have real cross-host deployments. + ## More tests -The e2e + invite tests cover the happy paths. Gaps: -- Non-owner calling `createSpace` (should succeed — anyone can create their - own) vs non-member trying to use someone else's space URI -- App policy enforcement in both `allow` and `deny` modes (clientId checks) -- `deleteRecord` by owner on another author's record -- Re-querying a soft-deleted space returns NotFound -- `leaveSpace` by owner (should error) -- `whoami` for owner, member, non-member + +Phase 3-5 added good coverage for credentials, binding, enrollment. Gaps that +predate the refactor and still apply: + +- Non-owner calling `createSpace` (should succeed — anyone can create their own). +- App policy enforcement in both `allow` and `deny` modes (clientId checks). +- `deleteRecord` by owner on another author's record (should fail). +- Re-querying a soft-deleted space returns NotFound. +- `leaveSpace` by owner (should error). +- `whoami` for owner, member, non-member, with and without community integration. ## Config-change behavior + What happens today if a deployment: + - Adds a new collection after spaces already contain data? The per-collection table (`spaces_records_<short>`) won't exist until schema init re-runs. `listCollections` swallows the missing-table error, but `putRecord` / @@ -36,42 +75,80 @@ What happens today if a deployment: Need a config-drift audit (or migration) story. ## Verify `clientId` actually flows through + `checkAccess` uses `ServiceAuth.clientId` for app policy checks. Confirm: + - JWT verifier actually extracts `client_id` from real atproto service tokens - (not just our test fixture) -- App policy with a populated `apps[]` blocks/allows correctly in practice -- Empty `apps[]` under `mode: "deny"` blocks everyone (is that what we want?) + (not just our test fixture). +- App policy with a populated `apps[]` blocks/allows correctly in practice. +- Empty `apps[]` under `mode: "deny"` blocks everyone (is that what we want?). If `clientId` is `undefined` in the wild, app policy is decorative. +App policy is also currently checked at credential-issuance time but not +enforced again on the record host. For very long-lived credentials (>2h), an +app removed from the allowlist could continue acting until expiry. The TTL is +the spec's revocation bound; live with it. + ## Join requests (spec-adjacent, not in spec) + The rough spec punts invite/onboarding mechanics to apps. A natural fit given our invite system: a fourth kind `request` where `redeem` creates a pending row for the owner to approve. Likely wants: + - `space.requestJoin` → creates pending row - `space.listJoinRequests` (owner) → pending rows - `space.approveJoinRequest` / `space.denyJoinRequest` -Should this live under `space.*` or a separate extras namespace? +Should this live under `<ns>.space.*` or `<ns>.spaceExt.*`? + +## Authority migration + +A space's authority can change in principle — the `recordHost.enroll` row maps +`spaceUri → authorityDid`, re-enroll with a new authority and credentials +from the new authority will start verifying. But: + +- Existing credentials from the old authority don't auto-revoke; they expire + within their TTL. +- The PDS-record / DID-doc discovery sources (if used) need updating in lockstep. +- No helper API for this — the deployer or owner does each step manually. -## Real-time: SSE / subscriptions -Every collaborative app wants "new records in this space, as they land." -The spec's sync model uses write-notifications through the space owner; we -don't have that yet. Lightweight interim: Server-Sent Events on -`space.subscribeRecords?spaceUri=&collection=`. Works for first-party apps -right away; swap to the real thing later. +A `<ns>.recordHost.transferAuthority` endpoint could automate the wire-level +parts (re-enroll, optionally short-circuit credential cache). -## Namespace split for contrail-specific extras — done -`space.invite.*` and `space.whoami` moved to `<ns>.spaceExt.*`. See -[spaces-spec-mapping.md § Contrail extras](./spaces-spec-mapping.md#contrail-extras-namespace-nsspaceext). -`leaveSpace` is still in `space.*` — revisit if the spec lands with different -self-remove semantics. +## Multi-authority spaces + +In principle the architecture allows several authorities to all sign for one +space (replication / failover scenarios). The host's enrollment is 1:1 today +(one authority per space) but could become 1:N with a small schema change. +Spec is silent. Defer until a real use case. ## Ownership transfer -Dropped for now. The space URI is `at://<ownerDid>/<type>/<key>` — owner DID + +Dropped for now. The space URI is `ats://<ownerDid>/<type>/<key>` — owner DID is baked into the URI, and every record/member/invite row keys off that URI string. Transferring would mean either rewriting every referencing row in a transaction (and breaking external refs to the old URI) or decoupling storage from URI with an internal stable space id (bigger refactor). Revisit once the spec pins down whether ownership transfer exists and what the URI authority is supposed to be post-transfer. + +## Real-time over credentials + +Realtime tickets are signed by `realtime.ticketSecret`, not by the space +authority. If the host/authority split goes far enough that they're operated +by different parties, the realtime ticket model may need rethinking — does +the host mint tickets it then validates itself, or does the authority issue +realtime grants the host honors? Today both run in one process so it doesn't +matter. + +--- + +## Resolved by the six-phase refactor (kept for history) + +- ~~Space-credential flow~~ — done in phase 3. +- ~~Binding resolution (PDS records, DID-doc service entries)~~ — done in phase 4. +- ~~Independent host/authority deployments~~ — done in phase 5. +- ~~Real-time SSE / subscriptions~~ — landed via the realtime module. +- ~~Namespace split for contrail-specific extras~~ — `<ns>.spaceExt.*` shipped. +- ~~Community as separate package~~ — done in phase 6 (`@atmo-dev/contrail-community`). diff --git a/refs/spaces-spec-mapping.md b/refs/spaces-spec-mapping.md index 293a425..70d697b 100644 --- a/refs/spaces-spec-mapping.md +++ b/refs/spaces-spec-mapping.md @@ -7,70 +7,71 @@ is this doc. The goal is to make it obvious, when the real spec lands, where contrail already lines up and where it needs to change. Contrail is a backend-in-a-bottle / simple appview, not a PDS. For permissioned -data it currently stores everything in its own database; the plan is to -switch permissioned reads to come from users' PDSes once the protocol-level -flow is shipped (same story we already have for public records via jetstream). +data it currently stores everything in its own database; the long-term plan +is to switch permissioned reads to come from users' PDSes once the +protocol-level flow is shipped (same story we already have for public records +via jetstream). + +The spaces implementation went through a six-phase refactor (phases 1–6, +documented in conversation history) that aligned the architecture with the +spec and split community out into its own package. This doc reflects the +post-refactor state. --- ## Concept-by-concept alignment -| Spec concept | Contrail | Alignment | Notes | -| ------------------------------ | ------------------------------------------------------------ | --------- | ------------------------------------------------------------------------------ | -| Space owner (DID) | `spaces.owner_did` | ✅ | 1:1 | -| Space type (NSID) | `spaces.type` | ✅ | 1:1 | -| Space key / skey | `spaces.key` | ✅ | TID-generated when caller omits it | -| Record addressing 6-tuple | `(owner, type, key, author-did, collection, rkey)` | ✅ | Storage is keyed by `(space_uri, did, rkey)`; `space_uri` encodes the first 3 | -| Single ACL = member list | `spaces_members (did)` | ✅ | Membership is binary: you're in or you're out. Owner is implicit member. No read/write tiering — apps filter writes themselves | -| Space credential (2–4h token) | _none; service-auth JWTs used directly_ | ❌ | Fine while contrail is a single appview. Add a shim when real PDS sync lands | -| App allow/deny | `appPolicy {mode, apps[]}` | ✅ | Matches spec's default-allow / default-deny model. Visible only to the owner | -| Permissioned repo per user | single DB (`spaces_records_<short>`) | ⚠️ | Structurally compatible — keyed per `(space, author)`. Federation is future | -| ECMH commit / sync log | _none_ | ❌ | Out of scope until federated sync exists | -| Pull-based sync, write notifs | _none_ | ❌ | Same | -| URI scheme | `at://<owner>/<type>/<key>` for spaces; records not exposed | ⚠️ | Spec floats `ats://`. We centralize construction in `src/core/spaces/uri.ts` | -| Authority model for record URI | sidestepped (records keyed, not URI-addressed) | ✅ | Spec is undecided; we don't commit either way | -| Managing app routing | _none (join-requests etc. not modeled yet)_ | ⚠️ | See [spaces-later.md](./spaces-later.md) | +| Spec concept | Contrail | Alignment | Notes | +| ------------------------------ | --------------------------------------------------------------- | --------- | ----------------------------------------------------------------------------------------------------------- | +| Space owner (DID) | `spaces.owner_did` | ✅ | 1:1 | +| Space type (NSID) | `spaces.type` | ✅ | 1:1 | +| Space key / skey | `spaces.key` | ✅ | TID-generated when caller omits it | +| Record addressing 6-tuple | `(owner, type, key, author-did, collection, rkey)` | ✅ | Storage is keyed by `(space_uri, did, rkey)`; `space_uri` encodes the first 3 | +| `ats://` URI scheme | `ats://<owner>/<type>/<key>` | ✅ | Centralized in `src/core/spaces/uri.ts` | +| Single ACL = member list | `spaces_members (did)` | ✅ | Membership is binary. Owner is implicit member. No read/write tiering. | +| Member list `(did, read\|write)` tuples | binary membership only | ⚠️ | Spec says read/write tiered. Pragmatic divergence; flip if/when spec firms | +| Member list as PDS record | server-side state on authority | ⚠️ | Spec says published on owner's PDS, synced. Future migration; the authority caches today | +| Space credential (2–4h token) | ES256 JWTs via `<ns>.space.getCredential` / `refreshCredential` | ✅ | Phase 3. Default 2h TTL. `iss` = authority DID; signed with the authority's published key | +| Credential signed by owner key | signed by **issuer** DID; binding from PDS record / DID-doc / owner-self | ⚠️ ext | Spec says owner-key. We extend so user-owned DIDs can authorize a separate issuer without DID-doc surgery | +| App allow/deny | `appPolicy {mode, apps[]}` | ✅ | Matches default-allow / default-deny. Checked at credential issuance only | +| Discovery via DID doc | `#atproto_space_authority` service entry resolver | ✅ | Phase 4. Plus PDS-record fallback (extension) and owner-self fallback | +| Permissioned repo per user | single DB (`spaces_records_<short>`) | ⚠️ | Structurally compatible — keyed per `(space, author)`. Federation is future | +| ECMH commit / sync log | _none_ | ❌ | Out of scope until federated sync exists | +| Pull-based sync, write notifs | _none_ | ❌ | Same | +| Authority model for record URI | sidestepped (records keyed, not URI-addressed) | ✅ | Spec is undecided; we don't commit either way | +| Managing app routing | _none (join-requests etc. not modeled yet)_ | ⚠️ | See [spaces-later.md](./spaces-later.md) | +| Host/AppView split | `spaces.authority` + `spaces.recordHost` independently runnable | ➕ | Phase 5. Spec implies but doesn't fully model. See [deployment-shapes](../docs/10-deployment-shapes.md) | +| Enrollment as host consent | `<ns>.recordHost.enroll` + `record_host_enrollments` table | ➕ | Phase 5. Spec doesn't address consent — we add explicit binding registration | + +Legend: ✅ aligned · ⚠️ pragmatic divergence · ❌ unimplemented (deliberate) · ➕ extension over the spec --- ## Endpoints -All endpoints are emitted under `<config.namespace>.space.*` from templates in -`lexicon-templates/spaces/`. - -### Read -- `space.listSpaces` — caller's spaces (scope=member|owner) -- `space.getSpace` — metadata; supports `?inviteToken=` bearer read -- `space.listMembers` — members for a space (member/owner only) -- `space.listRecords` — space-scoped record listing; bearer-read supported -- `space.getRecord` — single record; bearer-read supported - -### Write -- `space.putRecord` -- `space.deleteRecord` - -### Owner-gated (space management) -- `space.createSpace` -- `space.addMember` -- `space.removeMember` -- `space.leaveSpace` — self-remove; owner cannot leave (extra) - -### Contrail extras (namespace: `<ns>.spaceExt.*`) -Clearly-off-spec features live under a separate namespace so the `space.*` -surface stays close to whatever the permissioned-data spec becomes. Moved here -from `space.*` in an earlier refactor. - -- `spaceExt.whoami` — caller's relationship to a space (owner / member flags) -- `spaceExt.invite.create` — returns raw token once; hash stored -- `spaceExt.invite.redeem` -- `spaceExt.invite.list` -- `spaceExt.invite.revoke` - -Invites have three kinds: `join`, `read`, `read-join`. `read` tokens grant -bearer-only anonymous read access; `read-join` does both; `join` requires a -signed-in caller and grants a membership row. None of this is in the spec — -it lives here because the spec explicitly defers invite/onboarding mechanics -to apps, and shipping a working invite primitive is useful for every consumer. +All endpoints are emitted under `<config.namespace>.*` from templates in +`packages/lexicons/lexicon-templates/`. + +### Authority (`<ns>.space.*` — spec-aligned) +- `createSpace` `getSpace` `listSpaces` `deleteSpace` +- `listMembers` `addMember` `removeMember` `leaveSpace` +- `getCredential` `refreshCredential` + +### Record host +- `<ns>.space.putRecord` `deleteRecord` `getRecord` `listRecords` +- `<ns>.space.uploadBlob` `getBlob` `listBlobs` (optional) +- `<ns>.recordHost.enroll` + +### Contrail extras (`<ns>.spaceExt.*`) +Clearly-off-spec features that don't map cleanly to the rough spec. +- `whoami` — caller's relationship to a space (owner / member / extension fields) + +### Invites (`<ns>.invite.*`) +- `create` `redeem` `revoke` `list` + +Invites have three kinds: `join`, `read`, `read-join`. Spec defers +invite/onboarding mechanics to apps; we ship a working primitive because +every consumer needs one. ### Collection integration Per-collection `listRecords` / `getRecord` accept `?spaceUri=` (space-scoped) @@ -79,44 +80,54 @@ public + own-member-spaces union (see `src/core/router/collection.ts`). --- +## What changed in the six-phase refactor + +| Phase | Brought us | Notes | +|---|---|---| +| 1 | `SpaceAuthority` + `RecordHost` interface boundary | Pure refactor; `StorageAdapter` is their union | +| 2 | Spaces no longer imports community | Whoami extension hook + `CommunityInviteHandler` interface | +| 3 | Credential issuance + verification | ES256 JWTs, `X-Space-Credential` header, in-process verifier | +| 4 | Binding resolution | PDS-record + DID-doc resolvers; `iss != owner` allowed via the binding | +| 5 | Independent deployment + enrollment | Authority and host runnable as separate processes; `recordHost.enroll` consent | +| 6 | Community as separate package | `@atmo-dev/contrail-community` with integration interface | + +After phase 6 the architecture maps to the spec roughly as: + +``` + spec concept contrail mapping + ──────────── ──────────────── + "space host" ─→ space authority (signs creds, holds members) + "permissioned repo" ─→ record host (stores records, enrolls spaces) + "external space hosts" ─→ binding resolver chain (multi-authority support) + "managing app routing" ─→ not yet (deferred — spaces-later.md) +``` + +--- + ## Migration readiness -Hasn't shipped yet → nothing to migrate, but the shape of what changes when -the real spec lands: - -1. **URI scheme swap (if any).** Centralized in `src/core/spaces/uri.ts` — - flip `at://` to `ats://` (or whatever) in two helpers and every caller - follows. -2. **Space-credential flow.** Needs an endpoint that mints short-lived tokens - from an owner key, and a verifier that accepts them in place of a - service-auth JWT on read paths. The current JWT middleware - (`src/core/spaces/auth.ts`) is the right anchor for this. -3. **Read records from PDSes.** Mirrors the jetstream ingestion we already do - for public data: consume permissioned-repo sync, write into the same - `spaces_records_<short>` tables. The storage schema is already keyed per - `(space, author)` so no migration needed on that side. -4. **ECMH commits & sync log.** Greenfield; unrelated to existing storage. -5. **Endpoint naming.** Spec doesn't pin XRPC names. When it does, rename - lexicon template files + routes. No storage churn. - -### Design decisions worth preserving -- Keep the member list as the single ACL. Don't add roles or per-collection - policies just because it's easy — the spec is emphatic that the member list - is _the_ ACL. -- Membership is binary, not tiered. Previously had `perms: "read" | "write"` - per member row; collapsed to plain membership because the rough spec is - moving toward "member = access, apps filter writes." Delete keeps the - owner / own-record rule, but that's about *which records you can affect*, - not a permission tier on the member row. -- Don't over-engineer the space row with pre-emptive extension columns. - Previously had `member_list_ref` as a hook for externally-managed - membership; dropped because the community-module case is handled via - ownership (community-owned spaces are managed by the community module, no - flag column needed). If a future need for external membership sources - shows up, add the column then. -- Keep `space.whoami`, `space.leaveSpace`, and the invite endpoints clearly - labeled as contrail extras in docs. If the spec ends up naming some of - them, renaming is cheap; relying on them from the base spec isn't. -- Don't mint a canonical record URI. The spec is undecided on the authority - (user DID vs space owner DID); storing records by tuple avoids picking. +What still needs to change when the real spec lands: + +1. **Member list moves to PDS records.** The spec says it's a record on the owner's PDS, synced. Our authority holds it server-side. Future: a watcher consumes member-list records via Jetstream and reconciles into `spaces_members`. Auth-side `addMember` becomes a PDS write rather than an internal API. + +2. **Records federate from user PDSes.** Today the record host *is* the source of truth. When permissioned-repos ship, records federate; the host becomes an aggregator. Storage schema (`spaces_records_<short>`, keyed per `(space, author)`) already supports this — the change is in the write path, not the read path. + +3. **ECMH commits & sync log.** Greenfield. Required for federation. + +4. **Endpoint naming.** Spec doesn't pin XRPC names. When it does, rename lexicon template files + routes. No storage churn. + +5. **Possibly: `(did, read|write)` member tuples.** If the spec stays at tiered membership and doesn't move to binary, add an `access` column on `spaces_members` and branch the ACL check in `acl.ts`. One-day change. + +6. **Possibly: credential `iss = owner DID`.** If the spec forbids the issuer-DID indirection we use for user-owned DIDs, fall back to "owner adds a host-controlled verification method to their DID doc" (HappyView's hidden assumption). Operationally heavier; we hold the looser reading until forced to tighten. + +--- + +## Design decisions worth preserving +- **Keep the member list as the single ACL.** Don't add roles or per-collection policies just because it's easy — the spec is emphatic that the member list is _the_ ACL. +- **Membership is binary, not tiered.** Previously had `perms: "read" | "write"` per member row; collapsed to plain membership because the rough spec is moving toward "member = access, apps filter writes." Delete keeps the owner / own-record rule, but that's about *which records you can affect*, not a permission tier on the member row. +- **Don't over-engineer the space row with pre-emptive extension columns.** Previously had `member_list_ref` as a hook for externally-managed membership; dropped because the community-module case is handled via ownership (community-owned spaces are managed by the community module, no flag column needed). If a future need for external membership sources shows up, add the column then. +- **Keep `spaceExt.whoami`, `space.leaveSpace`, and the invite endpoints clearly labeled as contrail extras in docs.** If the spec ends up naming some of them, renaming is cheap; relying on them from the base spec isn't. +- **Don't mint a canonical record URI.** The spec is undecided on the authority (user DID vs space owner DID); storing records by tuple avoids picking. +- **Enrollment is the host's source of truth.** Even when PDS records and DID-doc service entries declare authority bindings, the host's local enrollment is what actually gates record acceptance. Keeps the host's consent explicit and prevents abuse of the open-ended discovery layer. +- **Keep the issuer-DID indirection as an extension, not a hard architectural choice.** The credential verifier supports `iss == owner` (literal-spec) and `iss != owner` (with binding). If the spec forbids the latter, we degrade gracefully. -- 2.51.2 From 0e9378293f595e8bc8c409734f82c93240839ebd Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 00:56:35 +0200 Subject: [PATCH 08/19] uff, split stuff up --- packages/contrail-base/package.json | 67 +++ .../contrail-base/src/adapters/postgres.ts | 93 +++ packages/contrail-base/src/adapters/sqlite.ts | 39 ++ packages/contrail-base/src/client.ts | 194 ++++++ .../src/community-integration.ts | 67 +++ packages/contrail-base/src/dialect.ts | 127 ++++ packages/contrail-base/src/identity.ts | 159 +++++ packages/contrail-base/src/index.ts | 67 +++ .../src/invite/community-handler.ts | 67 +++ packages/contrail-base/src/invite/token.ts | 43 ++ packages/contrail-base/src/labels/types.ts | 56 ++ .../src/realtime/durable-object.ts | 567 +++++++++++++++++ .../contrail-base/src/realtime/in-memory.ts | 116 ++++ packages/contrail-base/src/realtime/merge.ts | 77 +++ .../src/realtime/query-filter.ts | 235 ++++++++ packages/contrail-base/src/realtime/sse.ts | 98 +++ packages/contrail-base/src/realtime/ticket.ts | 179 ++++++ packages/contrail-base/src/realtime/types.ts | 127 ++++ .../contrail-base/src/realtime/websocket.ts | 84 +++ packages/contrail-base/src/spaces/acl.ts | 69 +++ packages/contrail-base/src/spaces/auth.ts | 177 ++++++ packages/contrail-base/src/spaces/binding.ts | 274 +++++++++ .../contrail-base/src/spaces/blob-adapter.ts | 94 +++ .../contrail-base/src/spaces/credentials.ts | 284 +++++++++ .../contrail-base/src/spaces/in-process.ts | 34 ++ packages/contrail-base/src/spaces/tid.ts | 20 + packages/contrail-base/src/spaces/types.ts | 273 +++++++++ packages/contrail-base/src/spaces/uri.ts | 37 ++ packages/contrail-base/src/types.ts | 513 ++++++++++++++++ packages/contrail-base/tsconfig.build.json | 7 + packages/contrail-base/tsconfig.json | 7 + packages/contrail-base/tsup.config.ts | 11 + packages/contrail-community/package.json | 1 + packages/contrail-community/vitest.config.ts | 9 +- packages/contrail/package.json | 1 + packages/contrail/src/adapters/postgres.ts | 94 +-- packages/contrail/src/adapters/sqlite.ts | 40 +- packages/contrail/src/core/backfill.ts | 1 + packages/contrail/src/core/client.ts | 195 +----- .../src/core/community-integration.ts | 55 +- packages/contrail/src/core/dialect.ts | 128 +--- packages/contrail/src/core/identity.ts | 160 +---- .../src/core/invite/community-handler.ts | 68 +-- packages/contrail/src/core/invite/token.ts | 44 +- packages/contrail/src/core/labels/types.ts | 60 +- .../src/core/realtime/durable-object.ts | 568 +----------------- .../contrail/src/core/realtime/in-memory.ts | 117 +--- packages/contrail/src/core/realtime/merge.ts | 78 +-- .../src/core/realtime/query-filter.ts | 236 +------- packages/contrail/src/core/realtime/sse.ts | 99 +-- packages/contrail/src/core/realtime/ticket.ts | 180 +----- packages/contrail/src/core/realtime/types.ts | 128 +--- .../contrail/src/core/realtime/websocket.ts | 85 +-- packages/contrail/src/core/refresh.ts | 1 + packages/contrail/src/core/spaces/acl.ts | 70 +-- packages/contrail/src/core/spaces/auth.ts | 178 +----- packages/contrail/src/core/spaces/binding.ts | 275 +-------- .../contrail/src/core/spaces/blob-adapter.ts | 95 +-- .../contrail/src/core/spaces/credentials.ts | 285 +-------- .../contrail/src/core/spaces/in-process.ts | 35 +- packages/contrail/src/core/spaces/tid.ts | 21 +- packages/contrail/src/core/spaces/types.ts | 274 +-------- packages/contrail/src/core/spaces/uri.ts | 38 +- packages/contrail/src/core/types.ts | 514 +--------------- packages/contrail/vitest.config.ts | 12 + pnpm-lock.yaml | 49 ++ 66 files changed, 4360 insertions(+), 4096 deletions(-) create mode 100644 packages/contrail-base/package.json create mode 100644 packages/contrail-base/src/adapters/postgres.ts create mode 100644 packages/contrail-base/src/adapters/sqlite.ts create mode 100644 packages/contrail-base/src/client.ts create mode 100644 packages/contrail-base/src/community-integration.ts create mode 100644 packages/contrail-base/src/dialect.ts create mode 100644 packages/contrail-base/src/identity.ts create mode 100644 packages/contrail-base/src/index.ts create mode 100644 packages/contrail-base/src/invite/community-handler.ts create mode 100644 packages/contrail-base/src/invite/token.ts create mode 100644 packages/contrail-base/src/labels/types.ts create mode 100644 packages/contrail-base/src/realtime/durable-object.ts create mode 100644 packages/contrail-base/src/realtime/in-memory.ts create mode 100644 packages/contrail-base/src/realtime/merge.ts create mode 100644 packages/contrail-base/src/realtime/query-filter.ts create mode 100644 packages/contrail-base/src/realtime/sse.ts create mode 100644 packages/contrail-base/src/realtime/ticket.ts create mode 100644 packages/contrail-base/src/realtime/types.ts create mode 100644 packages/contrail-base/src/realtime/websocket.ts create mode 100644 packages/contrail-base/src/spaces/acl.ts create mode 100644 packages/contrail-base/src/spaces/auth.ts create mode 100644 packages/contrail-base/src/spaces/binding.ts create mode 100644 packages/contrail-base/src/spaces/blob-adapter.ts create mode 100644 packages/contrail-base/src/spaces/credentials.ts create mode 100644 packages/contrail-base/src/spaces/in-process.ts create mode 100644 packages/contrail-base/src/spaces/tid.ts create mode 100644 packages/contrail-base/src/spaces/types.ts create mode 100644 packages/contrail-base/src/spaces/uri.ts create mode 100644 packages/contrail-base/src/types.ts create mode 100644 packages/contrail-base/tsconfig.build.json create mode 100644 packages/contrail-base/tsconfig.json create mode 100644 packages/contrail-base/tsup.config.ts diff --git a/packages/contrail-base/package.json b/packages/contrail-base/package.json new file mode 100644 index 0000000..e72201a --- /dev/null +++ b/packages/contrail-base/package.json @@ -0,0 +1,67 @@ +{ + "name": "@atmo-dev/contrail-base", + "version": "0.6.0", + "description": "Shared infrastructure for the contrail family of packages — interfaces (SpaceAuthority, RecordHost, CommunityIntegration), credential primitives, binding resolvers, realtime infra, schema scaffolding. No routes, no tables of its own.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + }, + "./sqlite": { + "types": "./dist/adapters/sqlite.d.ts", + "import": "./dist/adapters/sqlite.js" + }, + "./postgres": { + "types": "./dist/adapters/postgres.d.ts", + "import": "./dist/adapters/postgres.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-base" + }, + "keywords": [ + "atproto", + "contrail" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/atproto": "^3.1.10", + "@atcute/cid": "^2.4.1", + "@atcute/client": "^4.2.1", + "@atcute/identity": "^1.1.4", + "@atcute/identity-resolver": "^1.2.2", + "@atcute/lexicons": "^1.2.9", + "@atcute/xrpc-server": "^0.1.12", + "hono": "^4.12.8" + }, + "devDependencies": { + "@types/node": "^25.5.0", + "@types/pg": "^8.20.0", + "pg": "^8.20.0", + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "peerDependencies": { + "pg": "^8.0.0" + }, + "peerDependenciesMeta": { + "pg": { + "optional": true + } + }, + "license": "MIT" +} diff --git a/packages/contrail-base/src/adapters/postgres.ts b/packages/contrail-base/src/adapters/postgres.ts new file mode 100644 index 0000000..e8c2b14 --- /dev/null +++ b/packages/contrail-base/src/adapters/postgres.ts @@ -0,0 +1,93 @@ +import pg from "pg"; +import type { Database, Statement } from "../types"; +import { postgresDialect } from "../dialect"; + +/** Internal interface for statements that can run on a specific client */ +interface PgStatement extends Statement { + /** Execute on a specific client (used by batch for transaction isolation) */ + _runOn(client: pg.PoolClient): Promise<any>; +} + +/** Column names known to be BIGINT — PostgreSQL returns these as strings */ +const BIGINT_COLUMNS = new Set(["time_us", "indexed_at", "resolved_at"]); + +function normalizeRow(row: any): any { + if (!row) return row; + if (typeof row.record === "object" && row.record !== null) { + row.record = JSON.stringify(row.record); + } + for (const col of BIGINT_COLUMNS) { + if (typeof row[col] === "string") row[col] = Number(row[col]); + } + return row; +} + +export function createPostgresDatabase(pool: pg.Pool): Database { + function rewritePlaceholders(sql: string): string { + let idx = 0; + let inString = false; + let result = ""; + for (let i = 0; i < sql.length; i++) { + const ch = sql[i]; + if (ch === "'" && sql[i - 1] !== "\\") { + inString = !inString; + result += ch; + } else if (ch === "?" && !inString) { + result += `$${++idx}`; + } else { + result += ch; + } + } + return result; + } + + function wrapStatement(sql: string, boundValues: any[] = []): PgStatement { + const pgSql = rewritePlaceholders(sql); + + return { + bind(...values: any[]): PgStatement { + return wrapStatement(sql, values); + }, + async run() { + const result = await pool.query(pgSql, boundValues); + return { changes: result.rowCount }; + }, + async _runOn(client: pg.PoolClient) { + const result = await client.query(pgSql, boundValues); + return { changes: result.rowCount }; + }, + async all<T>() { + const result = await pool.query(pgSql, boundValues); + return { results: result.rows.map(normalizeRow) as T[] }; + }, + async first<T>() { + const result = await pool.query(pgSql, boundValues); + return result.rows[0] ? (normalizeRow(result.rows[0]) as T) : null; + }, + }; + } + + return { + prepare(sql: string): Statement { + return wrapStatement(sql); + }, + async batch(stmts: Statement[]): Promise<any[]> { + const client = await pool.connect(); + try { + await client.query("BEGIN"); + const results: any[] = []; + for (const stmt of stmts) { + results.push(await (stmt as PgStatement)._runOn(client)); + } + await client.query("COMMIT"); + return results; + } catch (e) { + await client.query("ROLLBACK"); + throw e; + } finally { + client.release(); + } + }, + dialect: postgresDialect, + }; +} diff --git a/packages/contrail-base/src/adapters/sqlite.ts b/packages/contrail-base/src/adapters/sqlite.ts new file mode 100644 index 0000000..305dc39 --- /dev/null +++ b/packages/contrail-base/src/adapters/sqlite.ts @@ -0,0 +1,39 @@ +import { DatabaseSync } from "node:sqlite"; +import type { Database, Statement } from "../types"; +import { sqliteDialect } from "../dialect"; + +export function createSqliteDatabase(path: string): Database { + const raw = new DatabaseSync(path); + raw.exec("PRAGMA journal_mode = WAL"); + + function wrapStatement(sql: string, boundValues: any[] = []): Statement { + return { + bind(...values: any[]): Statement { + return wrapStatement(sql, values); + }, + async run() { + return raw.prepare(sql).run(...boundValues); + }, + async all<T>() { + return { results: raw.prepare(sql).all(...boundValues) as T[] }; + }, + async first<T>() { + return (raw.prepare(sql).get(...boundValues) as T) ?? null; + }, + }; + } + + return { + prepare(sql: string): Statement { + return wrapStatement(sql); + }, + async batch(stmts: Statement[]): Promise<any[]> { + const results: any[] = []; + for (const stmt of stmts) { + results.push(await stmt.run()); + } + return results; + }, + dialect: sqliteDialect, + }; +} diff --git a/packages/contrail-base/src/client.ts b/packages/contrail-base/src/client.ts new file mode 100644 index 0000000..f37bea3 --- /dev/null +++ b/packages/contrail-base/src/client.ts @@ -0,0 +1,194 @@ +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, +} from "@atcute/identity-resolver"; +import { type Did } from "@atcute/lexicons"; +import { Client, simpleFetchHandler } from "@atcute/client"; +import type {} from "@atcute/atproto"; +import type { Database } from "./types"; + +// Slingshot-first PDS resolution with fallback to DID document resolution +const SLINGSHOT_URL = + "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; + +export interface ResolvedIdentity { + did: string; + handle: string | null; + pds: string | null; +} + +/** Reject PDS URLs that point to private/internal addresses or non-HTTPS */ +function validatePdsUrl(url: string): boolean { + try { + const parsed = new URL(url); + if (parsed.protocol !== "https:") return false; + const host = parsed.hostname; + // Block private/internal IP ranges + if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; + if (host.startsWith("10.")) return false; + if (host.startsWith("192.168.")) return false; + if (host.startsWith("169.254.")) return false; + if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; + return true; + } catch { + return false; + } +} + +async function resolveViaSlingshot( + identifier: string +): Promise<ResolvedIdentity | undefined> { + const url = new URL(SLINGSHOT_URL); + url.searchParams.set("identifier", identifier); + + try { + const response = await fetch(url.toString()); + if (!response.ok) return undefined; + const data = (await response.json()) as { + did?: string; + handle?: string; + pds?: string; + }; + if (!data.did && !data.pds) return undefined; + return { + did: data.did ?? identifier, + handle: data.handle ?? null, + pds: data.pds ?? null, + }; + } catch { + return undefined; + } +} + +const didResolver = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, +}); + +async function getPDSViaDidDoc(did: Did): Promise<string | undefined> { + const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); + return doc.service + ?.find((s) => s.id === "#atproto_pds") + ?.serviceEndpoint.toString(); +} + +/** + * Resolve identity info (did, handle, pds) for a DID or handle. + * Uses slingshot first, falls back to DID doc for PDS. + */ +export async function resolvePDS( + identifier: string +): Promise<ResolvedIdentity | undefined> { + const result = await resolveViaSlingshot(identifier); + if (result?.pds) { + if (!validatePdsUrl(result.pds)) return { ...result, pds: null }; + return result; + } + + // Fall back to DID doc resolution (only works for DIDs, not handles) + if (identifier.startsWith("did:")) { + try { + const pds = await getPDSViaDidDoc(identifier as Did); + if (pds && validatePdsUrl(pds)) { + return { + did: identifier, + handle: result?.handle ?? null, + pds, + }; + } + } catch { + // ignore + } + } + + return result; +} + +// In-memory PDS cache with TTL + size limit, plus in-flight deduplication +const PDS_CACHE_TTL = 60 * 60 * 1000; // 1 hour +const PDS_CACHE_MAX = 10_000; +const pdsCache = new Map<string, { pds: string; at: number }>(); +const pdsInflight = new Map<string, Promise<string | undefined>>(); + +function pdsCacheGet(did: string): string | undefined { + const entry = pdsCache.get(did); + if (!entry) return undefined; + if (Date.now() - entry.at > PDS_CACHE_TTL) { + pdsCache.delete(did); + return undefined; + } + return entry.pds; +} + +function pdsCacheSet(did: string, pds: string): void { + // Evict oldest entries if over limit + if (pdsCache.size >= PDS_CACHE_MAX) { + const first = pdsCache.keys().next().value; + if (first) pdsCache.delete(first); + } + pdsCache.set(did, { pds, at: Date.now() }); +} + +export async function getPDS( + did: Did, + db?: Database +): Promise<string | undefined> { + const mem = pdsCacheGet(did); + if (mem) return mem; + + // Deduplicate concurrent calls for the same DID + const inflight = pdsInflight.get(did); + if (inflight) return inflight; + + const promise = resolvePDSCached(did, db); + pdsInflight.set(did, promise); + try { + return await promise; + } finally { + pdsInflight.delete(did); + } +} + +async function resolvePDSCached( + did: Did, + db?: Database +): Promise<string | undefined> { + if (db) { + const cached = await db + .prepare("SELECT pds FROM identities WHERE did = ? AND pds IS NOT NULL") + .bind(did) + .first<{ pds: string }>(); + if (cached?.pds) { + pdsCacheSet(did, cached.pds); + return cached.pds; + } + } + + const resolved = await resolvePDS(did); + if (!resolved?.pds) return undefined; + + pdsCacheSet(did, resolved.pds); + + // Persist to DB for future runs + if (db) { + await db + .prepare( + "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET pds = excluded.pds, handle = COALESCE(excluded.handle, identities.handle), resolved_at = excluded.resolved_at" + ) + .bind(did, resolved.handle, resolved.pds, Date.now()) + .run(); + } + + return resolved.pds; +} + +export async function getClient(did: Did, db?: Database): Promise<Client> { + const pds = await getPDS(did, db); + if (!pds) throw new Error(`PDS not found for ${did}`); + return new Client({ + handler: simpleFetchHandler({ service: pds }), + }); +} diff --git a/packages/contrail-base/src/community-integration.ts b/packages/contrail-base/src/community-integration.ts new file mode 100644 index 0000000..bc2f936 --- /dev/null +++ b/packages/contrail-base/src/community-integration.ts @@ -0,0 +1,67 @@ +/** Pluggable integration surface for the community module. + * + * Phase 6 extracted community to its own package (`@atmo-dev/contrail-community`). + * The contrail core package never imports from it — couplings only flow + * through these interfaces. The community package's + * `createCommunityIntegration({ ... })` returns a {@link CommunityIntegration}, + * which the consumer hands to `createApp` via `options.community`. + * + * Two layers: + * - {@link CommunityProbe}: minimal "is this a community DID" / "what + * spaces does this caller reach" surface used by realtime + collection + * routes for community-aware dispatch. + * - {@link CommunityIntegration}: the umbrella bundle — probe, whoami + * extension, invite handler, plus route + schema wiring that the + * umbrella router calls during setup. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { Database } from "./types"; +import type { CommunityInviteHandler } from "./invite/community-handler"; + +/** Optional hook to extend `<ns>.spaceExt.whoami` with extra fields when a + * module above spaces (e.g. community) wants to override the default + * binary-membership response. If the hook returns a non-null object, that + * object is the entire response body. If null, falls through to the + * default behavior (just `isOwner`/`isMember`). + * + * Spaces stays community-agnostic: any consumer can plug in here. */ +export type WhoamiExtension = (input: { + spaceUri: string; + callerDid: string; + isOwner: boolean; + ownerDid: string; +}) => Promise<Record<string, unknown> | null>; + +/** Narrow interface for the deep callers (realtime/resolve, router/collection) + * that just need to ask "is this a community DID?" or "what spaces does this + * caller reach via community membership?" */ +export interface CommunityProbe { + /** Look up a community row by DID. Returns null for non-community DIDs. + * Callers usually only check truthiness — community-specific fields stay + * inside the community package. */ + getCommunity(did: string): Promise<{ did: string } | null>; + + /** Resolve the set of space URIs reachable by `callerDid` through community + * membership (direct grants + delegations). Used by realtime to expand + * community: topics into the caller's concrete space: topics. */ + resolveReachableSpaces(callerDid: string): Promise<Set<string>>; +} + +/** Umbrella integration the consumer constructs once and hands to createApp. + * contrail core treats this as an opaque bundle — it doesn't introspect + * community state, just calls these methods at the right wiring points. */ +export interface CommunityIntegration { + /** Probe used by realtime + collection cross-cutting concerns. */ + probe: CommunityProbe; + /** Whoami extension that returns `accessLevel` for community-owned spaces. */ + whoamiExtension: WhoamiExtension; + /** Handler for the community-grant path of the unified invite surface. */ + inviteHandler: CommunityInviteHandler; + /** Register `<ns>.community.*` routes onto the Hono app. */ + registerRoutes( + app: Hono, + options?: { authMiddleware?: MiddlewareHandler } + ): void; + /** Apply community schema (DDL) to the database. Called by initSchema. */ + applySchema(db: Database): Promise<void>; +} diff --git a/packages/contrail-base/src/dialect.ts b/packages/contrail-base/src/dialect.ts new file mode 100644 index 0000000..e3bb790 --- /dev/null +++ b/packages/contrail-base/src/dialect.ts @@ -0,0 +1,127 @@ +/** Get the dialect from a Database, defaulting to SQLite (for D1 compatibility) */ +export function getDialect(db: { dialect?: SqlDialect }): SqlDialect { + return db.dialect ?? sqliteDialect; +} + +const SAFE_FIELD = /^[a-zA-Z0-9_.]+$/; + +function assertSafeField(field: string): void { + if (!SAFE_FIELD.test(field)) { + throw new Error(`Invalid field name: ${field}`); + } +} + +export interface SqlDialect { + /** json_extract(col, '$.field') or col->>'field' */ + jsonExtract(column: string, field: string): string; + + /** Convert INSERT INTO to ignore-duplicates form. + * SQLite: INSERT INTO → INSERT OR IGNORE INTO + * PG: appends ON CONFLICT DO NOTHING + * Accepts full SQL starting with "INSERT INTO" (works with both VALUES and SELECT). */ + insertOrIgnore(sql: string): string; + + /** Column type for the record column: TEXT (SQLite) or JSONB (PostgreSQL) */ + readonly recordColumnType: string; + + /** FTS strategy: 'virtual-table' (SQLite FTS5) or 'generated-column' (PG tsvector) */ + readonly ftsStrategy: "virtual-table" | "generated-column"; + + /** INTEGER type name — same on both, but PostgreSQL may want BIGINT for time_us */ + readonly integerType: string; + + /** BIGINT type name for timestamps */ + readonly bigintType: string; + + /** Wrap an expression for use in CREATE INDEX — PostgreSQL requires parens around expressions */ + indexExpression(expr: string): string; +} + +export const sqliteDialect: SqlDialect = { + jsonExtract(column: string, field: string): string { + assertSafeField(field); + return `json_extract(${column}, '$.${field}')`; + }, + + insertOrIgnore(sql: string): string { + return sql.replace(/^INSERT INTO/, "INSERT OR IGNORE INTO"); + }, + + recordColumnType: "TEXT", + ftsStrategy: "virtual-table", + integerType: "INTEGER", + bigintType: "INTEGER", + + indexExpression(expr: string): string { + return expr; + }, +}; + +export const postgresDialect: SqlDialect = { + jsonExtract(column: string, field: string): string { + assertSafeField(field); + const parts = field.split("."); + if (parts.length === 1) { + return `${column}->>'${parts[0]}'`; + } + // a.b.c → col->'a'->'b'->>'c' + const intermediate = parts.slice(0, -1).map((p) => `->'${p}'`).join(""); + return `${column}${intermediate}->>'${parts[parts.length - 1]}'`; + }, + + insertOrIgnore(sql: string): string { + return `${sql} ON CONFLICT DO NOTHING`; + }, + + recordColumnType: "JSONB", + ftsStrategy: "generated-column", + integerType: "INTEGER", + bigintType: "BIGINT", + + indexExpression(expr: string): string { + return `(${expr})`; + }, +}; + +/** Generate FTS schema statements based on dialect */ +export function buildFtsSchema( + dialect: SqlDialect, + recordsTable: string, + fields: string[] +): string[] { + if (dialect.ftsStrategy === "virtual-table") { + const ftsTable = recordsTable.replace("records_", "fts_"); + return [ + `CREATE VIRTUAL TABLE IF NOT EXISTS ${ftsTable} USING fts5(uri UNINDEXED, content)` + ]; + } else { + const concatExpr = fields + .map((f) => `COALESCE(${dialect.jsonExtract("record", f)}, '')`) + .join(" || ' ' || "); + return [ + `ALTER TABLE ${recordsTable} ADD COLUMN IF NOT EXISTS search_vector TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', ${concatExpr})) STORED`, + `CREATE INDEX IF NOT EXISTS idx_${recordsTable}_search ON ${recordsTable} USING GIN (search_vector)`, + ]; + } +} + +/** Generate FTS query clause based on dialect */ +export function ftsQueryClause( + dialect: SqlDialect, + recordsTable: string +): { join: string; condition: string; orderExpr: string } { + if (dialect.ftsStrategy === "virtual-table") { + const ftsTable = recordsTable.replace("records_", "fts_"); + return { + join: `JOIN ${ftsTable} fts ON fts.uri = r.uri`, + condition: "fts.content MATCH ?", + orderExpr: "fts.rank", + }; + } else { + return { + join: "", + condition: "r.search_vector @@ plainto_tsquery('english', ?)", + orderExpr: "ts_rank(r.search_vector, plainto_tsquery('english', ?))", + }; + } +} diff --git a/packages/contrail-base/src/identity.ts b/packages/contrail-base/src/identity.ts new file mode 100644 index 0000000..05040fd --- /dev/null +++ b/packages/contrail-base/src/identity.ts @@ -0,0 +1,159 @@ +import type { Did } from "@atcute/lexicons"; +import type { Database, Logger } from "./types"; +import { isDid, isHandle } from "@atcute/lexicons/syntax"; +import { resolvePDS } from "./client"; + +const STALE_MS = 24 * 60 * 60 * 1000; // 24 hours + +export interface Identity { + did: string; + handle: string | null; + pds: string | null; + resolved_at: number; +} + +async function saveIdentity(db: Database, identity: Identity): Promise<void> { + await db + .prepare( + "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET handle = excluded.handle, pds = excluded.pds, resolved_at = excluded.resolved_at" + ) + .bind(identity.did, identity.handle, identity.pds, identity.resolved_at) + .run(); +} + +function isStale(resolvedAt: number): boolean { + return Date.now() - resolvedAt >= STALE_MS; +} + +async function fetchAndSave( + db: Database, + identifier: string, + cached?: Identity | null +): Promise<Identity> { + const resolved = await resolvePDS(identifier); + const identity: Identity = { + did: resolved?.did ?? identifier, + handle: resolved?.handle ?? cached?.handle ?? null, + pds: resolved?.pds ?? cached?.pds ?? null, + resolved_at: Date.now(), + }; + await saveIdentity(db, identity); + return identity; +} + +export async function resolveIdentity( + db: Database, + did: Did +): Promise<Identity> { + const cached = await db + .prepare("SELECT did, handle, pds, resolved_at FROM identities WHERE did = ?") + .bind(did) + .first<Identity>(); + + if (cached && !isStale(cached.resolved_at)) return cached; + + return fetchAndSave(db, did, cached); +} + +export async function resolveIdentities( + db: Database, + dids: string[] +): Promise<Map<string, Identity>> { + const map = new Map<string, Identity>(); + if (dids.length === 0) return map; + + // Batch lookup from DB + const BATCH = 50; + for (let i = 0; i < dids.length; i += BATCH) { + const chunk = dids.slice(i, i + BATCH); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare(`SELECT did, handle, pds, resolved_at FROM identities WHERE did IN (${placeholders})`) + .bind(...chunk) + .all<Identity>(); + for (const row of rows.results ?? []) { + map.set(row.did, row); + } + } + + // Resolve missing via slingshot directly (no redundant DB lookup) + for (const did of dids) { + if (map.has(did) || !isDid(did)) continue; + try { + const identity = await fetchAndSave(db, did); + map.set(did, identity); + } catch { + // Silently skip unresolvable identities + } + } + + return map; +} + +export async function resolveActor( + db: Database, + actor: string +): Promise<string | null> { + if (isDid(actor)) return actor; + if (!isHandle(actor)) return null; + + // Look up handle in identities table + const cached = await db + .prepare("SELECT did, resolved_at FROM identities WHERE handle = ?") + .bind(actor) + .first<{ did: string; resolved_at: number }>(); + + if (cached && !isStale(cached.resolved_at)) return cached.did; + + // Resolve via slingshot + const resolved = await resolvePDS(actor); + if (!resolved?.did || !isDid(resolved.did)) return null; + + await saveIdentity(db, { + did: resolved.did, + handle: resolved.handle ?? actor, + pds: resolved.pds ?? null, + resolved_at: Date.now(), + }); + + return resolved.did; +} + +export async function refreshStaleIdentities( + db: Database, + dids: string[] +): Promise<void> { + if (dids.length === 0) return; + + const unique = [...new Set(dids)].filter(isDid); + if (unique.length === 0) return; + + const staleThreshold = Date.now() - STALE_MS; + const toRefresh: string[] = []; + + const BATCH = 50; + for (let i = 0; i < unique.length; i += BATCH) { + const chunk = unique.slice(i, i + BATCH); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare(`SELECT did, resolved_at FROM identities WHERE did IN (${placeholders})`) + .bind(...chunk) + .all<{ did: string; resolved_at: number }>(); + + const found = new Map((rows.results ?? []).map((r) => [r.did, r.resolved_at])); + for (const did of chunk) { + const resolvedAt = found.get(did); + if (resolvedAt === undefined || resolvedAt < staleThreshold) { + toRefresh.push(did); + } + } + } + + for (const did of toRefresh) { + try { + await fetchAndSave(db, did); + } catch { + // Silently skip unresolvable identities + } + } +} diff --git a/packages/contrail-base/src/index.ts b/packages/contrail-base/src/index.ts new file mode 100644 index 0000000..e11f46c --- /dev/null +++ b/packages/contrail-base/src/index.ts @@ -0,0 +1,67 @@ +/** @atmo-dev/contrail-base — shared infrastructure for the contrail family. + * + * No routes. No tables of its own. Pure types, interfaces, primitives, and + * shared utilities used across contrail / contrail-appview / contrail-authority / + * contrail-record-host / contrail-community. + * + * Re-exported wholesale from each source module — anything internal that + * needed to be hidden would have an explicit subpath export instead. */ + +// Core types + config + helpers (Database, ContrailConfig, dialect helpers, etc.) +export * from "./types"; + +// Dialect (SqlDialect, getDialect, sqliteDialect, postgresDialect, buildFtsSchema) +export * from "./dialect"; + +// Identity (resolveActor, resolveIdentities, refreshStaleIdentities) +export * from "./identity"; + +// PDS client helpers (getPDS, getClient) +export * from "./client"; + +// Spaces interfaces + shared types +export * from "./spaces/types"; + +// Spaces URI helpers +export * from "./spaces/uri"; + +// TID generator +export * from "./spaces/tid"; + +// In-process auth marker +export * from "./spaces/in-process"; + +// Service-auth verification +export * from "./spaces/auth"; + +// ACL pure functions +export * from "./spaces/acl"; + +// Credentials +export * from "./spaces/credentials"; + +// Binding + key resolution +export * from "./spaces/binding"; + +// Blob adapter interface + built-in impls +export * from "./spaces/blob-adapter"; + +// Invite token primitives + community-handler interface +export * from "./invite/token"; +export * from "./invite/community-handler"; + +// Community integration interface + WhoamiExtension +export * from "./community-integration"; + +// Labels types +export * from "./labels/types"; + +// Realtime infrastructure +export * from "./realtime/types"; +export * from "./realtime/in-memory"; +export * from "./realtime/ticket"; +export * from "./realtime/durable-object"; +export * from "./realtime/sse"; +export * from "./realtime/websocket"; +export * from "./realtime/merge"; +export * from "./realtime/query-filter"; diff --git a/packages/contrail-base/src/invite/community-handler.ts b/packages/contrail-base/src/invite/community-handler.ts new file mode 100644 index 0000000..1043525 --- /dev/null +++ b/packages/contrail-base/src/invite/community-handler.ts @@ -0,0 +1,67 @@ +/** Pluggable handler for community-grant invites within the unified invite + * surface. The invite router calls into this when the target space is + * community-owned, or "tries" it on the redeem / revoke-without-spaceUri + * paths. Community module provides the impl; invite/router doesn't import + * from community at all. + * + * Each method returns a `HandlerResponse`: a `{status, body}` envelope that + * the router relays as JSON, or `null` (only on the "try" methods) meaning + * "not applicable, fall through to the user-owned path." */ + +export type HandlerResponse = { + status: number; + body: Record<string, unknown>; +}; + +export interface CommunityInviteHandler { + /** True iff this space is owned by a community (vs. a regular user DID). + * Used by the invite router to choose the dispatch path on + * create / list / revoke-with-spaceUri. */ + isCommunityOwned(spaceUri: string): Promise<boolean>; + + /** Create a community-grant invite. Caller is validated upstream for + * having a JWT; this method handles the access-level checks. */ + create(input: { + spaceUri: string; + callerDid: string; + /** Raw caller-supplied access level — implementation validates. */ + accessLevel?: string; + /** Caller-supplied `kind` field — community spaces don't accept this; the + * handler returns an InvalidRequest if set. */ + kind?: string; + expiresAt: number | null; + maxUses: number | null; + note: string | null; + }): Promise<HandlerResponse>; + + /** List invites for a community-owned space. */ + list(input: { + spaceUri: string; + callerDid: string; + includeRevoked: boolean; + }): Promise<HandlerResponse>; + + /** Revoke a known community-owned invite (caller already passed spaceUri + * and the router classified it as community-owned). */ + revoke(input: { + spaceUri: string; + tokenHash: string; + callerDid: string; + }): Promise<HandlerResponse>; + + /** Revoke without a spaceUri — try to find the invite in the community + * table; return null if not a community invite (router falls through). */ + tryRevokeByToken(input: { + tokenHash: string; + callerDid: string; + }): Promise<HandlerResponse | null>; + + /** Try to redeem a token as a community invite. Returns null if the token + * is not a community invite, in which case the router falls through to + * the user-owned redeem path. */ + tryRedeem(input: { + tokenHash: string; + callerDid: string; + now: number; + }): Promise<HandlerResponse | null>; +} diff --git a/packages/contrail-base/src/invite/token.ts b/packages/contrail-base/src/invite/token.ts new file mode 100644 index 0000000..76f4880 --- /dev/null +++ b/packages/contrail-base/src/invite/token.ts @@ -0,0 +1,43 @@ +const B64U_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +function bytesToB64Url(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += 3) { + const b0 = bytes[i]; + const b1 = bytes[i + 1] ?? 0; + const b2 = bytes[i + 2] ?? 0; + out += B64U_ALPHABET[b0 >> 2]; + out += B64U_ALPHABET[((b0 & 3) << 4) | (b1 >> 4)]; + if (i + 1 < bytes.length) out += B64U_ALPHABET[((b1 & 15) << 2) | (b2 >> 6)]; + if (i + 2 < bytes.length) out += B64U_ALPHABET[b2 & 63]; + } + return out; +} + +function bytesToHex(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i++) out += bytes[i].toString(16).padStart(2, "0"); + return out; +} + +/** Generate a fresh invite token (cryptographically random, 32 bytes base64url-encoded). */ +export function generateInviteToken(): string { + const bytes = new Uint8Array(32); + crypto.getRandomValues(bytes); + return bytesToB64Url(bytes); +} + +/** SHA-256 hash of a token, hex-encoded. Used as the PK in storage so raw tokens are never persisted. */ +export async function hashInviteToken(token: string): Promise<string> { + const encoded = new TextEncoder().encode(token); + const digest = await crypto.subtle.digest("SHA-256", encoded); + return bytesToHex(new Uint8Array(digest)); +} + +/** Convenience: generate a token and return both the raw form (returned to + * the creator once) and its hash (persisted as the stable ID). */ +export async function mintInviteToken(): Promise<{ token: string; tokenHash: string }> { + const token = generateInviteToken(); + const tokenHash = await hashInviteToken(token); + return { token, tokenHash }; +} diff --git a/packages/contrail-base/src/labels/types.ts b/packages/contrail-base/src/labels/types.ts new file mode 100644 index 0000000..f2e36c2 --- /dev/null +++ b/packages/contrail-base/src/labels/types.ts @@ -0,0 +1,56 @@ +import type { Database } from "../types"; + +/** A labeler the operator wants contrail to track. */ +export interface LabelerSource { + /** Labeler DID — `did:plc:...` or `did:web:...`. */ + did: string; + /** Override the service endpoint resolution. Otherwise resolved from the + * DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ + endpoint?: string; + /** Backfill from `cursor=0` on first sight. Defaults to true. Set false + * for "start from now" — useful for very chatty labelers. */ + backfill?: boolean; +} + +export interface LabelsConfig { + /** Labelers to subscribe to and index. */ + sources: LabelerSource[]; + /** DIDs honored when the caller sends no `atproto-accept-labelers` / + * `?labelers=`. Defaults to every entry in `sources`. Set `[]` for + * opt-in-only — clients see no labels unless they ask. */ + defaults?: string[]; + /** Per-request cap. Default: 20 (matches Bluesky). */ + maxPerRequest?: number; +} + +export const DEFAULT_LABELS_MAX_PER_REQUEST = 20; + +/** A single label as stored. Matches `com.atproto.label.defs#label`. */ +export interface LabelRow { + /** Issuing labeler DID. */ + src: string; + /** Subject — at-URI for record labels, plain DID for account labels. */ + uri: string; + /** Label value — kebab-case, ≤128 bytes per spec. */ + val: string; + /** Optional CID pin to a specific record version. */ + cid: string | null; + /** When true, retracts a previously-emitted label for the same (src, uri, val). */ + neg: boolean; + /** Expiry, unix seconds. Past this, hydration drops the row. */ + exp: number | null; + /** Creation timestamp, unix seconds — what we collapse on. */ + cts: number; + /** Raw signature bytes. Stored when present so we can re-emit later; + * not verified in v1. */ + sig: Uint8Array | null; +} + +/** Per-labeler state row — endpoint cache and last-seen seq cursor. */ +export interface LabelerCursorRow { + did: string; + cursor: number; + endpoint: string | null; + resolved_at: number | null; +} + diff --git a/packages/contrail-base/src/realtime/durable-object.ts b/packages/contrail-base/src/realtime/durable-object.ts new file mode 100644 index 0000000..966ab53 --- /dev/null +++ b/packages/contrail-base/src/realtime/durable-object.ts @@ -0,0 +1,567 @@ +/** Durable Object backend for realtime PubSub. + * + * Two pieces live here: + * 1. `RealtimePubSubDO` — the DO class. Ship it from your Worker via + * `export { RealtimePubSubDO } from "@atmo-dev/contrail";` and bind it in + * your `wrangler.toml`. One DO = one topic; addressed by name. + * 2. `DurableObjectPubSub` — client-side adapter implementing the PubSub + * interface against a DO namespace binding. + * + * Wire format between Worker and DO (internal, not a stable public contract): + * POST /publish — body = RealtimeEvent JSON + * GET /subscribe — server-sent events stream, optionally with + * `Upgrade: websocket` for WS connections. + * Auth/ACL is already checked at the Worker edge; + * the DO trusts anything that reaches it. */ + +import type { PubSub, RealtimeEvent } from "./types"; +import { translateForQuery, type TranslatedEnvelope } from "./query-filter"; +type TranslatedEvent = TranslatedEnvelope; + +/** Query spec attached to a WS subscriber, used to filter events before + * delivery. Shape matches what the Worker's `watchRecords` handler builds; + * forwarded to the DO via trusted internal headers on the WS upgrade. */ +export interface SubscriberQuerySpec { + /** NSID of the primary collection the client is watching. */ + collection: string; + /** Space URI this subscription is scoped to. Events outside are dropped. */ + spaceUri: string; + /** Hydrated relations. Keyed by relName — value is the child collection + * NSID and the field on the child record that references the parent. */ + hydrate?: Record<string, { childCollection: string; matchField: string }>; +} + +// ---- Minimal structural typings so we don't depend on @cloudflare/workers-types +// at the library level. Callers on Workers will have proper types. +// ---------------------------------------------------------------------------- + +export interface DurableObjectId { + toString(): string; +} + +export interface DurableObjectStub { + fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>; +} + +export interface DurableObjectNamespace { + idFromName(name: string): DurableObjectId; + get(id: DurableObjectId): DurableObjectStub; +} + +export interface DurableObjectState { + acceptWebSocket(ws: any, tags?: string[]): void; + getWebSockets(tag?: string): any[]; +} + +// ---------------------------------------------------------------------------- +// Client adapter +// ---------------------------------------------------------------------------- + +export class DurableObjectPubSub implements PubSub { + constructor(private readonly namespace: DurableObjectNamespace) {} + + private stub(topic: string): DurableObjectStub { + return this.namespace.get(this.namespace.idFromName(topic)); + } + + async publish(event: RealtimeEvent): Promise<void> { + const res = await this.stub(event.topic).fetch("https://do/publish", { + method: "POST", + headers: { "content-type": "application/json" }, + body: JSON.stringify(event), + }); + if (!res.ok) { + // Consume the body so the edge doesn't hold a dangling response. + await res.text().catch(() => ""); + throw new Error(`DO publish failed (${res.status})`); + } + } + + /** In-Worker server-side subscribe. Browsers should hit the SSE endpoint + * directly; this is the path for an in-process consumer that wants an + * AsyncIterable (tests, bots embedded in the Worker). */ + subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { + const stub = this.stub(topic); + return { + [Symbol.asyncIterator]() { + return pullIterator(stub, signal); + }, + }; + } + + /** Forward an incoming browser WS upgrade (or SSE GET) through to the DO + * that owns this topic, attaching a query-filter spec that the DO will use + * to decide what to deliver. The Worker must verify auth + spec validity + * before calling this — the DO trusts the headers. */ + async forwardSubscribe( + topic: string, + request: Request, + opts: { + did?: string; + querySpec?: SubscriberQuerySpec; + /** Unix ms. DO replays any buffered event with ts > sinceTs before + * going live — closes the snapshot→WS race window on the client. */ + sinceTs?: number; + } = {} + ): Promise<Response> { + const headers = new Headers(request.headers); + if (opts.querySpec) { + headers.set("X-Contrail-Query-Spec", JSON.stringify(opts.querySpec)); + } + const url = new URL("https://do/subscribe"); + if (opts.did) url.searchParams.set("did", opts.did); + if (opts.sinceTs && opts.sinceTs > 0) { + url.searchParams.set("sinceTs", String(opts.sinceTs)); + } + return this.stub(topic).fetch(url.toString(), { + method: "GET", + headers + }); + } +} + +function pullIterator( + stub: DurableObjectStub, + signal?: AbortSignal +): AsyncIterator<RealtimeEvent> { + let reader: ReadableStreamDefaultReader<Uint8Array> | null = null; + let buf = ""; + const decoder = new TextDecoder(); + const ac = new AbortController(); + if (signal) { + if (signal.aborted) ac.abort(); + else signal.addEventListener("abort", () => ac.abort(), { once: true }); + } + + const open = async () => { + const res = await stub.fetch("https://do/subscribe", { + method: "GET", + headers: { accept: "text/event-stream" }, + signal: ac.signal, + }); + if (!res.ok || !res.body) throw new Error(`DO subscribe failed (${res.status})`); + reader = res.body.getReader(); + }; + + return { + async next(): Promise<IteratorResult<RealtimeEvent>> { + if (!reader) await open(); + while (true) { + // Drain buffered frames. + while (true) { + const sep = buf.indexOf("\n\n"); + if (sep < 0) break; + const frame = buf.slice(0, sep); + buf = buf.slice(sep + 2); + let data: string | null = null; + for (const line of frame.split("\n")) { + if (line.startsWith(":")) continue; + if (line.startsWith("data:")) data = line.slice(5).trim(); + } + if (data) { + try { + return { value: JSON.parse(data) as RealtimeEvent, done: false }; + } catch { + /* skip malformed */ + } + } + } + if (ac.signal.aborted) return { value: undefined, done: true }; + const r = await reader!.read(); + if (r.done) return { value: undefined, done: true }; + buf += decoder.decode(r.value, { stream: true }); + } + }, + async return(): Promise<IteratorResult<RealtimeEvent>> { + ac.abort(); + try { + await reader?.cancel(); + } catch { + /* ignore */ + } + return { value: undefined, done: true }; + }, + }; +} + +// ---------------------------------------------------------------------------- +// Durable Object class +// ---------------------------------------------------------------------------- + +/** The Durable Object implementation. Each DO instance owns the fan-out for + * exactly one topic. WebSocket connections are stored via the Hibernation + * API (`state.acceptWebSocket`) so idle rooms cost near-zero. + * + * This class intentionally avoids the `DurableObject` base class so we don't + * have to depend on @cloudflare/workers-types at the library level — users + * wire it up directly in their Worker entry. */ +/** Rolling buffer of recent events, used to close the snapshot→WS race: + * when a new subscriber connects with `?sinceTs=X`, replay any buffered + * event with `event.ts > X` before going live. Bounded by count + age so + * memory stays small. */ +const RECENT_BUFFER_MS = 15_000; +const RECENT_BUFFER_MAX = 500; + +export class RealtimePubSubDO { + private readonly recentEvents: RealtimeEvent[] = []; + + constructor( + protected readonly state: DurableObjectState, + _env?: unknown + ) {} + + private pushRecent(event: RealtimeEvent): void { + this.recentEvents.push(event); + const cutoff = Date.now() - RECENT_BUFFER_MS; + while ( + this.recentEvents.length > RECENT_BUFFER_MAX || + (this.recentEvents.length > 0 && this.recentEvents[0]!.ts < cutoff) + ) { + this.recentEvents.shift(); + } + } + + /** Worker entry delegates `fetch` to this method. */ + async fetch(request: Request): Promise<Response> { + const url = new URL(request.url); + if (request.method === "POST" && url.pathname === "/publish") { + let event: RealtimeEvent; + try { + event = (await request.json()) as RealtimeEvent; + } catch { + return new Response(JSON.stringify({ error: "InvalidRequest" }), { status: 400 }); + } + this.publishEvent(event); + return new Response("{}", { status: 200 }); + } + if (request.method === "GET" && url.pathname === "/subscribe") { + const did = url.searchParams.get("did") ?? undefined; + const sinceTsRaw = url.searchParams.get("sinceTs"); + const sinceTs = sinceTsRaw ? Number(sinceTsRaw) : 0; + // Optional query-filter spec, forwarded by the Worker after it has + // verified the caller's auth + access. Parsed once here; the parsed + // object is serialized into the WS attachment so the DO can filter + // events on publish without re-parsing. + let querySpec: SubscriberQuerySpec | undefined; + const rawSpec = request.headers.get("X-Contrail-Query-Spec"); + if (rawSpec) { + try { + querySpec = JSON.parse(rawSpec) as SubscriberQuerySpec; + } catch { + return new Response( + JSON.stringify({ error: "InvalidRequest", message: "bad X-Contrail-Query-Spec" }), + { status: 400 } + ); + } + } + + if (request.headers.get("Upgrade")?.toLowerCase() === "websocket") { + const Pair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; + if (!Pair) return new Response("websockets require Workers", { status: 426 }); + const pair = new Pair(); + this.acceptWebSocketSubscriber(pair[1], did, querySpec); + if (sinceTs > 0) this.replayRecentTo(pair[1], sinceTs); + return new Response(null, { + status: 101, + // Workers-specific init field + webSocket: pair[0], + } as ResponseInit & { webSocket: unknown }); + } + return this.openSseResponse(did, querySpec, sinceTs); + } + return new Response("not found", { status: 404 }); + } + + /** Fan-out an event to every connected subscriber (WS + SSE). + * Public so tests + advanced callers can skip the HTTP layer. + * + * If a subscriber has attached a `querySpec`, we translate the raw event + * into 0–1 watchRecords-shaped events (record.created, record.deleted, + * hydration.added, hydration.removed) and deliver only those. Otherwise + * the raw event is delivered as-is (topic-firehose behaviour for the + * `realtime.subscribe` endpoint). */ + publishEvent(event: RealtimeEvent): void { + // Buffer first so a subscriber connecting mid-publish (race-window + // replay) can pick up this event too once they provide their sinceTs. + this.pushRecent(event); + + const rawPayload = JSON.stringify(event); + const rawFrame = `event: ${event.kind}\ndata: ${rawPayload}\n\n`; + + for (const ws of this.state.getWebSockets()) { + const attachment = getAttachment(ws); + + if (attachment?.querySpec) { + const translated = translateForQuery(event, attachment); + if (translated) this.writeSubscriberState(ws, attachment, translated); + for (const msg of translated ?? []) { + try { + ws.send(JSON.stringify(msg)); + } catch { + /* ignore */ + } + } + } else { + try { + ws.send(rawPayload); + } catch { + /* ignore */ + } + } + + if ( + event.kind === "member.removed" && + attachment?.did && + event.payload.did === attachment.did + ) { + try { + ws.close(4003, "membership-revoked"); + } catch { + /* ignore */ + } + } + } + + for (const entry of this.sseControllers) { + if (entry.querySpec) { + const translated = translateForQuery(event, entry); + if (translated) this.writeSubscriberStateForSse(entry, translated); + for (const msg of translated ?? []) { + try { + entry.controller.enqueue( + this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) + ); + } catch { + /* drop */ + } + } + } else { + try { + entry.controller.enqueue(this.encoder.encode(rawFrame)); + } catch { + /* drop; cleanup happens on the subscribe-side */ + } + } + + if ( + event.kind === "member.removed" && + entry.did && + event.payload.did === entry.did + ) { + try { + entry.controller.close(); + } catch { + /* ignore */ + } + } + } + } + + /** Register a server-side WebSocket as a subscriber. Wires the DID + + * optional query spec into the hibernation attachment so this DO can + * filter and route events after going to sleep. */ + acceptWebSocketSubscriber( + serverWs: any, + did?: string, + querySpec?: SubscriberQuerySpec + ): void { + this.state.acceptWebSocket(serverWs, did ? [did] : undefined); + if (did || querySpec) { + setAttachment(serverWs, { + did, + querySpec, + parentUris: [], + childToParent: {} + }); + } + } + + /** Open an SSE subscriber; returns the streaming Response. */ + openSseResponse( + did?: string, + querySpec?: SubscriberQuerySpec, + sinceTs = 0 + ): Response { + let entry: SseEntry; + const stream = new ReadableStream<Uint8Array>({ + start: (controller) => { + entry = { + controller, + did, + querySpec, + parentUris: new Set(), + childToParent: new Map() + }; + this.sseControllers.add(entry); + controller.enqueue(this.encoder.encode(`: open\n\n`)); + if (sinceTs > 0) this.replayRecentToSse(entry, sinceTs); + }, + cancel: () => { + this.sseControllers.delete(entry); + }, + }); + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + }, + }); + } + + /** Replay buffered events with ts > sinceTs through this subscriber's + * query-spec filter. Called once, synchronously, on WS connect. */ + private replayRecentTo(ws: any, sinceTs: number): void { + const attachment = getAttachment(ws); + for (const event of this.recentEvents) { + if (event.ts <= sinceTs) continue; + if (attachment?.querySpec) { + const translated = translateForQuery(event, attachment); + if (translated) this.writeSubscriberState(ws, attachment, translated); + for (const msg of translated ?? []) { + try { + ws.send(JSON.stringify(msg)); + } catch { + /* ignore */ + } + } + } else { + try { + ws.send(JSON.stringify(event)); + } catch { + /* ignore */ + } + } + } + } + + private replayRecentToSse(entry: SseEntry, sinceTs: number): void { + for (const event of this.recentEvents) { + if (event.ts <= sinceTs) continue; + if (entry.querySpec) { + const translated = translateForQuery(event, entry); + if (translated) this.writeSubscriberStateForSse(entry, translated); + for (const msg of translated ?? []) { + try { + entry.controller.enqueue( + this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) + ); + } catch { + /* drop */ + } + } + } else { + try { + entry.controller.enqueue( + this.encoder.encode(`event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`) + ); + } catch { + /* drop */ + } + } + } + } + + /** Update the persisted WS attachment state after we've decided which + * events to forward. Keeps the parent/child tracking tables warm across + * hibernation. */ + private writeSubscriberState( + ws: any, + attachment: WsAttachment, + translated: TranslatedEvent[] + ): void { + let dirty = false; + for (const msg of translated) { + if (msg.kind === "record.created" && msg.data.record?.uri) { + attachment.parentUris = Array.from( + new Set([...(attachment.parentUris ?? []), msg.data.record.uri]) + ); + dirty = true; + } else if (msg.kind === "record.deleted" && msg.data.uri) { + const before = attachment.parentUris ?? []; + attachment.parentUris = before.filter((u) => u !== msg.data.uri); + if (attachment.parentUris.length !== before.length) dirty = true; + } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { + attachment.childToParent = { + ...(attachment.childToParent ?? {}), + [msg.data.child.rkey]: { + parentUri: msg.data.parentUri, + relName: msg.data.relation + } + }; + dirty = true; + } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { + const next = { ...(attachment.childToParent ?? {}) }; + if (next[msg.data.childRkey]) { + delete next[msg.data.childRkey]; + attachment.childToParent = next; + dirty = true; + } + } + } + if (dirty) setAttachment(ws, attachment); + } + + private writeSubscriberStateForSse( + entry: SseEntry, + translated: TranslatedEvent[] + ): void { + for (const msg of translated) { + if (msg.kind === "record.created" && msg.data.record?.uri) { + entry.parentUris?.add(msg.data.record.uri); + } else if (msg.kind === "record.deleted" && msg.data.uri) { + entry.parentUris?.delete(msg.data.uri); + } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { + entry.childToParent?.set(msg.data.child.rkey, { + parentUri: msg.data.parentUri, + relName: msg.data.relation + }); + } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { + entry.childToParent?.delete(msg.data.childRkey); + } + } + } + + private readonly sseControllers = new Set<SseEntry>(); + private readonly encoder = new TextEncoder(); +} + +interface SseEntry { + controller: ReadableStreamDefaultController<Uint8Array>; + did: string | undefined; + querySpec?: SubscriberQuerySpec; + parentUris?: Set<string>; + childToParent?: Map<string, { parentUri: string; relName: string }>; +} + +interface WsAttachment { + did?: string; + querySpec?: SubscriberQuerySpec; + /** URIs of primary records currently in this subscriber's result set. */ + parentUris?: string[]; + /** childRkey → parent info, for routing child delete events. */ + childToParent?: Record<string, { parentUri: string; relName: string }>; +} + +function setAttachment(ws: any, attachment: WsAttachment): void { + try { + ws.serializeAttachment?.(attachment); + } catch { + /* non-hibernating socket — fall back to a direct property */ + ws.__attachment = attachment; + } +} + +function getAttachment(ws: any): WsAttachment | null { + try { + const a = ws.deserializeAttachment?.(); + if (a) return a as WsAttachment; + } catch { + /* ignore */ + } + return (ws.__attachment as WsAttachment | undefined) ?? null; +} + +// Query-spec filtering lives in ./query-filter so the Worker can reuse it for +// non-DO (InMemoryPubSub) watchRecords paths without bundling the whole DO. diff --git a/packages/contrail-base/src/realtime/in-memory.ts b/packages/contrail-base/src/realtime/in-memory.ts new file mode 100644 index 0000000..311c0a5 --- /dev/null +++ b/packages/contrail-base/src/realtime/in-memory.ts @@ -0,0 +1,116 @@ +import type { PubSub, RealtimeEvent } from "./types"; +import { DEFAULT_QUEUE_BOUND } from "./types"; + +/** Single-process PubSub backed by in-memory subscriber sets. + * + * Each subscriber owns a bounded queue; when full, oldest events are dropped. + * `publish` returns once every subscriber has been offered the event — it + * never awaits a subscriber's consumption, so a slow consumer can't block + * producers. The cost of that guarantee is the drop-oldest policy. */ +export class InMemoryPubSub implements PubSub { + private readonly subscribers = new Map<string, Set<Subscriber>>(); + private readonly queueBound: number; + + constructor(opts: { queueBound?: number } = {}) { + this.queueBound = opts.queueBound ?? DEFAULT_QUEUE_BOUND; + } + + async publish(event: RealtimeEvent): Promise<void> { + const set = this.subscribers.get(event.topic); + if (!set) return; + for (const sub of set) sub.push(event); + } + + subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { + const sub = new Subscriber(this.queueBound); + let set = this.subscribers.get(topic); + if (!set) { + set = new Set(); + this.subscribers.set(topic, set); + } + set.add(sub); + + const cleanup = () => { + sub.close(); + const s = this.subscribers.get(topic); + if (s) { + s.delete(sub); + if (s.size === 0) this.subscribers.delete(topic); + } + }; + + if (signal) { + if (signal.aborted) cleanup(); + else signal.addEventListener("abort", cleanup, { once: true }); + } + + return sub.iterate(cleanup); + } + + /** Test-only: current subscriber count for a topic. */ + subscriberCount(topic: string): number { + return this.subscribers.get(topic)?.size ?? 0; + } +} + +class Subscriber { + private readonly queue: RealtimeEvent[] = []; + private pending: ((v: RealtimeEvent | null) => void) | null = null; + private closed = false; + /** Number of events dropped because the queue was full. The consumer can + * observe a gap by comparing monotonic event timestamps; exposing the + * count on a side channel is future work. */ + public droppedCount = 0; + + constructor(private readonly bound: number) {} + + push(event: RealtimeEvent): void { + if (this.closed) return; + if (this.pending) { + const p = this.pending; + this.pending = null; + p(event); + return; + } + if (this.queue.length >= this.bound) { + this.queue.shift(); + this.droppedCount += 1; + } + this.queue.push(event); + } + + close(): void { + if (this.closed) return; + this.closed = true; + if (this.pending) { + const p = this.pending; + this.pending = null; + p(null); + } + } + + iterate(cleanup: () => void): AsyncIterable<RealtimeEvent> { + const self = this; + return { + [Symbol.asyncIterator]() { + return { + async next(): Promise<IteratorResult<RealtimeEvent>> { + if (self.queue.length > 0) { + return { value: self.queue.shift()!, done: false }; + } + if (self.closed) return { value: undefined, done: true }; + const event = await new Promise<RealtimeEvent | null>((resolve) => { + self.pending = resolve; + }); + if (event === null) return { value: undefined, done: true }; + return { value: event, done: false }; + }, + async return(): Promise<IteratorResult<RealtimeEvent>> { + cleanup(); + return { value: undefined, done: true }; + }, + }; + }, + }; + } +} diff --git a/packages/contrail-base/src/realtime/merge.ts b/packages/contrail-base/src/realtime/merge.ts new file mode 100644 index 0000000..e0fa7df --- /dev/null +++ b/packages/contrail-base/src/realtime/merge.ts @@ -0,0 +1,77 @@ +/** Merge N AsyncIterables into one, interleaving events as they arrive. + * Terminates when every source terminates, or when `signal` aborts. */ + +export function mergeAsyncIterables<T>( + sources: AsyncIterable<T>[], + signal?: AbortSignal +): AsyncIterable<T> { + if (sources.length === 0) { + return { + async *[Symbol.asyncIterator]() { + /* nothing to yield */ + }, + }; + } + + return { + [Symbol.asyncIterator]() { + const iterators = sources.map((s) => s[Symbol.asyncIterator]()); + // One in-flight next() per source, racing each other. + type Slot = { + idx: number; + promise: Promise<{ idx: number; result: IteratorResult<T> }>; + }; + const pending = new Map<number, Slot>(); + let doneCount = 0; + + const schedule = (idx: number) => { + const slot: Slot = { + idx, + promise: iterators[idx]! + .next() + .then((result) => ({ idx, result })), + }; + pending.set(idx, slot); + }; + + for (let i = 0; i < iterators.length; i++) schedule(i); + + const cleanup = () => { + for (const it of iterators) { + try { + it.return?.(); + } catch { + /* ignore */ + } + } + }; + + if (signal) { + if (signal.aborted) cleanup(); + else signal.addEventListener("abort", cleanup, { once: true }); + } + + return { + async next(): Promise<IteratorResult<T>> { + while (pending.size > 0) { + const slots = [...pending.values()]; + const { idx, result } = await Promise.race(slots.map((s) => s.promise)); + pending.delete(idx); + if (result.done) { + doneCount += 1; + if (doneCount === iterators.length) return { value: undefined, done: true }; + continue; + } + schedule(idx); + return { value: result.value, done: false }; + } + return { value: undefined, done: true }; + }, + async return(): Promise<IteratorResult<T>> { + cleanup(); + return { value: undefined, done: true }; + }, + }; + }, + }; +} diff --git a/packages/contrail-base/src/realtime/query-filter.ts b/packages/contrail-base/src/realtime/query-filter.ts new file mode 100644 index 0000000..1b79eb6 --- /dev/null +++ b/packages/contrail-base/src/realtime/query-filter.ts @@ -0,0 +1,235 @@ +/** Shared query-spec → event-translation logic. + * + * Used by: + * - the Durable Object's WS publish path (per-subscriber filter after hibernation) + * - the Worker's SSE / Worker-terminated WS path (in-process filter) + * + * Given a raw RealtimeEvent and a SubscriberQuerySpec, returns the list of + * `{kind, data}` envelopes to send to the subscriber, or `null` if the + * subscriber has no spec (i.e. raw-firehose mode). */ + +import type { RealtimeEvent } from "./types"; +import type { SubscriberQuerySpec } from "./durable-object"; + +export type TranslatedEnvelope = + | { + kind: "record.created"; + data: { + record: { + uri: string; + did: string; + rkey: string; + collection: string; + cid: string | null | undefined; + record: Record<string, unknown>; + time_us: number; + indexed_at: number; + space: string; + }; + }; + } + | { kind: "record.deleted"; data: { uri: string; did: string; rkey: string } } + | { + kind: "hydration.added"; + data: { + parentUri: string; + relation: string; + child: { + uri: string; + did: string; + rkey: string; + collection: string; + cid: string | null | undefined; + record: Record<string, unknown>; + space: string; + }; + }; + } + | { + kind: "hydration.removed"; + data: { + parentUri: string; + relation: string; + childRkey: string; + childDid?: string; + }; + }; + +export interface SubscriberView { + querySpec?: SubscriberQuerySpec; + parentUris?: Set<string> | string[]; + childToParent?: + | Map<string, { parentUri: string; relName: string }> + | Record<string, { parentUri: string; relName: string }>; +} + +export function translateForQuery( + event: RealtimeEvent, + sub: SubscriberView +): TranslatedEnvelope[] | null { + const spec = sub.querySpec; + if (!spec) return null; + if (event.kind !== "record.created" && event.kind !== "record.deleted") return []; + if (event.payload.space !== spec.spaceUri) return []; + + const primaryUri = `at://${event.payload.did}/${event.payload.collection}/${event.payload.rkey}`; + + if (event.payload.collection === spec.collection) { + if (event.kind === "record.created") { + return [ + { + kind: "record.created", + data: { + record: { + uri: primaryUri, + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + record: event.payload.record, + time_us: event.ts * 1000, + indexed_at: event.ts, + space: spec.spaceUri + } + } + } + ]; + } + return [ + { + kind: "record.deleted", + data: { + uri: primaryUri, + did: event.payload.did, + rkey: event.payload.rkey + } + } + ]; + } + + if (!spec.hydrate) return []; + for (const [relName, rel] of Object.entries(spec.hydrate)) { + if (rel.childCollection !== event.payload.collection) continue; + if (event.kind === "record.created") { + const parentUri = getNestedValue( + event.payload.record as Record<string, unknown>, + rel.matchField + ); + if (typeof parentUri !== "string") continue; + if (!hasParent(sub.parentUris, parentUri)) continue; + return [ + { + kind: "hydration.added", + data: { + parentUri, + relation: relName, + child: { + uri: primaryUri, + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + record: event.payload.record, + space: spec.spaceUri + } + } + } + ]; + } + const info = getChildInfo(sub.childToParent, event.payload.rkey); + if (!info || info.relName !== relName) continue; + return [ + { + kind: "hydration.removed", + data: { + parentUri: info.parentUri, + relation: relName, + childRkey: event.payload.rkey, + childDid: event.payload.did + } + } + ]; + } + return []; +} + +export function applyEnvelopesToSubscriber( + subscriber: SubscriberView, + envs: TranslatedEnvelope[] +): void { + for (const msg of envs) { + if (msg.kind === "record.created") { + ensureParentSet(subscriber).add(msg.data.record.uri); + } else if (msg.kind === "record.deleted") { + const set = subscriber.parentUris; + if (set instanceof Set) set.delete(msg.data.uri); + else if (Array.isArray(set)) { + const idx = set.indexOf(msg.data.uri); + if (idx >= 0) set.splice(idx, 1); + } + } else if (msg.kind === "hydration.added") { + ensureChildMap(subscriber).set(msg.data.child.rkey, { + parentUri: msg.data.parentUri, + relName: msg.data.relation + }); + } else if (msg.kind === "hydration.removed") { + const map = subscriber.childToParent; + if (map instanceof Map) map.delete(msg.data.childRkey); + else if (map && typeof map === "object") { + delete (map as Record<string, unknown>)[msg.data.childRkey]; + } + } + } +} + +function ensureParentSet(sub: SubscriberView): Set<string> { + if (sub.parentUris instanceof Set) return sub.parentUris; + const set = new Set<string>(sub.parentUris ?? []); + sub.parentUris = set; + return set; +} + +function ensureChildMap( + sub: SubscriberView +): Map<string, { parentUri: string; relName: string }> { + if (sub.childToParent instanceof Map) return sub.childToParent; + const map = new Map<string, { parentUri: string; relName: string }>(); + if (sub.childToParent && typeof sub.childToParent === "object") { + for (const [k, v] of Object.entries(sub.childToParent)) map.set(k, v); + } + sub.childToParent = map; + return map; +} + +function hasParent( + parents: Set<string> | string[] | undefined, + uri: string +): boolean { + if (!parents) return false; + if (parents instanceof Set) return parents.has(uri); + return parents.includes(uri); +} + +function getChildInfo( + map: + | Map<string, { parentUri: string; relName: string }> + | Record<string, { parentUri: string; relName: string }> + | undefined, + rkey: string +): { parentUri: string; relName: string } | undefined { + if (!map) return undefined; + if (map instanceof Map) return map.get(rkey); + return map[rkey]; +} + +function getNestedValue( + obj: Record<string, unknown>, + path: string +): unknown { + let cur: unknown = obj; + for (const key of path.split(".")) { + if (cur == null || typeof cur !== "object") return undefined; + cur = (cur as Record<string, unknown>)[key]; + } + return cur; +} diff --git a/packages/contrail-base/src/realtime/sse.ts b/packages/contrail-base/src/realtime/sse.ts new file mode 100644 index 0000000..65d89c1 --- /dev/null +++ b/packages/contrail-base/src/realtime/sse.ts @@ -0,0 +1,98 @@ +/** Server-Sent Events transport. + * + * Wraps an AsyncIterable<RealtimeEvent> as a streaming Response. The caller + * (the router) has already done auth and has an AbortSignal it can use to + * tear the stream down (e.g. on `member.removed` for the subscriber's DID). */ + +import type { RealtimeEvent } from "./types"; +import { DEFAULT_KEEPALIVE_MS } from "./types"; + +export interface SseOptions { + keepaliveMs?: number; + /** Called before the stream closes. Useful for cleanup that the caller + * can't do via the signal (e.g. removing a subscriber from a set). */ + onClose?: () => void; +} + +export function sseResponse( + iter: AsyncIterable<RealtimeEvent>, + signal: AbortSignal, + opts: SseOptions = {} +): Response { + const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + const encoder = new TextEncoder(); + + const stream = new ReadableStream<Uint8Array>({ + start(controller) { + let closed = false; + let keepalive: ReturnType<typeof setInterval> | null = null; + + const close = () => { + if (closed) return; + closed = true; + if (keepalive) clearInterval(keepalive); + try { + controller.close(); + } catch { + /* already closed */ + } + opts.onClose?.(); + }; + + signal.addEventListener("abort", close, { once: true }); + + keepalive = setInterval(() => { + if (closed) return; + try { + controller.enqueue(encoder.encode(`: keepalive\n\n`)); + } catch { + close(); + } + }, keepaliveMs); + + (async () => { + // Opening comment — helps some clients / proxies initialize promptly. + controller.enqueue(encoder.encode(`: open\n\n`)); + try { + for await (const event of iter) { + if (closed) break; + controller.enqueue(encoder.encode(frameEvent(event))); + } + } catch (err) { + if (!closed) { + try { + controller.enqueue( + encoder.encode( + `event: error\ndata: ${JSON.stringify({ + message: err instanceof Error ? err.message : String(err), + })}\n\n` + ) + ); + } catch { + /* stream already torn down */ + } + } + } finally { + close(); + } + })(); + }, + cancel() { + opts.onClose?.(); + }, + }); + + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + }, + }); +} + +function frameEvent(event: RealtimeEvent): string { + return `event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`; +} diff --git a/packages/contrail-base/src/realtime/ticket.ts b/packages/contrail-base/src/realtime/ticket.ts new file mode 100644 index 0000000..1c6e3de --- /dev/null +++ b/packages/contrail-base/src/realtime/ticket.ts @@ -0,0 +1,179 @@ +/** Subscription tickets — HMAC-signed short-lived `{topics, did, exp}` blobs. + * + * Wire format: `<payload>.<sig>` where + * payload = base64url(JSON({ topics, did, exp, iat })) + * sig = base64url(HMAC-SHA256(key, payload)) + * + * Tickets are integrity-only (not encrypted). Browsers use them because + * EventSource / WebSocket can't send Authorization headers; server-side + * consumers skip the ticket dance and send their JWT directly. */ + +export interface TicketPayload { + /** Concrete delivery topics this ticket authorizes. `community:<did>` is + * expanded to the caller's visible spaces before signing — a ticket never + * carries a community alias. */ + topics: string[]; + did: string; + /** Unix ms. */ + exp: number; + /** Unix ms — useful for debugging; ignored on verify. */ + iat: number; + /** Optional: query-scoped watchRecords spec this ticket authorizes. Present + * when the ticket was minted from a watchRecords handshake. The server + * trusts the signed spec on upgrade and forwards it to the DO. */ + querySpec?: TicketQuerySpec; +} + +export interface TicketQuerySpec { + collection: string; + /** Exactly one of `spaceUri` or `actor` is set. `spaceUri` = per-space + * watch; `actor` = cross-space watch for records authored by this DID + * (the ticket's `topics` list carries the expanded delivery topics). */ + spaceUri?: string; + actor?: string; + hydrate?: Record<string, { childCollection: string; matchField: string }>; +} + +function normalizeSecret(secret: Uint8Array | string): Uint8Array { + if (typeof secret !== "string") { + if (secret.length !== 32) { + throw new Error(`realtime ticketSecret must be 32 bytes, got ${secret.length}`); + } + return secret; + } + // 64 hex chars would also round-trip as base64 (to 48 bytes). Prefer hex + // when the input matches the hex alphabet exactly; fall back to base64. + const hex = tryHex(secret); + if (hex && hex.length === 32) return hex; + const b64 = tryBase64(secret); + if (b64 && b64.length === 32) return b64; + if (hex || b64) { + const got = (hex ?? b64)!.length; + throw new Error(`realtime ticketSecret must decode to 32 bytes, got ${got}`); + } + throw new Error("realtime ticketSecret must be a 32-byte Uint8Array or base64/hex string"); +} + +function tryBase64(s: string): Uint8Array | null { + try { + const normal = s.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); + if (!/^[A-Za-z0-9+/]*=*$/.test(padded)) return null; + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; + } catch { + return null; + } +} + +function tryHex(s: string): Uint8Array | null { + if (!/^[0-9a-fA-F]+$/.test(s) || s.length % 2 !== 0) return null; + const out = new Uint8Array(s.length / 2); + for (let i = 0; i < out.length; i++) { + out[i] = parseInt(s.slice(i * 2, i * 2 + 2), 16); + } + return out; +} + +function b64urlFromBytes(bytes: Uint8Array): string { + let bin = ""; + for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); + return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +function b64urlToBytes(s: string): Uint8Array { + const normal = s.replace(/-/g, "+").replace(/_/g, "/"); + const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function b64urlFromString(s: string): string { + return b64urlFromBytes(new TextEncoder().encode(s)); +} + +function stringFromB64url(s: string): string { + return new TextDecoder().decode(b64urlToBytes(s)); +} + +function constantTimeEq(a: Uint8Array, b: Uint8Array): boolean { + if (a.length !== b.length) return false; + let diff = 0; + for (let i = 0; i < a.length; i++) diff |= a[i]! ^ b[i]!; + return diff === 0; +} + +export class TicketSigner { + private readonly keyPromise: Promise<CryptoKey>; + + constructor(secret: Uint8Array | string) { + const raw = normalizeSecret(secret); + this.keyPromise = crypto.subtle.importKey( + "raw", + raw as BufferSource, + { name: "HMAC", hash: "SHA-256" }, + false, + ["sign", "verify"] + ); + } + + async sign(input: { + topics: string[]; + did: string; + ttlMs: number; + querySpec?: TicketQuerySpec; + }): Promise<string> { + const now = Date.now(); + const payload: TicketPayload = { + topics: input.topics, + did: input.did, + exp: now + input.ttlMs, + iat: now, + ...(input.querySpec ? { querySpec: input.querySpec } : {}), + }; + const payloadPart = b64urlFromString(JSON.stringify(payload)); + const sig = await crypto.subtle.sign( + "HMAC", + await this.keyPromise, + new TextEncoder().encode(payloadPart) as BufferSource + ); + const sigPart = b64urlFromBytes(new Uint8Array(sig)); + return `${payloadPart}.${sigPart}`; + } + + /** Returns the decoded payload if the ticket is valid + unexpired, else null. */ + async verify(ticket: string): Promise<TicketPayload | null> { + const dot = ticket.indexOf("."); + if (dot < 0) return null; + const payloadPart = ticket.slice(0, dot); + const sigPart = ticket.slice(dot + 1); + let expectedSig: Uint8Array; + try { + expectedSig = b64urlToBytes(sigPart); + } catch { + return null; + } + const computedRaw = await crypto.subtle.sign( + "HMAC", + await this.keyPromise, + new TextEncoder().encode(payloadPart) as BufferSource + ); + const computed = new Uint8Array(computedRaw); + if (!constantTimeEq(expectedSig, computed)) return null; + let parsed: TicketPayload; + try { + parsed = JSON.parse(stringFromB64url(payloadPart)); + } catch { + return null; + } + if (!parsed || !Array.isArray(parsed.topics) || typeof parsed.did !== "string") { + return null; + } + if (typeof parsed.exp !== "number" || parsed.exp <= Date.now()) return null; + return parsed; + } +} diff --git a/packages/contrail-base/src/realtime/types.ts b/packages/contrail-base/src/realtime/types.ts new file mode 100644 index 0000000..05a246e --- /dev/null +++ b/packages/contrail-base/src/realtime/types.ts @@ -0,0 +1,127 @@ +/** Realtime module — canonical types + interfaces. See docs/realtime.md. */ + +/** Discriminated union of every event kind that flows through the PubSub. + * + * `record.created` carries the full record body so a subscriber can apply an + * insert or upsert without a follow-up `getRecord` call. Writing a new record + * to the same `(did, collection, rkey)` publishes another `record.created` — + * treat it as upsert. + * + * **Payload shape mirrors `listRecords` output** (`uri`, `did`, `space?`, + * `time_us`), so a subscriber can render a live row the same way it renders + * a fetched row. + * + * **Publisher/topic matrix (intentional trust split):** + * - `collection:<nsid>` and `actor:<did>` carry *public* record events only + * (from jetstream ingestion) — no `space`. + * - `space:<uri>` and `community:<did>` carry *space* events — `space` is + * always set. Never cross-published to public topics (privacy). */ +export type RealtimeEvent = + | { + topic: string; + kind: "record.created"; + payload: { + uri: string; + did: string; + collection: string; + rkey: string; + cid: string | null; + record: Record<string, unknown>; + time_us: number; + /** Present only for space records; absent for public records. */ + space?: string; + }; + ts: number; + } + | { + topic: string; + kind: "record.deleted"; + payload: { + uri: string; + did: string; + collection: string; + rkey: string; + /** Present only for space records; absent for public records. */ + space?: string; + }; + ts: number; + } + | { + topic: string; + kind: "member.added"; + payload: { space: string; did: string }; + ts: number; + } + | { + topic: string; + kind: "member.removed"; + payload: { space: string; did: string }; + ts: number; + }; + +export type RealtimeEventKind = RealtimeEvent["kind"]; + +/** Core pubsub abstraction. Implementations: InMemoryPubSub, DurableObjectPubSub. */ +export interface PubSub { + publish(event: RealtimeEvent): Promise<void>; + /** Stream events on the topic until the signal aborts (or the iterator is + * returned/broken out of). Implementations use a bounded per-subscriber + * queue with drop-oldest semantics — a slow subscriber can't stall publishers. */ + subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent>; +} + +// ---- Canonical topic strings ----------------------------------------------- +// `community:<did>` is an alias resolved at ticket-mint time to the concrete +// set of `space:<uri>` topics the caller can see; it is never a real delivery +// topic. The other three are real. + +export function spaceTopic(uri: string): string { + return `space:${uri}`; +} + +export function communityTopic(did: string): string { + return `community:${did}`; +} + +export function collectionTopic(nsid: string): string { + return `collection:${nsid}`; +} + +export function actorTopic(did: string): string { + return `actor:${did}`; +} + +export function isCommunityTopic(topic: string): boolean { + return topic.startsWith("community:"); +} + +export function parseCommunityTopic(topic: string): string | null { + return isCommunityTopic(topic) ? topic.slice("community:".length) : null; +} + +export function parseSpaceTopic(topic: string): string | null { + return topic.startsWith("space:") ? topic.slice("space:".length) : null; +} + +// ---- Config ----------------------------------------------------------------- + +export interface RealtimeConfig { + /** Backing pubsub. Default: new InMemoryPubSub() (single-process only). On + * Workers, pass `new DurableObjectPubSub(env.REALTIME)`. */ + pubsub?: PubSub; + /** HMAC secret used to sign subscription tickets. 32 bytes. Accepts raw + * Uint8Array or base64 / hex string. Envelope-encrypts nothing — tickets + * are integrity-only, not confidential. */ + ticketSecret: Uint8Array | string; + /** Ticket lifetime in ms. Default 120_000 (2 minutes). */ + ticketTtlMs?: number; + /** SSE/WS keepalive interval in ms. Default 15_000. */ + keepaliveMs?: number; + /** Per-subscriber queue bound. Default 1024. Events beyond this are dropped + * oldest-first and the subscriber receives a `lag` signal (out of band). */ + queueBound?: number; +} + +export const DEFAULT_TICKET_TTL_MS = 120_000; +export const DEFAULT_KEEPALIVE_MS = 15_000; +export const DEFAULT_QUEUE_BOUND = 1024; diff --git a/packages/contrail-base/src/realtime/websocket.ts b/packages/contrail-base/src/realtime/websocket.ts new file mode 100644 index 0000000..833483d --- /dev/null +++ b/packages/contrail-base/src/realtime/websocket.ts @@ -0,0 +1,84 @@ +/** WebSocket transport. + * + * Accepts a new WebSocket connection (either via `WebSocketPair` on Workers + * or a platform-provided server-side socket) and pumps events to it from an + * AsyncIterable<RealtimeEvent>. Messages are UTF-8 JSON, one event per frame. + * + * Close codes (subset, RFC 6455 + app-custom): + * - 4001: server error pumping + * - 4003: membership revoked + * - 4008: ticket/auth invalid (used by the router, not here) + */ + +import type { RealtimeEvent } from "./types"; +import { DEFAULT_KEEPALIVE_MS } from "./types"; + +export interface WebSocketLike { + send(data: string): void; + close(code?: number, reason?: string): void; + addEventListener(type: "message" | "close" | "error", listener: (ev: any) => void): void; +} + +export interface WebSocketPumpOptions { + keepaliveMs?: number; + onClose?: () => void; +} + +/** Pump events from `iter` to `ws` until the signal aborts or the iter ends. + * Caller is responsible for having already accept()ed the socket. */ +export async function pumpWebSocket( + ws: WebSocketLike, + iter: AsyncIterable<RealtimeEvent>, + signal: AbortSignal, + opts: WebSocketPumpOptions = {} +): Promise<void> { + const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + let closed = false; + + const close = (code?: number, reason?: string) => { + if (closed) return; + closed = true; + try { + ws.close(code, reason); + } catch { + /* already closed */ + } + opts.onClose?.(); + }; + + ws.addEventListener("close", () => { + closed = true; + opts.onClose?.(); + }); + ws.addEventListener("error", () => { + closed = true; + opts.onClose?.(); + }); + signal.addEventListener("abort", () => close(1000, "aborted"), { once: true }); + + const keepalive = setInterval(() => { + if (closed) return; + try { + ws.send(JSON.stringify({ kind: "$keepalive" })); + } catch { + close(); + } + }, keepaliveMs); + + try { + for await (const event of iter) { + if (closed) break; + try { + ws.send(JSON.stringify(event)); + } catch { + close(4001, "send-failed"); + break; + } + } + } catch { + close(4001, "pump-error"); + } finally { + clearInterval(keepalive); + close(); + } +} diff --git a/packages/contrail-base/src/spaces/acl.ts b/packages/contrail-base/src/spaces/acl.ts new file mode 100644 index 0000000..6e0da0a --- /dev/null +++ b/packages/contrail-base/src/spaces/acl.ts @@ -0,0 +1,69 @@ +import type { AppPolicy, SpaceMemberRow, SpaceRow } from "./types"; + +export type AclOp = "read" | "write" | "delete"; + +export interface AclInput { + op: AclOp; + space: SpaceRow; + callerDid: string; + /** Membership row for the caller (or null). Owner does not require a row. */ + member: SpaceMemberRow | null; + /** OAuth client_id of the app calling on caller's behalf, for app policy checks. */ + clientId?: string; + /** For per-record ops (get/delete), the record's author DID. */ + targetAuthorDid?: string; +} + +export type AclResult = + | { allow: true } + | { allow: false; reason: AclDenyReason }; + +export type AclDenyReason = + | "not-member" + | "not-own-record" + | "app-not-allowed" + | "unknown-op"; + +/** Check whether the caller's app is permitted to act in this space. */ +export function checkAppPolicy( + appPolicy: AppPolicy | null, + clientId: string | undefined +): boolean { + if (!appPolicy) return true; // no policy = allow-all + const listed = clientId ? appPolicy.apps.includes(clientId) : false; + if (appPolicy.mode === "allow") return !listed; // apps[] is a denylist + return listed; // mode === "deny": apps[] is an allowlist +} + +const isOwner = (space: SpaceRow, did: string) => space.ownerDid === did; +const hasMember = (space: SpaceRow, member: SpaceMemberRow | null, did: string) => + isOwner(space, did) || member != null; + +/** Space-level access check. + * Membership = access. Any member (including owner) can read and write. + * Delete is scoped to the caller's own records — owners don't get a bypass. + * A random member can't nuke other people's records, and neither can the + * owner. To remove a non-author record, delete the whole space. */ +export function checkAccess(input: AclInput): AclResult { + if (!checkAppPolicy(input.space.appPolicy, input.clientId)) { + return { allow: false, reason: "app-not-allowed" }; + } + + if (input.op === "read" || input.op === "write") { + return hasMember(input.space, input.member, input.callerDid) + ? { allow: true } + : { allow: false, reason: "not-member" }; + } + + if (input.op === "delete") { + if (!hasMember(input.space, input.member, input.callerDid)) { + return { allow: false, reason: "not-member" }; + } + if (input.targetAuthorDid && input.targetAuthorDid !== input.callerDid) { + return { allow: false, reason: "not-own-record" }; + } + return { allow: true }; + } + + return { allow: false, reason: "unknown-op" }; +} diff --git a/packages/contrail-base/src/spaces/auth.ts b/packages/contrail-base/src/spaces/auth.ts new file mode 100644 index 0000000..4c1e817 --- /dev/null +++ b/packages/contrail-base/src/spaces/auth.ts @@ -0,0 +1,177 @@ +import type { Context, MiddlewareHandler } from "hono"; +import { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, + type DidDocumentResolver, +} from "@atcute/identity-resolver"; +import type { Did, Nsid } from "@atcute/lexicons"; +import type { AuthorityConfig } from "./types"; +import { readInProcess } from "./in-process"; + +export { ServiceJwtVerifier }; + +/** Build a ServiceJwtVerifier from an AuthorityConfig, using the configured + * resolver or a default PLC+Web composite. The verifier checks that incoming + * JWTs target this authority's serviceDid (aud claim). */ +export function buildVerifier(authority: AuthorityConfig): ServiceJwtVerifier { + const resolver = + authority.resolver ?? + new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, + }); + return new ServiceJwtVerifier({ + serviceDid: authority.serviceDid as Did, + resolver, + }); +} + +export interface ServiceAuth { + issuer: string; + audience: string; + lxm: string | undefined; + /** OAuth client_id of the caller, if the JWT carries one. */ + clientId?: string; +} + +export interface ServiceAuthOptions { + serviceDid: Did; + resolver: DidDocumentResolver; +} + +/** Hono middleware that authenticates XRPC requests. Order of precedence: + * 1. In-process marker (same-module calls; see `core/spaces/in-process.ts`) + * 2. Authorization: Bearer <JWT> as an atproto service-auth token + * + * On success, attaches the claims to `c.var.serviceAuth`. Expected Nsid is + * taken from the route pattern (last segment after `/xrpc/`). */ +export function createServiceAuthMiddleware( + verifier: ServiceJwtVerifier +): MiddlewareHandler { + return async (c, next) => { + const lxm = extractLxmFromPath(c); + + const inProcess = readInProcess(c.req.raw); + if (inProcess) { + c.set("serviceAuth", { + issuer: inProcess.did, + audience: "", + lxm: lxm ?? undefined, + } satisfies ServiceAuth); + await next(); + return; + } + + const header = c.req.header("Authorization"); + if (!header || !header.startsWith("Bearer ")) { + return c.json({ error: "AuthRequired", message: "Missing bearer token" }, 401); + } + const token = header.slice(7).trim(); + + const result = await verifier.verify(token, { lxm }); + if (!result.ok) { + const err = result.error as { error?: string; description?: string } | undefined; + return c.json( + { + error: "AuthRequired", + message: err?.description ?? err?.error ?? String(result.error), + }, + 401, + ); + } + + c.set("serviceAuth", { + issuer: result.value.issuer, + audience: result.value.audience, + lxm: result.value.lxm, + } satisfies ServiceAuth); + + await next(); + }; +} + +function extractLxmFromPath(c: Context): Nsid | null { + const path = new URL(c.req.url).pathname; + const match = path.match(/\/xrpc\/([a-zA-Z0-9.-]+)/); + return (match?.[1] as Nsid) ?? null; +} + +/** Read the service auth claims set by the middleware. Throws if unset. */ +export function requireServiceAuth(c: Context): ServiceAuth { + const auth = c.get("serviceAuth") as ServiceAuth | undefined; + if (!auth) throw new Error("service auth missing; middleware not attached"); + return auth; +} + +/** Out-of-band auth check for handlers that don't always require auth. + * Returns claims on success, or null if no valid credentials are present. + * Order of precedence: in-process marker → service-auth JWT. */ +export async function verifyServiceAuthRequest( + verifier: ServiceJwtVerifier, + request: Request, + lxm?: Nsid | null +): Promise<ServiceAuth | null> { + const inProcess = readInProcess(request); + if (inProcess) { + return { + issuer: inProcess.did, + audience: "", + lxm: lxm ?? undefined, + }; + } + + const header = request.headers.get("Authorization"); + if (!header || !header.startsWith("Bearer ")) return null; + const token = header.slice(7).trim(); + const result = await verifier.verify(token, { lxm: lxm ?? null }); + if (!result.ok) return null; + return { + issuer: result.value.issuer, + audience: result.value.audience, + lxm: result.value.lxm, + }; +} + +/** Pull a space credential off the request — `X-Space-Credential: <jwt>` + * header. Returns the raw token or null. */ +export function extractSpaceCredential(request: Request): string | null { + const header = request.headers.get("X-Space-Credential"); + return header ? header.trim() : null; +} + +/** Pull a read-grant invite token off the request — query string `?inviteToken=` + * or `Authorization: Bearer atmo-invite:<token>`. Returns the raw token (not + * hashed) or null. Routes hash + look up via the adapter. */ +export function extractInviteToken(request: Request): string | null { + const url = new URL(request.url); + const q = url.searchParams.get("inviteToken"); + if (q) return q.trim(); + const header = request.headers.get("Authorization"); + if (header?.startsWith("Bearer atmo-invite:")) { + return header.slice("Bearer atmo-invite:".length).trim(); + } + return null; +} + +/** Validate a read-grant invite token against a target spaceUri. Returns true + * if the token exists, scopes to this space, has a kind that grants read + * (`read` or `read-join`), and is not expired/revoked. */ +export async function checkInviteReadGrant( + adapter: { getInvite(tokenHash: string): Promise<{ spaceUri: string; kind: string; revokedAt: number | null; expiresAt: number | null } | null> }, + rawToken: string, + spaceUri: string, + hashFn: (token: string) => Promise<string> +): Promise<boolean> { + const tokenHash = await hashFn(rawToken); + const invite = await adapter.getInvite(tokenHash); + if (!invite) return false; + if (invite.spaceUri !== spaceUri) return false; + if (invite.kind !== "read" && invite.kind !== "read-join") return false; + if (invite.revokedAt != null) return false; + if (invite.expiresAt != null && invite.expiresAt <= Date.now()) return false; + return true; +} diff --git a/packages/contrail-base/src/spaces/binding.ts b/packages/contrail-base/src/spaces/binding.ts new file mode 100644 index 0000000..a81b95f --- /dev/null +++ b/packages/contrail-base/src/spaces/binding.ts @@ -0,0 +1,274 @@ +/** Binding resolution: given a space URI, which DID is authorized to sign + * credentials for it, and where do we find that DID's verification key? + * + * Two layers of pluggable resolvers compose into a credential verifier: + * + * BindingResolver — `ats://<owner>/<type>/<key>` → authority DID + * KeyResolver — (DID, kid) → JsonWebKey + * + * The BindingResolver is what makes user-owned-DID-with-PDS-record work: + * given a space URI, we resolve the owner's PDS, fetch the declaration + * record, and read its `authority` field. For provisioned (no-PDS) DIDs we + * fall back to the owner DID's `#atproto_space_authority` service entry. + * And finally for the trivial case (HappyView-style "owner self-issues"), + * we return the owner DID itself. + * + * See conversation history (phase 4 design) for the rationale on why these + * three sources, in this order. */ + +import type { DidDocumentResolver } from "@atcute/identity-resolver"; +import type { Did } from "@atcute/lexicons"; +import { parseSpaceUri } from "./uri"; +import type { RecordHost } from "./types"; + +export interface BindingResolver { + /** Resolve the DID authorized to sign credentials for this space. Returns + * null if no binding could be found via this resolver — the composite + * walks down its list looking for a non-null. */ + resolveAuthority(spaceUri: string): Promise<string | null>; +} + +export interface KeyResolver { + /** Resolve `did`'s verification key for credential signing. `kid` is the + * full header `kid` value (e.g. "did:web:x.com#atproto_space_authority"), + * used to disambiguate when a DID doc lists multiple methods. */ + resolveKey(did: string, kid: string | undefined): Promise<JsonWebKey | null>; +} + +// --------------------------------------------------------------------------- +// Binding resolvers +// --------------------------------------------------------------------------- + +/** Always returns the configured authority DID. Used in-process when the + * authority and record host run in one deployment — no need to walk DID + * docs or PDSes; we know what we are. */ +export function createLocalBindingResolver(args: { + authorityDid: string; +}): BindingResolver { + const { authorityDid } = args; + return { + async resolveAuthority() { + return authorityDid; + }, + }; +} + +/** Reads the record host's local enrollment table. This is the *canonical* + * binding source on a record host: the host owner explicitly consented to a + * given authority for a given space (via the `recordHost.enroll` endpoint + * or auto-enrollment from the authority's createSpace). PDS-record / + * DID-doc resolvers are out-of-band discovery aids; the enrollment is what + * actually gates whether records get stored here. */ +export function createEnrollmentBindingResolver(args: { + recordHost: RecordHost; +}): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const e = await args.recordHost.getEnrollment(spaceUri); + return e?.authorityDid ?? null; + }, + }; +} + +/** Returns the space owner DID as the authority. This is the implicit + * fallback ("HappyView path") — when no PDS record and no DID-doc service + * entry declare an issuer, the owner is taken to be its own. Whether the + * resulting credential actually verifies depends on whether the owner's DID + * doc publishes a usable signing key. */ +export function createOwnerSelfBindingResolver(): BindingResolver { + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + return parts ? parts.ownerDid : null; + }, + }; +} + +/** Walks the resolver list in order, returns the first non-null. Use this + * to compose [pdsRecord, didDocService, ownerSelf] etc. */ +export function createCompositeBindingResolver( + resolvers: BindingResolver[] +): BindingResolver { + return { + async resolveAuthority(spaceUri) { + for (const r of resolvers) { + const did = await r.resolveAuthority(spaceUri); + if (did) return did; + } + return null; + }, + }; +} + +/** Reads a space-declaration record from the owner's PDS at + * `at://<owner>/<type>/<key>` and returns its `authority` field if present. + * + * This is the user-owned-DID path: the user writes a record to their PDS + * authorizing some service as the space's authority, no DID-doc edits + * required. */ +export function createPdsBindingResolver(args: { + /** DID resolver, used to look up the owner's PDS endpoint. */ + resolver: DidDocumentResolver; + /** Fetch impl. Defaults to `globalThis.fetch`. */ + fetch?: typeof fetch; + /** Per-request timeout in ms. Defaults to 5000. */ + timeoutMs?: number; +}): BindingResolver { + const fetchImpl = args.fetch ?? globalThis.fetch; + const timeoutMs = args.timeoutMs ?? 5000; + + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + const pds = await pdsEndpointFor(args.resolver, parts.ownerDid); + if (!pds) return null; + + const url = new URL(`${pds}/xrpc/com.atproto.repo.getRecord`); + url.searchParams.set("repo", parts.ownerDid); + url.searchParams.set("collection", parts.type); + url.searchParams.set("rkey", parts.key); + + const ctrl = new AbortController(); + const timer = setTimeout(() => ctrl.abort(), timeoutMs); + let res: Response; + try { + res = await fetchImpl(url.toString(), { signal: ctrl.signal }); + } catch { + return null; + } finally { + clearTimeout(timer); + } + if (!res.ok) return null; + const body = (await res.json().catch(() => null)) as + | { value?: { authority?: unknown } } + | null; + const authority = body?.value?.authority; + return typeof authority === "string" && authority.startsWith("did:") ? authority : null; + }, + }; +} + +/** Reads `service[id="#atproto_space_authority"].serviceEndpoint` from the + * owner's DID doc. This is the no-PDS path — useful for provisioned space + * DIDs that exist as DID docs only. + * + * Note the service endpoint here is a *DID*, not a URL. The DID names the + * authority; the key resolver's job is to then fetch its verification key. + * For DID docs that declare a URL endpoint, we treat the URL as a + * did:web hint — caller can normalize. */ +export function createDidDocBindingResolver(args: { + resolver: DidDocumentResolver; + /** Service id to look up. Defaults to "#atproto_space_authority". */ + serviceId?: string; +}): BindingResolver { + const serviceId = args.serviceId ?? "#atproto_space_authority"; + return { + async resolveAuthority(spaceUri) { + const parts = parseSpaceUri(spaceUri); + if (!parts) return null; + let doc; + try { + doc = await args.resolver.resolve(parts.ownerDid as Did); + } catch { + return null; + } + const entry = doc.service?.find((s: { id?: string }) => s.id === serviceId); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + if (typeof endpoint !== "string") return null; + // Endpoint may be a DID (preferred) or a URL hint. Only DIDs are + // verifiable downstream; URLs require the caller to map URL → DID. + return endpoint.startsWith("did:") ? endpoint : null; + }, + }; +} + +// --------------------------------------------------------------------------- +// Key resolvers +// --------------------------------------------------------------------------- + +/** Knows the local authority's public key directly. Returns null for any + * other DID — composite with a DID-doc resolver if you also accept + * external authorities. */ +export function createLocalKeyResolver(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): KeyResolver { + return { + async resolveKey(did) { + return did === args.authorityDid ? args.publicKey : null; + }, + }; +} + +/** Resolves a DID, finds the verification method matching `kid`, returns + * its `publicKeyJwk`. */ +export function createDidDocKeyResolver(args: { + resolver: DidDocumentResolver; +}): KeyResolver { + return { + async resolveKey(did, kid) { + let doc; + try { + doc = await args.resolver.resolve(did as Did); + } catch { + return null; + } + const methods = (doc as { verificationMethod?: VerificationMethod[] }).verificationMethod; + if (!methods) return null; + // kid is "<did>#<methodId>" — we match against the method.id which DID + // docs spell as "<did>#<methodId>" too. + const method = kid + ? methods.find((m) => m.id === kid) + : methods[0]; + if (!method?.publicKeyJwk) return null; + return method.publicKeyJwk as JsonWebKey; + }, + }; +} + +/** Walks resolvers in order; returns the first non-null. */ +export function createCompositeKeyResolver( + resolvers: KeyResolver[] +): KeyResolver { + return { + async resolveKey(did, kid) { + for (const r of resolvers) { + const k = await r.resolveKey(did, kid); + if (k) return k; + } + return null; + }, + }; +} + +interface VerificationMethod { + id: string; + type?: string; + controller?: string; + publicKeyJwk?: unknown; + publicKeyMultibase?: string; +} + +// --------------------------------------------------------------------------- +// Internal: PDS endpoint lookup +// --------------------------------------------------------------------------- + +async function pdsEndpointFor( + resolver: DidDocumentResolver, + did: string +): Promise<string | null> { + let doc; + try { + doc = await resolver.resolve(did as Did); + } catch { + return null; + } + const entry = doc.service?.find( + (s: { id?: string }) => s.id === "#atproto_pds" + ); + if (!entry) return null; + const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; + return typeof endpoint === "string" ? endpoint : null; +} diff --git a/packages/contrail-base/src/spaces/blob-adapter.ts b/packages/contrail-base/src/spaces/blob-adapter.ts new file mode 100644 index 0000000..632b566 --- /dev/null +++ b/packages/contrail-base/src/spaces/blob-adapter.ts @@ -0,0 +1,94 @@ +/** + * Bytes-only storage adapter for space blobs. Metadata (CID, mime, size, + * author, space) lives in the `spaces_blobs` table on the main StorageAdapter; + * this interface only moves bytes in and out of a backend (R2, S3, fs, …). + * + * Keys are opaque strings formed by the router as `blobKey(spaceUri, cid)`. + */ + +export interface BlobUploadMeta { + mimeType: string; + size: number; +} + +export interface BlobAdapter { + put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void>; + get(key: string): Promise<Uint8Array | null>; + /** Bulk delete. Adapters that don't support batch can implement serially. */ + delete(keys: string[]): Promise<void>; +} + +/** In-memory adapter. Useful for tests and local development. */ +export class MemoryBlobAdapter implements BlobAdapter { + private readonly store = new Map<string, Uint8Array>(); + + async put(key: string, bytes: Uint8Array): Promise<void> { + this.store.set(key, bytes.slice()); + } + + async get(key: string): Promise<Uint8Array | null> { + const v = this.store.get(key); + return v ? v.slice() : null; + } + + async delete(keys: string[]): Promise<void> { + for (const k of keys) this.store.delete(k); + } + + /** Test helper. */ + size(): number { + return this.store.size; + } +} + +/** Minimal Cloudflare R2 bucket shape — matches @cloudflare/workers-types' R2Bucket + * without forcing a types dependency here. */ +export interface R2BucketLike { + put( + key: string, + value: ArrayBuffer | ArrayBufferView | ReadableStream | Blob, + options?: { httpMetadata?: { contentType?: string }; customMetadata?: Record<string, string> } + ): Promise<unknown>; + get(key: string): Promise<{ arrayBuffer(): Promise<ArrayBuffer> } | null>; + delete(keys: string | string[]): Promise<void>; +} + +/** Cloudflare R2 adapter. Pass the `env.BLOBS` binding from your Worker. */ +export class R2BlobAdapter implements BlobAdapter { + constructor(private readonly bucket: R2BucketLike) {} + + async put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void> { + await this.bucket.put(key, bytes, { + httpMetadata: { contentType: meta.mimeType }, + }); + } + + async get(key: string): Promise<Uint8Array | null> { + const obj = await this.bucket.get(key); + if (!obj) return null; + const buf = await obj.arrayBuffer(); + return new Uint8Array(buf); + } + + async delete(keys: string[]): Promise<void> { + if (keys.length === 0) return; + await this.bucket.delete(keys); + } +} + +/** Hash a space URI to a short, filesystem/R2-safe key segment. + * Used as the first segment of a blob key so all blobs for one space + * share a common prefix (enables bulk delete on space deletion). */ +export async function spaceKeyPrefix(spaceUri: string): Promise<string> { + const bytes = new TextEncoder().encode(spaceUri); + const digest = await crypto.subtle.digest("SHA-256", bytes); + const hex = Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join(""); + return hex.slice(0, 16); +} + +/** Compose an adapter key from a space URI and CID. + * Shape: `<16-hex-chars-of-sha256(spaceUri)>/<cid>`. */ +export async function blobKey(spaceUri: string, cid: string): Promise<string> { + const prefix = await spaceKeyPrefix(spaceUri); + return `${prefix}/${cid}`; +} diff --git a/packages/contrail-base/src/spaces/credentials.ts b/packages/contrail-base/src/spaces/credentials.ts new file mode 100644 index 0000000..741b82d --- /dev/null +++ b/packages/contrail-base/src/spaces/credentials.ts @@ -0,0 +1,284 @@ +/** Space-credential primitives: ES256 (P-256) JWTs minted by the authority, + * verified by the record host (or any third party that can resolve the + * authority's DID document). + * + * Format is a compact JWS: + * header = { alg: "ES256", typ: "JWT", kid: "<authorityDid>#<keyId>" } + * payload = { iss, sub, space, scope, iat, exp } + * + * - `iss` is the authority DID (the signer; for phase 3 this is the local + * authority's serviceDid; phase 4 adds a binding-resolution layer that + * lets the issuer be a *different* DID from the space owner). + * - `sub` is the caller DID — the credential bearer. + * - `space` is the full `ats://<owner>/<type>/<key>` URI. + * - `scope` is "rw" or "read". + * + * We don't use a JWT library — Web Crypto's subtle covers everything (P-256 + * generate, sign, verify, JWK import/export) and saves a runtime dep. */ + +const ALG = "ES256"; +const TYP = "JWT"; +const DEFAULT_KEY_ID = "atproto_space_authority"; + +export type CredentialScope = "rw" | "read"; + +export interface CredentialClaims { + iss: string; + sub: string; + space: string; + scope: CredentialScope; + iat: number; // seconds since epoch + exp: number; // seconds since epoch +} + +export interface CredentialKeyMaterial { + /** Private key in JWK form. P-256 / ES256. */ + privateKey: JsonWebKey; + /** Public key in JWK form. Must match privateKey. */ + publicKey: JsonWebKey; + /** DID-doc verification method id. The full JWT `kid` becomes + * `<authorityDid>#<keyId>`. Defaults to "atproto_space_authority". */ + keyId?: string; +} + +/** Generate a fresh P-256 keypair as JWKs. Useful for local dev / tests; in + * production the operator generates once and stores out-of-band. */ +export async function generateAuthoritySigningKey(): Promise<CredentialKeyMaterial> { + const pair = (await crypto.subtle.generateKey( + { name: "ECDSA", namedCurve: "P-256" }, + true, + ["sign", "verify"] + )) as CryptoKeyPair; + const privateKey = (await crypto.subtle.exportKey("jwk", pair.privateKey)) as JsonWebKey; + const publicKey = (await crypto.subtle.exportKey("jwk", pair.publicKey)) as JsonWebKey; + return { privateKey, publicKey }; +} + +const enc = new TextEncoder(); +const dec = new TextDecoder(); + +function base64urlEncode(bytes: Uint8Array): string { + let s = btoa(String.fromCharCode(...bytes)); + return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} + +function base64urlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} + +function jsonEncode(value: unknown): string { + return base64urlEncode(enc.encode(JSON.stringify(value))); +} + +function jsonDecode<T>(seg: string): T { + return JSON.parse(dec.decode(base64urlDecode(seg))) as T; +} + +async function importPrivate(jwk: JsonWebKey): Promise<CryptoKey> { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["sign"] + ); +} + +async function importPublic(jwk: JsonWebKey): Promise<CryptoKey> { + return crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"] + ); +} + +/** Sign a credential payload with the authority's private key. + * `iat` and `exp` are filled in by the caller (so tests can mint expired + * tokens deterministically). */ +export async function signCredential( + payload: CredentialClaims, + key: CredentialKeyMaterial +): Promise<string> { + const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; + const header = { alg: ALG, typ: TYP, kid }; + const head = jsonEncode(header); + const body = jsonEncode(payload); + const signingInput = `${head}.${body}`; + const privateKey = await importPrivate(key.privateKey); + const sig = await crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + privateKey, + enc.encode(signingInput) + ); + return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; +} + +/** Issue a credential using the current wall-clock for iat/exp. */ +export async function issueCredential( + args: Omit<CredentialClaims, "iat" | "exp"> & { ttlMs: number }, + key: CredentialKeyMaterial +): Promise<{ credential: string; expiresAt: number }> { + const now = Math.floor(Date.now() / 1000); + const expSec = now + Math.floor(args.ttlMs / 1000); + const claims: CredentialClaims = { + iss: args.iss, + sub: args.sub, + space: args.space, + scope: args.scope, + iat: now, + exp: expSec, + }; + const credential = await signCredential(claims, key); + return { credential, expiresAt: expSec * 1000 }; +} + +export type VerifyOk = { ok: true; claims: CredentialClaims }; +export type VerifyErr = { + ok: false; + reason: + | "malformed" + | "bad-alg" + | "bad-signature" + | "expired" + | "not-yet-valid" + | "wrong-space" + | "wrong-scope" + | "unknown-issuer"; +}; + +export interface VerifyOptions { + /** Optional: when set, rejects credentials whose `space` claim differs. + * Omit when verifying in middleware where the target space isn't known + * yet — handlers can do the match themselves against the verified + * claims. */ + expectedSpace?: string; + /** Optional: required scope (e.g. "rw" rejects read-only credentials on writes). */ + requiredScope?: CredentialScope; + /** Resolve a verification key for `iss`. If null, verification fails with + * unknown-issuer. */ + resolveKey: (iss: string, kid: string | undefined) => Promise<JsonWebKey | null>; + /** Time provider for tests. Returns ms since epoch. */ + now?: () => number; +} + +export async function verifyCredential( + jwt: string, + opts: VerifyOptions +): Promise<VerifyOk | VerifyErr> { + const parts = jwt.split("."); + if (parts.length !== 3) return { ok: false, reason: "malformed" }; + const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; + + let header: { alg?: string; typ?: string; kid?: string }; + let claims: CredentialClaims; + try { + header = jsonDecode(headSeg); + claims = jsonDecode(bodySeg); + } catch { + return { ok: false, reason: "malformed" }; + } + if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; + if (opts.expectedSpace !== undefined && claims.space !== opts.expectedSpace) { + return { ok: false, reason: "wrong-space" }; + } + if (opts.requiredScope === "rw" && claims.scope !== "rw") { + return { ok: false, reason: "wrong-scope" }; + } + + const nowMs = (opts.now ?? Date.now)(); + const nowSec = Math.floor(nowMs / 1000); + if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; + if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; + + const jwk = await opts.resolveKey(claims.iss, header.kid); + if (!jwk) return { ok: false, reason: "unknown-issuer" }; + + const publicKey = await importPublic(jwk); + const sigBytes = base64urlDecode(sigSeg); + const signingInput = `${headSeg}.${bodySeg}`; + const valid = await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + sigBytes as BufferSource, + enc.encode(signingInput) + ); + if (!valid) return { ok: false, reason: "bad-signature" }; + return { ok: true, claims }; +} + +/** Header reader for handlers that want to peek at `iss` before resolving the + * key (e.g. to short-circuit DID-doc fetches for the local authority). */ +export function decodeUnverifiedClaims(jwt: string): CredentialClaims | null { + const parts = jwt.split("."); + if (parts.length !== 3) return null; + try { + return jsonDecode<CredentialClaims>(parts[1]!); + } catch { + return null; + } +} + +/** Verifier interface consumed by the record host. The record host doesn't + * care HOW credentials get verified — it only cares whether a given JWT is + * valid. Phase 3 ships an in-process verifier that knows the local + * authority's public key; phase 4 adds a binding-resolving verifier that + * consults PDS records / DID docs. */ +export interface CredentialVerifier { + /** Verify a credential's signature, expiry, and `not-before` window. Does + * NOT enforce a space match — handlers do that against the request URI. */ + verify(jwt: string): Promise<VerifyOk | VerifyErr>; +} + +/** In-process verifier for the simple deployment: the authority and record + * host run in one process and the record host has direct access to the + * authority's public key. Rejects any credential whose `iss` isn't the + * configured authority. Phase 4 has a more general + * {@link createBindingCredentialVerifier} that does proper binding lookup. */ +export function createInProcessVerifier(args: { + authorityDid: string; + publicKey: JsonWebKey; +}): CredentialVerifier { + return { + verify(jwt) { + return verifyCredential(jwt, { + resolveKey: async (iss) => (iss === args.authorityDid ? args.publicKey : null), + }); + }, + }; +} + +/** Verifier composed of a {@link BindingResolver} (which DID is authorized + * to issue for this space?) and a {@link KeyResolver} (what's that DID's + * public key?). This is the production-shape verifier — phase 4's main + * contribution. + * + * Verification flow: + * 1. Decode the JWT's claims (no signature check yet). + * 2. Ask the binding resolver: who's authorized for `claims.space`? + * 3. Confirm `claims.iss === authorizedDid`. + * 4. Ask the key resolver for that DID's verification key. + * 5. Verify signature + expiry + scope match. + */ +export function createBindingCredentialVerifier(args: { + bindings: import("./binding").BindingResolver; + keys: import("./binding").KeyResolver; +}): CredentialVerifier { + return { + async verify(jwt) { + const peek = decodeUnverifiedClaims(jwt); + if (!peek) return { ok: false, reason: "malformed" }; + const authorizedDid = await args.bindings.resolveAuthority(peek.space); + if (!authorizedDid) return { ok: false, reason: "unknown-issuer" }; + if (peek.iss !== authorizedDid) return { ok: false, reason: "unknown-issuer" }; + return verifyCredential(jwt, { + resolveKey: (iss, kid) => args.keys.resolveKey(iss, kid), + }); + }, + }; +} diff --git a/packages/contrail-base/src/spaces/in-process.ts b/packages/contrail-base/src/spaces/in-process.ts new file mode 100644 index 0000000..ec1d7a1 --- /dev/null +++ b/packages/contrail-base/src/spaces/in-process.ts @@ -0,0 +1,34 @@ +/** In-process auth marker. + * + * For same-module callers (e.g. a SvelteKit worker that imports contrail and + * dispatches requests directly to the handler), service-auth JWTs are pure + * overhead: no network boundary is crossed, so there's nothing for the JWT to + * protect against. Instead, the caller tags the `Request` with a principal via + * a module-private WeakMap, and the auth middleware reads it back. + * + * Security note: this is unforgeable from outside the module because + * - WeakMap keys are `Request` object identities, not serialized data; + * - no HTTP request crossing a network boundary can reach into this map; + * - exploiting it requires code execution inside the same isolate, at + * which point auth is already game over. + * + * This is the strongest auth adapter contrail offers — it has no secret to + * leak. */ + +export interface InProcessPrincipal { + did: string; +} + +const PRINCIPALS = new WeakMap<Request, InProcessPrincipal>(); + +/** Tag a Request with an in-process principal. The returned Request is the + * same reference; the return value is for ergonomics. */ +export function markInProcess(req: Request, did: string): Request { + PRINCIPALS.set(req, { did }); + return req; +} + +/** Read the in-process principal for a Request, or null if unmarked. */ +export function readInProcess(req: Request): InProcessPrincipal | null { + return PRINCIPALS.get(req) ?? null; +} diff --git a/packages/contrail-base/src/spaces/tid.ts b/packages/contrail-base/src/spaces/tid.ts new file mode 100644 index 0000000..c419090 --- /dev/null +++ b/packages/contrail-base/src/spaces/tid.ts @@ -0,0 +1,20 @@ +const B32_CHARSET = "234567abcdefghijklmnopqrstuvwxyz"; + +let lastTimestamp = 0; +let clockId = Math.floor(Math.random() * 1024); + +/** Generate an atproto TID: 13-char base32-sortable (timestamp-ordered). */ +export function nextTid(): string { + let now = Date.now() * 1000; + if (now <= lastTimestamp) now = lastTimestamp + 1; + lastTimestamp = now; + + const n = BigInt(now) * 1024n + BigInt(clockId); + let s = ""; + let v = n; + for (let i = 0; i < 13; i++) { + s = B32_CHARSET[Number(v & 31n)] + s; + v >>= 5n; + } + return s; +} diff --git a/packages/contrail-base/src/spaces/types.ts b/packages/contrail-base/src/spaces/types.ts new file mode 100644 index 0000000..6561773 --- /dev/null +++ b/packages/contrail-base/src/spaces/types.ts @@ -0,0 +1,273 @@ +import type { Database } from "../types"; +import type { DidDocumentResolver } from "@atcute/identity-resolver"; +import type { BlobAdapter } from "./blob-adapter"; +import type { CredentialKeyMaterial } from "./credentials"; + +export type AppPolicyMode = "allow" | "deny"; + +export interface AppPolicy { + mode: AppPolicyMode; + apps: string[]; +} + +export interface SpacesBlobsConfig { + /** Bytes backend (R2, S3, in-memory, …). */ + adapter: BlobAdapter; + /** Max blob size in bytes. Defaults to 2 MiB. */ + maxSize?: number; + /** MIME allowlist. If set, only these content types are accepted. */ + accept?: string[]; + /** Orphan blobs (those with no referencing record) are kept this long before + * GC can delete them, to allow upload-then-putRecord flows. + * Defaults to 24 hours. */ + gcOrphanAfterMs?: number; +} + +export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; +export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; + +/** Default credential lifetime. The rough spec calls for 2–4h; we pick the + * lower bound so revocation (kicked-from-space) is observable within 2h. */ +export const DEFAULT_CREDENTIAL_TTL_MS = 2 * 60 * 60 * 1000; + +/** Configuration for the **space authority** role: holds the member list, + * signs credentials, and gates space-management operations. In a fully-split + * deployment, the authority can run in a different process (or even a + * different operator) than the record host. */ +export interface AuthorityConfig { + /** NSID that identifies the kind of space this authority hosts, + * e.g. "tools.atmo.event.space". */ + type: string; + /** Service DID that service-auth tokens must target (aud claim) AND that + * signs credentials it issues (`iss` claim on emitted JWTs). */ + serviceDid: string; + /** Default app policy applied to new spaces. */ + defaultAppPolicy?: AppPolicy; + /** DID document resolver for service-auth JWT verification. + * Defaults to a composite PLC + did:web resolver if omitted. */ + resolver?: DidDocumentResolver; + /** Signing key material for issuing space credentials. When omitted, + * `<ns>.space.getCredential` returns 501 NotImplemented and the record + * host's credential-verifying middleware can't be wired up. */ + signing?: CredentialKeyMaterial; + /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ + credentialTtlMs?: number; +} + +/** Configuration for the **record host** role: stores per-space records and + * blobs and serves reads. Verifies space credentials (later phases) on + * incoming traffic. */ +export interface RecordHostConfig { + /** Blob-upload backend. When omitted, blob XRPCs are not exposed. */ + blobs?: SpacesBlobsConfig; +} + +/** Spaces config — host an authority, a record host, or both. + * Today both run in one process and most deployments will set both; the + * shape is split now so phase 5 can run them independently without churning + * every consumer's config. */ +export interface SpacesConfig { + /** Space-authority config — member list, credentials (later), space + * management. Required for any space to exist. */ + authority?: AuthorityConfig; + /** Record-host config — record + blob storage. Required for records to be + * written/read on this deployment. */ + recordHost?: RecordHostConfig; +} + +export interface SpaceRow { + uri: string; + ownerDid: string; + type: string; + key: string; + serviceDid: string; + appPolicyRef: string | null; + appPolicy: AppPolicy | null; + createdAt: number; + deletedAt: number | null; +} + +export interface SpaceMemberRow { + spaceUri: string; + did: string; + addedAt: number; + addedBy: string | null; +} + +export interface StoredRecord { + spaceUri: string; + collection: string; + authorDid: string; + rkey: string; + cid: string | null; + record: Record<string, unknown>; + createdAt: number; +} + +export interface ListOptions { + byUser?: string; + cursor?: string; + limit?: number; +} + +export interface ListResult { + records: StoredRecord[]; + cursor?: string; +} + +export interface ListSpacesOptions { + type?: string; + ownerDid?: string; + memberDid?: string; + limit?: number; + cursor?: string; +} + +export interface CollectionCount { + collection: string; + count: number; +} + +/** What a token holder can do with this invite. + * - `'join'`: must be redeemed while signed in; redeemer becomes a member. + * - `'read'`: bearer-only — token itself grants read access to the space; cannot be redeemed. + * - `'read-join'`: both — anonymous holders read; signed-in holders may also redeem to join. */ +export type InviteKind = "join" | "read" | "read-join"; + +export interface InviteRow { + tokenHash: string; + spaceUri: string; + kind: InviteKind; + expiresAt: number | null; + maxUses: number | null; + usedCount: number; + createdBy: string; + createdAt: number; + revokedAt: number | null; + note: string | null; +} + +export interface CreateInviteInput { + spaceUri: string; + tokenHash: string; + kind: InviteKind; + expiresAt: number | null; + maxUses: number | null; + createdBy: string; + note: string | null; +} + +export interface RedeemInviteResult { + spaceUri: string; +} + +export interface BlobMetaRow { + spaceUri: string; + cid: string; + mimeType: string; + size: number; + authorDid: string; + createdAt: number; +} + +/** Local cache on the record host: this space is accepted here, and `authority_did` + * is the DID authorized to sign credentials for it. Populated via the + * `recordHost.enroll` endpoint or auto-populated by the authority's + * createSpace when both roles run in one process. */ +export interface EnrollmentRow { + spaceUri: string; + authorityDid: string; + enrolledAt: number; + enrolledBy: string; +} + +export interface ListBlobsOptions { + byUser?: string; + cursor?: string; + limit?: number; +} + +export interface ListBlobsResult { + blobs: BlobMetaRow[]; + cursor?: string; +} + +/** **Space authority** interface — owner of the space's ACL state and + * (eventually) credential issuer. Holds the member list, manages invites, + * governs space lifecycle and app policy. Does NOT touch records or blobs. + * + * In a fully-split deployment this is a separate service; today the + * HostedAdapter implements both this and {@link RecordHost} against one DB. */ +export interface SpaceAuthority { + // Space lifecycle + createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow>; + getSpace(spaceUri: string): Promise<SpaceRow | null>; + listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }>; + deleteSpace(spaceUri: string): Promise<void>; + updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void>; + + // Members + addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void>; + removeMember(spaceUri: string, did: string): Promise<void>; + getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null>; + listMembers(spaceUri: string): Promise<SpaceMemberRow[]>; + /** Bulk-apply a membership diff. Used only by the community module's reconciler; + * not exposed as an XRPC endpoint. */ + applyMembershipDiff( + spaceUri: string, + adds: string[], + removes: string[], + addedBy: string | null + ): Promise<void>; + + // Invites (token primitive — issued by the authority, scoped to a space) + createInvite(input: CreateInviteInput): Promise<InviteRow>; + listInvites(spaceUri: string, options?: { includeRevoked?: boolean }): Promise<InviteRow[]>; + revokeInvite(tokenHash: string): Promise<boolean>; + /** Look up an invite without consuming it. Used to validate read-token bearer access. */ + getInvite(tokenHash: string): Promise<InviteRow | null>; + /** Atomically mark a join-capable invite as used. Returns the row if usable + * (kind allows join, not expired/revoked/exhausted), null otherwise. */ + redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null>; +} + +/** **Record host** interface — stores records and blobs for a space, plus + * the local enrollment table that decides which spaces this host accepts. + * + * Trust model: the host trusts whatever credential the authority signs, so + * long as the authority is the one named in the local enrollment for this + * space. Enrollment is the consent step — the host owner agrees to spend + * storage on a given space, scoped to a specific authority. */ +export interface RecordHost { + // Enrollment + enroll(input: EnrollmentRow): Promise<void>; + getEnrollment(spaceUri: string): Promise<EnrollmentRow | null>; + listEnrollments(options?: { authorityDid?: string; limit?: number }): Promise<EnrollmentRow[]>; + removeEnrollment(spaceUri: string): Promise<void>; + + // Records + putRecord(record: StoredRecord): Promise<void>; + getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<StoredRecord | null>; + listRecords(spaceUri: string, collection: string, options?: ListOptions): Promise<ListResult>; + deleteRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<void>; + listCollections(spaceUri: string, options?: { byUser?: string }): Promise<CollectionCount[]>; + + // Blobs (metadata only; bytes live on BlobAdapter) + putBlobMeta(row: BlobMetaRow): Promise<void>; + getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null>; + listBlobMeta(spaceUri: string, options?: ListBlobsOptions): Promise<ListBlobsResult>; + deleteBlobMeta(spaceUri: string, cid: string): Promise<void>; + /** Find blob rows older than `cutoff` whose CIDs are not referenced in any + * record JSON in this space. Capped at `limit` to bound a single GC pass. */ + findOrphanBlobs(spaceUri: string, cutoff: number, limit: number): Promise<BlobMetaRow[]>; +} + +/** Combined adapter. Used internally where a single object satisfies both + * roles (today's HostedAdapter, the community reconciler, the realtime + * publishing wrapper). Phases 5+ replace consumers of this with two + * injected interfaces. */ +export type StorageAdapter = SpaceAuthority & RecordHost; + +export interface AdapterContext { + db: Database; +} diff --git a/packages/contrail-base/src/spaces/uri.ts b/packages/contrail-base/src/spaces/uri.ts new file mode 100644 index 0000000..e73ec21 --- /dev/null +++ b/packages/contrail-base/src/spaces/uri.ts @@ -0,0 +1,37 @@ +/** Centralized space URI construction / parsing. + * + * Permissioned spaces are addressed by (ownerDid, type, key) and use the + * `ats://` scheme — distinct from atproto record URIs (`at://`) so the two + * can't be confused at any layer (logs, params, dispatch). Tracks the rough + * spec at https://dholms.leaflet.pub/3mhj6bcqats2o. + * + * Record URIs inside a space are minted by authorDid for index purposes + * (`at://<authorDid>/<collection>/<rkey>`); the spec is explicitly undecided + * about authority (user vs space owner), so we don't expose those as a + * canonical record address — they're storage-internal. */ + +export interface SpaceUriParts { + ownerDid: string; + type: string; + key: string; +} + +/** Build a space URI from its three addressing components. */ +export function buildSpaceUri(parts: SpaceUriParts): string { + return `ats://${parts.ownerDid}/${parts.type}/${parts.key}`; +} + +/** Parse a space URI into its components, or null if malformed. */ +export function parseSpaceUri(uri: string): SpaceUriParts | null { + if (!uri.startsWith("ats://")) return null; + const rest = uri.slice("ats://".length); + const [ownerDid, type, key, ...extra] = rest.split("/"); + if (!ownerDid || !type || !key || extra.length > 0) return null; + return { ownerDid, type, key }; +} + +/** Build a record URI under a given author. Used only as a secondary index key + * inside storage — not a canonical address for permissioned records. */ +export function buildRecordUri(authorDid: string, collection: string, rkey: string): string { + return `at://${authorDid}/${collection}/${rkey}`; +} diff --git a/packages/contrail-base/src/types.ts b/packages/contrail-base/src/types.ts new file mode 100644 index 0000000..3b9dc48 --- /dev/null +++ b/packages/contrail-base/src/types.ts @@ -0,0 +1,513 @@ +import type { SqlDialect } from "./dialect"; + +// Database interface — D1 implements this natively +export interface Database { + prepare(sql: string): Statement; + batch(stmts: Statement[]): Promise<any[]>; + dialect?: SqlDialect; +} + +export interface Statement { + bind(...values: any[]): Statement; + run(): Promise<any>; + all<T = any>(): Promise<{ results: T[] }>; + first<T = any>(): Promise<T | null>; +} + +// Config types + +export interface QueryableField { + type?: "range"; +} + +export interface RelationConfig { + /** Short name of the child collection (a key in `collections`). */ + collection: string; + field?: string; + match?: "uri" | "did"; + groupBy?: string; + /** Enable materialized count columns on the parent. Defaults to true. */ + count?: boolean; + /** Count distinct values of a field (e.g. "did" for unique users) instead of total records. */ + countDistinct?: string; + /** Pre-resolved group mappings: shortName → full token (e.g. { going: "community.lexicon.calendar.rsvp#going" }). Auto-computed from groupBy if omitted. */ + groups?: Record<string, string>; +} + +/** A forward reference: this collection's records point at another collection. */ +export interface ReferenceConfig { + /** Short name of the target collection. */ + collection: string; + /** Field on this collection's records containing the target URI. */ + field: string; +} + +export type CustomQueryHandler = ( + db: Database, + params: URLSearchParams, + config: ContrailConfig +) => Promise<Response>; + +export interface RecordSource { + joins?: string; + conditions?: string[]; + params?: (string | number)[]; +} + +export type PipelineQueryHandler = ( + db: Database, + params: URLSearchParams, + config: ContrailConfig +) => Promise<RecordSource>; + +export interface FeedConfig { + /** Short name of the follow collection. */ + follow: string; + /** Short names of target collections to fan out to. */ + targets: string[]; + /** Max feed items per user (default: 200). Oldest items are pruned after backfill. */ + maxItems?: number; +} + +export const DEFAULT_FEED_MAX_ITEMS = 200; + +export type CollectionMethod = "listRecords" | "getRecord"; +export const DEFAULT_COLLECTION_METHODS: CollectionMethod[] = [ + "listRecords", + "getRecord", +]; + +export interface CollectionConfig { + /** Full NSID of the record type this collection indexes. */ + collection: string; + /** Include this collection in Jetstream ingest / discovery (default true). + * Set false for dependent collections (auto-fetched on demand). */ + discover?: boolean; + queryable?: Record<string, QueryableField>; + relations?: Record<string, RelationConfig>; + /** Forward references: fields on this collection's records that point at another collection. */ + references?: Record<string, ReferenceConfig>; + queries?: Record<string, CustomQueryHandler>; + pipelineQueries?: Record<string, PipelineQueryHandler>; + /** FTS5 search fields. Provide an array of field names to enable full-text search. Omit or set to false to disable. */ + searchable?: string[] | false; + /** XRPC methods to emit. Defaults to ['listRecords', 'getRecord']. */ + methods?: CollectionMethod[]; + /** When spaces are enabled globally, emit a parallel spaces_records_<short> table + * so this collection can also live inside spaces. Defaults to true. */ + allowInSpaces?: boolean; +} + +export interface ProfileConfig { + /** Full NSID of the profile record type. */ + collection: string; + /** Short name used for table/endpoint naming. Defaults to the NSID's last segment. */ + shortName?: string; + rkey?: string; // defaults to "self" +} + +export const DEFAULT_PROFILES: ProfileConfig[] = [ + { collection: "app.bsky.actor.profile", shortName: "profile" }, +]; + +/** Normalize a profiles config entry (string or object) into ProfileConfig. */ +export function normalizeProfileConfig( + p: string | ProfileConfig +): ProfileConfig { + if (typeof p === "string") { + return { collection: p, shortName: deriveShortName(p) }; + } + return { ...p, shortName: p.shortName ?? deriveShortName(p.collection) }; +} + +/** Last NSID segment, used as fallback short name. */ +export function deriveShortName(nsid: string): string { + const parts = nsid.split("."); + return parts[parts.length - 1] ?? nsid; +} + +export const DEFAULT_JETSTREAMS = [ + "wss://jetstream1.us-east.bsky.network", +]; + +export const DEFAULT_RELAYS = [ + "https://relay1.us-east.bsky.network" +]; + +export interface Logger { + log(...args: any[]): void; + warn(...args: any[]): void; + error(...args: any[]): void; +} + +export interface ContrailConfig { + namespace: string; + /** Collections to index, keyed by short name. Short names become endpoint URL segments + * (`<namespace>.<short>.listRecords`) and table suffixes (`records_<short>`). */ + collections: Record<string, CollectionConfig>; + profiles?: (string | ProfileConfig)[]; + relays?: string[]; + jetstreams?: string[]; + feeds?: Record<string, FeedConfig>; + logger?: Logger; + /** Expose the notifyOfUpdate HTTP endpoint. Off by default. + * Set to `true` for open access, or a string to require `Authorization: Bearer <secret>`. */ + notify?: boolean | string; + /** Permissioned spaces configuration. When set, the service exposes space XRPCs. */ + spaces?: import("./spaces/types").SpacesConfig; + /** Community module configuration. Typed by the community package via + * declaration merging — contrail core only knows it's "something the + * community package consumes." Set when wiring community via + * `createCommunityIntegration({ ... })`. Requires `spaces.authority`. */ + community?: unknown; + /** Realtime module configuration. When set, the service exposes ticket + SSE/WS + * subscribe XRPCs, and wraps the spaces adapter to publish events after writes. */ + realtime?: import("./realtime/types").RealtimeConfig; + /** Labels module configuration. When set, contrail subscribes to the + * configured labelers, indexes their labels into a single `labels` table, + * and hydrates `record.labels` onto `listRecords` / `getRecord` / profile + * responses gated by the caller's `atproto-accept-labelers` header. */ + labels?: import("./labels/types").LabelsConfig; + /** Customize the auto-generated `<namespace>.authFull` lexicon. */ + permissionSet?: PermissionSetConfig; +} + +/** Single entry in an atproto permission-set's `permissions` array. + * See https://atproto.com/guides/permission-sets for the full schema. */ +export type PermissionEntry = + | { type: "permission"; resource: "rpc"; lxm?: string[]; aud?: string; inheritAud?: boolean } + | { type: "permission"; resource: "repo"; collection?: string[] } + | { type: "permission"; resource: "blob"; accept?: string[]; maxSize?: number } + | { type: "permission"; resource: "account"; attr?: string[] } + | { type: "permission"; resource: string; [key: string]: unknown }; + +export interface PermissionSetConfig { + /** Shown on the OAuth consent screen. Defaults to the namespace. */ + title?: string; + /** Shown on the OAuth consent screen. Defaults to a generated description. */ + description?: string; + /** Extra permission entries appended after the auto-generated rpc entry — + * e.g. repo writes for collections your app needs the user to create, or + * blob permissions for uploads. */ + additional?: PermissionEntry[]; +} + +export interface ResolvedRelation { + /** Short name of the child collection. */ + collection: string; + groupBy: string; + groups: Record<string, string>; // shortName → full token value +} + +export interface ResolvedMaps { + queryable: Record<string, Record<string, QueryableField>>; + relations: Record<string, Record<string, ResolvedRelation>>; + /** Reverse map: full record NSID → short name. */ + nsidToShort: Record<string, string>; +} + +/** Config after resolveConfig() — has computed queryable/relation maps attached. */ +export interface ResolvedContrailConfig extends ContrailConfig { + _resolved: ResolvedMaps; +} + +/** + * Resolve config: apply defaults, auto-add profile collections, compute queryable maps. + */ +export function resolveConfig(config: ContrailConfig): ResolvedContrailConfig { + const profiles = (config.profiles ?? DEFAULT_PROFILES).map( + normalizeProfileConfig + ); + const collections = { ...config.collections }; + for (const p of profiles) { + const short = p.shortName!; + if (!collections[short]) { + collections[short] = { collection: p.collection, discover: false }; + } + } + + // Auto-add follow collections from feed configs as dependent collections if they're + // not already listed. Feed config already uses short names so nothing to resolve — + // but if the user forgot to declare the follow collection, we can't auto-add it without + // knowing its NSID. In that case we warn later via validateConfig. + + const base = { + ...config, + collections, + profiles, + jetstreams: config.jetstreams ?? DEFAULT_JETSTREAMS, + relays: config.relays ?? DEFAULT_RELAYS, + logger: config.logger ?? console, + }; + + return { + ...base, + _resolved: _resolveQueryableMaps(base), + }; +} + +function _resolveQueryableMaps(config: ContrailConfig): ResolvedMaps { + const queryable: Record<string, Record<string, QueryableField>> = {}; + const relations: Record<string, Record<string, ResolvedRelation>> = {}; + const nsidToShort: Record<string, string> = {}; + + for (const [short, colConfig] of Object.entries(config.collections)) { + nsidToShort[colConfig.collection] = short; + + if (colConfig.queryable) { + queryable[short] = colConfig.queryable; + } + + if (colConfig.relations) { + for (const [relName, rel] of Object.entries(colConfig.relations)) { + if (!rel.groupBy) continue; + const groups: Record<string, string> = rel.groups ? { ...rel.groups } : {}; + if (Object.keys(groups).length > 0) { + if (!relations[short]) relations[short] = {}; + relations[short][relName] = { + collection: rel.collection, + groupBy: rel.groupBy, + groups, + }; + } + } + } + } + + return { queryable, relations, nsidToShort }; +} + +export function getFeedFollowShortNames(config: ContrailConfig): string[] { + if (!config.feeds) return []; + return [...new Set(Object.values(config.feeds).map((f) => f.follow))]; +} + +/** Alias for getFeedFollowShortNames. */ +export const getFeedFollowCollections = getFeedFollowShortNames; + +// Record types + +export interface RecordRow { + uri: string; + did: string; + collection: string; // full NSID + rkey: string; + cid: string | null; + record: string | null; + time_us: number; + indexed_at: number; + /** Set when the row originates from a per-space table. Used by the + * pipeline/hydration/response layers to route child queries to the same + * space and tag the output. */ + space?: string; +} + +export interface IngestEvent { + uri: string; + did: string; + collection: string; // full NSID + rkey: string; + operation: "create" | "update" | "delete"; + cid: string | null; + record: string | null; + time_us: number; + indexed_at: number; +} + +// Validation + +const SAFE_FIELD_NAME = /^[a-zA-Z0-9_.]+$/; +const SAFE_SHORT_NAME = /^[a-zA-Z][a-zA-Z0-9]*$/; + +export function validateFieldName(field: string): string { + if (!SAFE_FIELD_NAME.test(field)) { + throw new Error(`Invalid field name: ${field}`); + } + return field; +} + +function validateShortName(short: string): void { + if (!SAFE_SHORT_NAME.test(short)) { + throw new Error( + `Invalid collection short name: "${short}". Must be alphanumeric, starting with a letter.` + ); + } +} + +export function validateConfig(config: ContrailConfig): void { + const shortNames = new Set<string>(); + for (const [short, colConfig] of Object.entries(config.collections)) { + validateShortName(short); + if (shortNames.has(short)) { + throw new Error(`Duplicate collection short name: ${short}`); + } + shortNames.add(short); + + if (!colConfig.collection) { + throw new Error(`Collection "${short}" is missing required 'collection' field (NSID)`); + } + + for (const field of Object.keys(colConfig.queryable ?? {})) { + validateFieldName(field); + } + for (const [, rel] of Object.entries(colConfig.relations ?? {})) { + if (rel.field) validateFieldName(rel.field); + if (rel.groupBy) validateFieldName(rel.groupBy); + if (rel.countDistinct) validateFieldName(rel.countDistinct); + if (!config.collections[rel.collection]) { + throw new Error( + `Relation in "${short}" references unknown collection short name "${rel.collection}"` + ); + } + } + for (const [, ref] of Object.entries(colConfig.references ?? {})) { + validateFieldName(ref.field); + if (!config.collections[ref.collection]) { + throw new Error( + `Reference in "${short}" references unknown collection short name "${ref.collection}"` + ); + } + } + if (Array.isArray(colConfig.searchable)) { + for (const field of colConfig.searchable) { + validateFieldName(field); + } + } + } + + if (config.feeds) { + for (const [feedName, feed] of Object.entries(config.feeds)) { + if (!config.collections[feed.follow]) { + throw new Error( + `Feed "${feedName}" references unknown follow collection "${feed.follow}"` + ); + } + for (const target of feed.targets) { + if (!config.collections[target]) { + throw new Error( + `Feed "${feedName}" references unknown target collection "${target}"` + ); + } + } + } + } + + if (config.community && !config.spaces?.authority) { + throw new Error( + "Invalid config: `community` requires `spaces.authority`. Community-owned spaces reuse the spaces storage adapter." + ); + } +} + +// Helpers + +export function getNestedValue(obj: any, path: string): any { + let current = obj; + for (const key of path.split(".")) { + if (current == null) return undefined; + current = current[key]; + } + return current; +} + +const DEFAULT_RELATION_FIELD = "subject.uri"; + +export function getRelationField(rel: RelationConfig): string { + return rel.field ?? DEFAULT_RELATION_FIELD; +} + +/** Sanitize a short name for use in SQL identifiers (already-validated; kept for paranoia). */ +function sanitizeIdentifier(name: string): string { + return name.replace(/[^a-zA-Z0-9]/g, "_"); +} + +/** Total-count column name for a relation targeting the given short name. */ +export function countColumnName(childShortName: string): string { + return "count_" + sanitizeIdentifier(childShortName); +} + +/** Grouped-count column name: `count_<child-short>_<groupKey>`. */ +export function groupedCountColumnName( + childShortName: string, + groupKey: string +): string { + return `count_${sanitizeIdentifier(childShortName)}_${sanitizeIdentifier(groupKey)}`; +} + +/** Table name for a collection's records. */ +export function recordsTableName(shortName: string): string { + return "records_" + sanitizeIdentifier(shortName); +} + +/** Table name for a collection's records inside spaces. */ +export function spacesRecordsTableName(shortName: string): string { + return "spaces_records_" + sanitizeIdentifier(shortName); +} + +/** All collection short names. */ +export function getCollectionShortNames(config: ContrailConfig): string[] { + return Object.keys(config.collections); +} + +/** Alias: collection short names (same as getCollectionShortNames). */ +export const getCollectionNames = getCollectionShortNames; + +/** All indexed record NSIDs (what Jetstream filters on). */ +export function getCollectionNsids(config: ContrailConfig): string[] { + return Object.values(config.collections).map((c) => c.collection); +} + +export function getDependentShortNames(config: ContrailConfig): string[] { + return Object.entries(config.collections) + .filter(([, c]) => c.discover === false) + .map(([name]) => name); +} + +export function getDiscoverableShortNames(config: ContrailConfig): string[] { + return Object.entries(config.collections) + .filter(([, c]) => c.discover !== false) + .map(([name]) => name); +} + +/** Aliases for readability elsewhere. These return short names (new semantic). */ +export const getDependentCollections = getDependentShortNames; +export const getDiscoverableCollections = getDiscoverableShortNames; + +/** Short names of collections the user declared with `discover !== false`, mapped to NSIDs. */ +export function getDiscoverableNsids(config: ContrailConfig): string[] { + return Object.values(config.collections) + .filter((c) => c.discover !== false) + .map((c) => c.collection); +} + +export function getDependentNsids(config: ContrailConfig): string[] { + return Object.values(config.collections) + .filter((c) => c.discover === false) + .map((c) => c.collection); +} + +/** Short name for a record NSID, if known. */ +export function shortNameForNsid( + config: ContrailConfig, + nsid: string +): string | undefined { + const resolved = (config as ResolvedContrailConfig)._resolved; + if (resolved?.nsidToShort) return resolved.nsidToShort[nsid]; + for (const [short, c] of Object.entries(config.collections)) { + if (c.collection === nsid) return short; + } + return undefined; +} + +/** Full NSID for a collection short name. */ +export function nsidForShortName( + config: ContrailConfig, + short: string +): string | undefined { + return config.collections[short]?.collection; +} + +/** The methods a collection should expose via XRPC. */ +export function getCollectionMethods(cfg: CollectionConfig): CollectionMethod[] { + return cfg.methods ?? DEFAULT_COLLECTION_METHODS; +} diff --git a/packages/contrail-base/tsconfig.build.json b/packages/contrail-base/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-base/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-base/tsconfig.json b/packages/contrail-base/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-base/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-base/tsup.config.ts b/packages/contrail-base/tsup.config.ts new file mode 100644 index 0000000..3ad3082 --- /dev/null +++ b/packages/contrail-base/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts", "src/adapters/sqlite.ts", "src/adapters/postgres.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["pg", "node:sqlite"], +}); diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index 43fe2e1..c6abcb2 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -41,6 +41,7 @@ "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", "@atmo-dev/contrail": "workspace:*", + "@atmo-dev/contrail-base": "workspace:*", "hono": "^4.12.8" }, "devDependencies": { diff --git a/packages/contrail-community/vitest.config.ts b/packages/contrail-community/vitest.config.ts index 56720e0..a440f50 100644 --- a/packages/contrail-community/vitest.config.ts +++ b/packages/contrail-community/vitest.config.ts @@ -2,13 +2,16 @@ import { defineConfig } from "vitest/config"; import path from "node:path"; const contrailSrc = path.resolve(__dirname, "../contrail/src"); +const baseSrc = path.resolve(__dirname, "../contrail-base/src"); -// Alias `@atmo-dev/contrail` and its subpaths to the source so tests don't -// run through the built dist. Mirrors the in-tree-source-resolution pattern -// the contrail package's own tests use (they import via ../src/...). +// Resolve workspace-internal imports to source so tests don't run through +// dists (where tsup mangles `node:sqlite` → `sqlite`). export default defineConfig({ resolve: { alias: { + "@atmo-dev/contrail-base/sqlite": path.join(baseSrc, "adapters/sqlite.ts"), + "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), + "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), "@atmo-dev/contrail/sqlite": path.join(contrailSrc, "adapters/sqlite.ts"), "@atmo-dev/contrail/postgres": path.join(contrailSrc, "adapters/postgres.ts"), "@atmo-dev/contrail": path.join(contrailSrc, "index.ts"), diff --git a/packages/contrail/package.json b/packages/contrail/package.json index a64484d..8409809 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,6 +73,7 @@ "@atcute/jetstream": "^1.0.2", "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-base": "workspace:*", "cac": "^7.0.0", "hono": "^4.12.8", "jiti": "^2.4.0" diff --git a/packages/contrail/src/adapters/postgres.ts b/packages/contrail/src/adapters/postgres.ts index ae75c9f..6ab2902 100644 --- a/packages/contrail/src/adapters/postgres.ts +++ b/packages/contrail/src/adapters/postgres.ts @@ -1,93 +1 @@ -import pg from "pg"; -import type { Database, Statement } from "../core/types"; -import { postgresDialect } from "../core/dialect"; - -/** Internal interface for statements that can run on a specific client */ -interface PgStatement extends Statement { - /** Execute on a specific client (used by batch for transaction isolation) */ - _runOn(client: pg.PoolClient): Promise<any>; -} - -/** Column names known to be BIGINT — PostgreSQL returns these as strings */ -const BIGINT_COLUMNS = new Set(["time_us", "indexed_at", "resolved_at"]); - -function normalizeRow(row: any): any { - if (!row) return row; - if (typeof row.record === "object" && row.record !== null) { - row.record = JSON.stringify(row.record); - } - for (const col of BIGINT_COLUMNS) { - if (typeof row[col] === "string") row[col] = Number(row[col]); - } - return row; -} - -export function createPostgresDatabase(pool: pg.Pool): Database { - function rewritePlaceholders(sql: string): string { - let idx = 0; - let inString = false; - let result = ""; - for (let i = 0; i < sql.length; i++) { - const ch = sql[i]; - if (ch === "'" && sql[i - 1] !== "\\") { - inString = !inString; - result += ch; - } else if (ch === "?" && !inString) { - result += `$${++idx}`; - } else { - result += ch; - } - } - return result; - } - - function wrapStatement(sql: string, boundValues: any[] = []): PgStatement { - const pgSql = rewritePlaceholders(sql); - - return { - bind(...values: any[]): PgStatement { - return wrapStatement(sql, values); - }, - async run() { - const result = await pool.query(pgSql, boundValues); - return { changes: result.rowCount }; - }, - async _runOn(client: pg.PoolClient) { - const result = await client.query(pgSql, boundValues); - return { changes: result.rowCount }; - }, - async all<T>() { - const result = await pool.query(pgSql, boundValues); - return { results: result.rows.map(normalizeRow) as T[] }; - }, - async first<T>() { - const result = await pool.query(pgSql, boundValues); - return result.rows[0] ? (normalizeRow(result.rows[0]) as T) : null; - }, - }; - } - - return { - prepare(sql: string): Statement { - return wrapStatement(sql); - }, - async batch(stmts: Statement[]): Promise<any[]> { - const client = await pool.connect(); - try { - await client.query("BEGIN"); - const results: any[] = []; - for (const stmt of stmts) { - results.push(await (stmt as PgStatement)._runOn(client)); - } - await client.query("COMMIT"); - return results; - } catch (e) { - await client.query("ROLLBACK"); - throw e; - } finally { - client.release(); - } - }, - dialect: postgresDialect, - }; -} +export * from "@atmo-dev/contrail-base/postgres"; diff --git a/packages/contrail/src/adapters/sqlite.ts b/packages/contrail/src/adapters/sqlite.ts index 2f46dce..42032b3 100644 --- a/packages/contrail/src/adapters/sqlite.ts +++ b/packages/contrail/src/adapters/sqlite.ts @@ -1,39 +1 @@ -import { DatabaseSync } from "node:sqlite"; -import type { Database, Statement } from "../core/types"; -import { sqliteDialect } from "../core/dialect"; - -export function createSqliteDatabase(path: string): Database { - const raw = new DatabaseSync(path); - raw.exec("PRAGMA journal_mode = WAL"); - - function wrapStatement(sql: string, boundValues: any[] = []): Statement { - return { - bind(...values: any[]): Statement { - return wrapStatement(sql, values); - }, - async run() { - return raw.prepare(sql).run(...boundValues); - }, - async all<T>() { - return { results: raw.prepare(sql).all(...boundValues) as T[] }; - }, - async first<T>() { - return (raw.prepare(sql).get(...boundValues) as T) ?? null; - }, - }; - } - - return { - prepare(sql: string): Statement { - return wrapStatement(sql); - }, - async batch(stmts: Statement[]): Promise<any[]> { - const results: any[] = []; - for (const stmt of stmts) { - results.push(await stmt.run()); - } - return results; - }, - dialect: sqliteDialect, - }; -} +export * from "@atmo-dev/contrail-base/sqlite"; diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index 3d23dc1..713e23e 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -1,3 +1,4 @@ +import type {} from "@atcute/atproto"; import { type Did } from "@atcute/lexicons"; import { isDid, isNsid } from "@atcute/lexicons/syntax"; diff --git a/packages/contrail/src/core/client.ts b/packages/contrail/src/core/client.ts index f37bea3..1129419 100644 --- a/packages/contrail/src/core/client.ts +++ b/packages/contrail/src/core/client.ts @@ -1,194 +1 @@ -import { - CompositeDidDocumentResolver, - PlcDidDocumentResolver, - WebDidDocumentResolver, -} from "@atcute/identity-resolver"; -import { type Did } from "@atcute/lexicons"; -import { Client, simpleFetchHandler } from "@atcute/client"; -import type {} from "@atcute/atproto"; -import type { Database } from "./types"; - -// Slingshot-first PDS resolution with fallback to DID document resolution -const SLINGSHOT_URL = - "https://slingshot.microcosm.blue/xrpc/com.bad-example.identity.resolveMiniDoc"; - -export interface ResolvedIdentity { - did: string; - handle: string | null; - pds: string | null; -} - -/** Reject PDS URLs that point to private/internal addresses or non-HTTPS */ -function validatePdsUrl(url: string): boolean { - try { - const parsed = new URL(url); - if (parsed.protocol !== "https:") return false; - const host = parsed.hostname; - // Block private/internal IP ranges - if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; - if (host.startsWith("10.")) return false; - if (host.startsWith("192.168.")) return false; - if (host.startsWith("169.254.")) return false; - if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; - return true; - } catch { - return false; - } -} - -async function resolveViaSlingshot( - identifier: string -): Promise<ResolvedIdentity | undefined> { - const url = new URL(SLINGSHOT_URL); - url.searchParams.set("identifier", identifier); - - try { - const response = await fetch(url.toString()); - if (!response.ok) return undefined; - const data = (await response.json()) as { - did?: string; - handle?: string; - pds?: string; - }; - if (!data.did && !data.pds) return undefined; - return { - did: data.did ?? identifier, - handle: data.handle ?? null, - pds: data.pds ?? null, - }; - } catch { - return undefined; - } -} - -const didResolver = new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, -}); - -async function getPDSViaDidDoc(did: Did): Promise<string | undefined> { - const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); - return doc.service - ?.find((s) => s.id === "#atproto_pds") - ?.serviceEndpoint.toString(); -} - -/** - * Resolve identity info (did, handle, pds) for a DID or handle. - * Uses slingshot first, falls back to DID doc for PDS. - */ -export async function resolvePDS( - identifier: string -): Promise<ResolvedIdentity | undefined> { - const result = await resolveViaSlingshot(identifier); - if (result?.pds) { - if (!validatePdsUrl(result.pds)) return { ...result, pds: null }; - return result; - } - - // Fall back to DID doc resolution (only works for DIDs, not handles) - if (identifier.startsWith("did:")) { - try { - const pds = await getPDSViaDidDoc(identifier as Did); - if (pds && validatePdsUrl(pds)) { - return { - did: identifier, - handle: result?.handle ?? null, - pds, - }; - } - } catch { - // ignore - } - } - - return result; -} - -// In-memory PDS cache with TTL + size limit, plus in-flight deduplication -const PDS_CACHE_TTL = 60 * 60 * 1000; // 1 hour -const PDS_CACHE_MAX = 10_000; -const pdsCache = new Map<string, { pds: string; at: number }>(); -const pdsInflight = new Map<string, Promise<string | undefined>>(); - -function pdsCacheGet(did: string): string | undefined { - const entry = pdsCache.get(did); - if (!entry) return undefined; - if (Date.now() - entry.at > PDS_CACHE_TTL) { - pdsCache.delete(did); - return undefined; - } - return entry.pds; -} - -function pdsCacheSet(did: string, pds: string): void { - // Evict oldest entries if over limit - if (pdsCache.size >= PDS_CACHE_MAX) { - const first = pdsCache.keys().next().value; - if (first) pdsCache.delete(first); - } - pdsCache.set(did, { pds, at: Date.now() }); -} - -export async function getPDS( - did: Did, - db?: Database -): Promise<string | undefined> { - const mem = pdsCacheGet(did); - if (mem) return mem; - - // Deduplicate concurrent calls for the same DID - const inflight = pdsInflight.get(did); - if (inflight) return inflight; - - const promise = resolvePDSCached(did, db); - pdsInflight.set(did, promise); - try { - return await promise; - } finally { - pdsInflight.delete(did); - } -} - -async function resolvePDSCached( - did: Did, - db?: Database -): Promise<string | undefined> { - if (db) { - const cached = await db - .prepare("SELECT pds FROM identities WHERE did = ? AND pds IS NOT NULL") - .bind(did) - .first<{ pds: string }>(); - if (cached?.pds) { - pdsCacheSet(did, cached.pds); - return cached.pds; - } - } - - const resolved = await resolvePDS(did); - if (!resolved?.pds) return undefined; - - pdsCacheSet(did, resolved.pds); - - // Persist to DB for future runs - if (db) { - await db - .prepare( - "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET pds = excluded.pds, handle = COALESCE(excluded.handle, identities.handle), resolved_at = excluded.resolved_at" - ) - .bind(did, resolved.handle, resolved.pds, Date.now()) - .run(); - } - - return resolved.pds; -} - -export async function getClient(did: Did, db?: Database): Promise<Client> { - const pds = await getPDS(did, db); - if (!pds) throw new Error(`PDS not found for ${did}`); - return new Client({ - handler: simpleFetchHandler({ service: pds }), - }); -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/community-integration.ts b/packages/contrail/src/core/community-integration.ts index d91b3a4..1129419 100644 --- a/packages/contrail/src/core/community-integration.ts +++ b/packages/contrail/src/core/community-integration.ts @@ -1,54 +1 @@ -/** Pluggable integration surface for the community module. - * - * Phase 6 extracted community to its own package (`@atmo-dev/contrail-community`). - * The contrail core package never imports from it — couplings only flow - * through these interfaces. The community package's - * `createCommunityIntegration({ ... })` returns a {@link CommunityIntegration}, - * which the consumer hands to `createApp` via `options.community`. - * - * Two layers: - * - {@link CommunityProbe}: minimal "is this a community DID" / "what - * spaces does this caller reach" surface used by realtime + collection - * routes for community-aware dispatch. - * - {@link CommunityIntegration}: the umbrella bundle — probe, whoami - * extension, invite handler, plus route + schema wiring that the - * umbrella router calls during setup. */ - -import type { Hono, MiddlewareHandler } from "hono"; -import type { Database } from "./types"; -import type { CommunityInviteHandler } from "./invite/community-handler"; -import type { WhoamiExtension } from "./spaces/router"; - -/** Narrow interface for the deep callers (realtime/resolve, router/collection) - * that just need to ask "is this a community DID?" or "what spaces does this - * caller reach via community membership?" */ -export interface CommunityProbe { - /** Look up a community row by DID. Returns null for non-community DIDs. - * Callers usually only check truthiness — community-specific fields stay - * inside the community package. */ - getCommunity(did: string): Promise<{ did: string } | null>; - - /** Resolve the set of space URIs reachable by `callerDid` through community - * membership (direct grants + delegations). Used by realtime to expand - * community: topics into the caller's concrete space: topics. */ - resolveReachableSpaces(callerDid: string): Promise<Set<string>>; -} - -/** Umbrella integration the consumer constructs once and hands to createApp. - * contrail core treats this as an opaque bundle — it doesn't introspect - * community state, just calls these methods at the right wiring points. */ -export interface CommunityIntegration { - /** Probe used by realtime + collection cross-cutting concerns. */ - probe: CommunityProbe; - /** Whoami extension that returns `accessLevel` for community-owned spaces. */ - whoamiExtension: WhoamiExtension; - /** Handler for the community-grant path of the unified invite surface. */ - inviteHandler: CommunityInviteHandler; - /** Register `<ns>.community.*` routes onto the Hono app. */ - registerRoutes( - app: Hono, - options?: { authMiddleware?: MiddlewareHandler } - ): void; - /** Apply community schema (DDL) to the database. Called by initSchema. */ - applySchema(db: Database): Promise<void>; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/dialect.ts b/packages/contrail/src/core/dialect.ts index e3bb790..1129419 100644 --- a/packages/contrail/src/core/dialect.ts +++ b/packages/contrail/src/core/dialect.ts @@ -1,127 +1 @@ -/** Get the dialect from a Database, defaulting to SQLite (for D1 compatibility) */ -export function getDialect(db: { dialect?: SqlDialect }): SqlDialect { - return db.dialect ?? sqliteDialect; -} - -const SAFE_FIELD = /^[a-zA-Z0-9_.]+$/; - -function assertSafeField(field: string): void { - if (!SAFE_FIELD.test(field)) { - throw new Error(`Invalid field name: ${field}`); - } -} - -export interface SqlDialect { - /** json_extract(col, '$.field') or col->>'field' */ - jsonExtract(column: string, field: string): string; - - /** Convert INSERT INTO to ignore-duplicates form. - * SQLite: INSERT INTO → INSERT OR IGNORE INTO - * PG: appends ON CONFLICT DO NOTHING - * Accepts full SQL starting with "INSERT INTO" (works with both VALUES and SELECT). */ - insertOrIgnore(sql: string): string; - - /** Column type for the record column: TEXT (SQLite) or JSONB (PostgreSQL) */ - readonly recordColumnType: string; - - /** FTS strategy: 'virtual-table' (SQLite FTS5) or 'generated-column' (PG tsvector) */ - readonly ftsStrategy: "virtual-table" | "generated-column"; - - /** INTEGER type name — same on both, but PostgreSQL may want BIGINT for time_us */ - readonly integerType: string; - - /** BIGINT type name for timestamps */ - readonly bigintType: string; - - /** Wrap an expression for use in CREATE INDEX — PostgreSQL requires parens around expressions */ - indexExpression(expr: string): string; -} - -export const sqliteDialect: SqlDialect = { - jsonExtract(column: string, field: string): string { - assertSafeField(field); - return `json_extract(${column}, '$.${field}')`; - }, - - insertOrIgnore(sql: string): string { - return sql.replace(/^INSERT INTO/, "INSERT OR IGNORE INTO"); - }, - - recordColumnType: "TEXT", - ftsStrategy: "virtual-table", - integerType: "INTEGER", - bigintType: "INTEGER", - - indexExpression(expr: string): string { - return expr; - }, -}; - -export const postgresDialect: SqlDialect = { - jsonExtract(column: string, field: string): string { - assertSafeField(field); - const parts = field.split("."); - if (parts.length === 1) { - return `${column}->>'${parts[0]}'`; - } - // a.b.c → col->'a'->'b'->>'c' - const intermediate = parts.slice(0, -1).map((p) => `->'${p}'`).join(""); - return `${column}${intermediate}->>'${parts[parts.length - 1]}'`; - }, - - insertOrIgnore(sql: string): string { - return `${sql} ON CONFLICT DO NOTHING`; - }, - - recordColumnType: "JSONB", - ftsStrategy: "generated-column", - integerType: "INTEGER", - bigintType: "BIGINT", - - indexExpression(expr: string): string { - return `(${expr})`; - }, -}; - -/** Generate FTS schema statements based on dialect */ -export function buildFtsSchema( - dialect: SqlDialect, - recordsTable: string, - fields: string[] -): string[] { - if (dialect.ftsStrategy === "virtual-table") { - const ftsTable = recordsTable.replace("records_", "fts_"); - return [ - `CREATE VIRTUAL TABLE IF NOT EXISTS ${ftsTable} USING fts5(uri UNINDEXED, content)` - ]; - } else { - const concatExpr = fields - .map((f) => `COALESCE(${dialect.jsonExtract("record", f)}, '')`) - .join(" || ' ' || "); - return [ - `ALTER TABLE ${recordsTable} ADD COLUMN IF NOT EXISTS search_vector TSVECTOR GENERATED ALWAYS AS (to_tsvector('english', ${concatExpr})) STORED`, - `CREATE INDEX IF NOT EXISTS idx_${recordsTable}_search ON ${recordsTable} USING GIN (search_vector)`, - ]; - } -} - -/** Generate FTS query clause based on dialect */ -export function ftsQueryClause( - dialect: SqlDialect, - recordsTable: string -): { join: string; condition: string; orderExpr: string } { - if (dialect.ftsStrategy === "virtual-table") { - const ftsTable = recordsTable.replace("records_", "fts_"); - return { - join: `JOIN ${ftsTable} fts ON fts.uri = r.uri`, - condition: "fts.content MATCH ?", - orderExpr: "fts.rank", - }; - } else { - return { - join: "", - condition: "r.search_vector @@ plainto_tsquery('english', ?)", - orderExpr: "ts_rank(r.search_vector, plainto_tsquery('english', ?))", - }; - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/identity.ts b/packages/contrail/src/core/identity.ts index 05040fd..1129419 100644 --- a/packages/contrail/src/core/identity.ts +++ b/packages/contrail/src/core/identity.ts @@ -1,159 +1 @@ -import type { Did } from "@atcute/lexicons"; -import type { Database, Logger } from "./types"; -import { isDid, isHandle } from "@atcute/lexicons/syntax"; -import { resolvePDS } from "./client"; - -const STALE_MS = 24 * 60 * 60 * 1000; // 24 hours - -export interface Identity { - did: string; - handle: string | null; - pds: string | null; - resolved_at: number; -} - -async function saveIdentity(db: Database, identity: Identity): Promise<void> { - await db - .prepare( - "INSERT INTO identities (did, handle, pds, resolved_at) VALUES (?, ?, ?, ?) ON CONFLICT(did) DO UPDATE SET handle = excluded.handle, pds = excluded.pds, resolved_at = excluded.resolved_at" - ) - .bind(identity.did, identity.handle, identity.pds, identity.resolved_at) - .run(); -} - -function isStale(resolvedAt: number): boolean { - return Date.now() - resolvedAt >= STALE_MS; -} - -async function fetchAndSave( - db: Database, - identifier: string, - cached?: Identity | null -): Promise<Identity> { - const resolved = await resolvePDS(identifier); - const identity: Identity = { - did: resolved?.did ?? identifier, - handle: resolved?.handle ?? cached?.handle ?? null, - pds: resolved?.pds ?? cached?.pds ?? null, - resolved_at: Date.now(), - }; - await saveIdentity(db, identity); - return identity; -} - -export async function resolveIdentity( - db: Database, - did: Did -): Promise<Identity> { - const cached = await db - .prepare("SELECT did, handle, pds, resolved_at FROM identities WHERE did = ?") - .bind(did) - .first<Identity>(); - - if (cached && !isStale(cached.resolved_at)) return cached; - - return fetchAndSave(db, did, cached); -} - -export async function resolveIdentities( - db: Database, - dids: string[] -): Promise<Map<string, Identity>> { - const map = new Map<string, Identity>(); - if (dids.length === 0) return map; - - // Batch lookup from DB - const BATCH = 50; - for (let i = 0; i < dids.length; i += BATCH) { - const chunk = dids.slice(i, i + BATCH); - const placeholders = chunk.map(() => "?").join(","); - const rows = await db - .prepare(`SELECT did, handle, pds, resolved_at FROM identities WHERE did IN (${placeholders})`) - .bind(...chunk) - .all<Identity>(); - for (const row of rows.results ?? []) { - map.set(row.did, row); - } - } - - // Resolve missing via slingshot directly (no redundant DB lookup) - for (const did of dids) { - if (map.has(did) || !isDid(did)) continue; - try { - const identity = await fetchAndSave(db, did); - map.set(did, identity); - } catch { - // Silently skip unresolvable identities - } - } - - return map; -} - -export async function resolveActor( - db: Database, - actor: string -): Promise<string | null> { - if (isDid(actor)) return actor; - if (!isHandle(actor)) return null; - - // Look up handle in identities table - const cached = await db - .prepare("SELECT did, resolved_at FROM identities WHERE handle = ?") - .bind(actor) - .first<{ did: string; resolved_at: number }>(); - - if (cached && !isStale(cached.resolved_at)) return cached.did; - - // Resolve via slingshot - const resolved = await resolvePDS(actor); - if (!resolved?.did || !isDid(resolved.did)) return null; - - await saveIdentity(db, { - did: resolved.did, - handle: resolved.handle ?? actor, - pds: resolved.pds ?? null, - resolved_at: Date.now(), - }); - - return resolved.did; -} - -export async function refreshStaleIdentities( - db: Database, - dids: string[] -): Promise<void> { - if (dids.length === 0) return; - - const unique = [...new Set(dids)].filter(isDid); - if (unique.length === 0) return; - - const staleThreshold = Date.now() - STALE_MS; - const toRefresh: string[] = []; - - const BATCH = 50; - for (let i = 0; i < unique.length; i += BATCH) { - const chunk = unique.slice(i, i + BATCH); - const placeholders = chunk.map(() => "?").join(","); - const rows = await db - .prepare(`SELECT did, resolved_at FROM identities WHERE did IN (${placeholders})`) - .bind(...chunk) - .all<{ did: string; resolved_at: number }>(); - - const found = new Map((rows.results ?? []).map((r) => [r.did, r.resolved_at])); - for (const did of chunk) { - const resolvedAt = found.get(did); - if (resolvedAt === undefined || resolvedAt < staleThreshold) { - toRefresh.push(did); - } - } - } - - for (const did of toRefresh) { - try { - await fetchAndSave(db, did); - } catch { - // Silently skip unresolvable identities - } - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/invite/community-handler.ts b/packages/contrail/src/core/invite/community-handler.ts index 1043525..1129419 100644 --- a/packages/contrail/src/core/invite/community-handler.ts +++ b/packages/contrail/src/core/invite/community-handler.ts @@ -1,67 +1 @@ -/** Pluggable handler for community-grant invites within the unified invite - * surface. The invite router calls into this when the target space is - * community-owned, or "tries" it on the redeem / revoke-without-spaceUri - * paths. Community module provides the impl; invite/router doesn't import - * from community at all. - * - * Each method returns a `HandlerResponse`: a `{status, body}` envelope that - * the router relays as JSON, or `null` (only on the "try" methods) meaning - * "not applicable, fall through to the user-owned path." */ - -export type HandlerResponse = { - status: number; - body: Record<string, unknown>; -}; - -export interface CommunityInviteHandler { - /** True iff this space is owned by a community (vs. a regular user DID). - * Used by the invite router to choose the dispatch path on - * create / list / revoke-with-spaceUri. */ - isCommunityOwned(spaceUri: string): Promise<boolean>; - - /** Create a community-grant invite. Caller is validated upstream for - * having a JWT; this method handles the access-level checks. */ - create(input: { - spaceUri: string; - callerDid: string; - /** Raw caller-supplied access level — implementation validates. */ - accessLevel?: string; - /** Caller-supplied `kind` field — community spaces don't accept this; the - * handler returns an InvalidRequest if set. */ - kind?: string; - expiresAt: number | null; - maxUses: number | null; - note: string | null; - }): Promise<HandlerResponse>; - - /** List invites for a community-owned space. */ - list(input: { - spaceUri: string; - callerDid: string; - includeRevoked: boolean; - }): Promise<HandlerResponse>; - - /** Revoke a known community-owned invite (caller already passed spaceUri - * and the router classified it as community-owned). */ - revoke(input: { - spaceUri: string; - tokenHash: string; - callerDid: string; - }): Promise<HandlerResponse>; - - /** Revoke without a spaceUri — try to find the invite in the community - * table; return null if not a community invite (router falls through). */ - tryRevokeByToken(input: { - tokenHash: string; - callerDid: string; - }): Promise<HandlerResponse | null>; - - /** Try to redeem a token as a community invite. Returns null if the token - * is not a community invite, in which case the router falls through to - * the user-owned redeem path. */ - tryRedeem(input: { - tokenHash: string; - callerDid: string; - now: number; - }): Promise<HandlerResponse | null>; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/invite/token.ts b/packages/contrail/src/core/invite/token.ts index 76f4880..1129419 100644 --- a/packages/contrail/src/core/invite/token.ts +++ b/packages/contrail/src/core/invite/token.ts @@ -1,43 +1 @@ -const B64U_ALPHABET = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; - -function bytesToB64Url(bytes: Uint8Array): string { - let out = ""; - for (let i = 0; i < bytes.length; i += 3) { - const b0 = bytes[i]; - const b1 = bytes[i + 1] ?? 0; - const b2 = bytes[i + 2] ?? 0; - out += B64U_ALPHABET[b0 >> 2]; - out += B64U_ALPHABET[((b0 & 3) << 4) | (b1 >> 4)]; - if (i + 1 < bytes.length) out += B64U_ALPHABET[((b1 & 15) << 2) | (b2 >> 6)]; - if (i + 2 < bytes.length) out += B64U_ALPHABET[b2 & 63]; - } - return out; -} - -function bytesToHex(bytes: Uint8Array): string { - let out = ""; - for (let i = 0; i < bytes.length; i++) out += bytes[i].toString(16).padStart(2, "0"); - return out; -} - -/** Generate a fresh invite token (cryptographically random, 32 bytes base64url-encoded). */ -export function generateInviteToken(): string { - const bytes = new Uint8Array(32); - crypto.getRandomValues(bytes); - return bytesToB64Url(bytes); -} - -/** SHA-256 hash of a token, hex-encoded. Used as the PK in storage so raw tokens are never persisted. */ -export async function hashInviteToken(token: string): Promise<string> { - const encoded = new TextEncoder().encode(token); - const digest = await crypto.subtle.digest("SHA-256", encoded); - return bytesToHex(new Uint8Array(digest)); -} - -/** Convenience: generate a token and return both the raw form (returned to - * the creator once) and its hash (persisted as the stable ID). */ -export async function mintInviteToken(): Promise<{ token: string; tokenHash: string }> { - const token = generateInviteToken(); - const tokenHash = await hashInviteToken(token); - return { token, tokenHash }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/labels/types.ts b/packages/contrail/src/core/labels/types.ts index f8bc2e6..1129419 100644 --- a/packages/contrail/src/core/labels/types.ts +++ b/packages/contrail/src/core/labels/types.ts @@ -1,59 +1 @@ -import type { Database } from "../types"; - -/** A labeler the operator wants contrail to track. */ -export interface LabelerSource { - /** Labeler DID — `did:plc:...` or `did:web:...`. */ - did: string; - /** Override the service endpoint resolution. Otherwise resolved from the - * DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ - endpoint?: string; - /** Backfill from `cursor=0` on first sight. Defaults to true. Set false - * for "start from now" — useful for very chatty labelers. */ - backfill?: boolean; -} - -export interface LabelsConfig { - /** Labelers to subscribe to and index. */ - sources: LabelerSource[]; - /** DIDs honored when the caller sends no `atproto-accept-labelers` / - * `?labelers=`. Defaults to every entry in `sources`. Set `[]` for - * opt-in-only — clients see no labels unless they ask. */ - defaults?: string[]; - /** Per-request cap. Default: 20 (matches Bluesky). */ - maxPerRequest?: number; -} - -export const DEFAULT_LABELS_MAX_PER_REQUEST = 20; - -/** A single label as stored. Matches `com.atproto.label.defs#label`. */ -export interface LabelRow { - /** Issuing labeler DID. */ - src: string; - /** Subject — at-URI for record labels, plain DID for account labels. */ - uri: string; - /** Label value — kebab-case, ≤128 bytes per spec. */ - val: string; - /** Optional CID pin to a specific record version. */ - cid: string | null; - /** When true, retracts a previously-emitted label for the same (src, uri, val). */ - neg: boolean; - /** Expiry, unix seconds. Past this, hydration drops the row. */ - exp: number | null; - /** Creation timestamp, unix seconds — what we collapse on. */ - cts: number; - /** Raw signature bytes. Stored when present so we can re-emit later; - * not verified in v1. */ - sig: Uint8Array | null; -} - -/** Per-labeler state row — endpoint cache and last-seen seq cursor. */ -export interface LabelerCursorRow { - did: string; - cursor: number; - endpoint: string | null; - resolved_at: number | null; -} - -export interface AdapterContext { - db: Database; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/durable-object.ts b/packages/contrail/src/core/realtime/durable-object.ts index 966ab53..1129419 100644 --- a/packages/contrail/src/core/realtime/durable-object.ts +++ b/packages/contrail/src/core/realtime/durable-object.ts @@ -1,567 +1 @@ -/** Durable Object backend for realtime PubSub. - * - * Two pieces live here: - * 1. `RealtimePubSubDO` — the DO class. Ship it from your Worker via - * `export { RealtimePubSubDO } from "@atmo-dev/contrail";` and bind it in - * your `wrangler.toml`. One DO = one topic; addressed by name. - * 2. `DurableObjectPubSub` — client-side adapter implementing the PubSub - * interface against a DO namespace binding. - * - * Wire format between Worker and DO (internal, not a stable public contract): - * POST /publish — body = RealtimeEvent JSON - * GET /subscribe — server-sent events stream, optionally with - * `Upgrade: websocket` for WS connections. - * Auth/ACL is already checked at the Worker edge; - * the DO trusts anything that reaches it. */ - -import type { PubSub, RealtimeEvent } from "./types"; -import { translateForQuery, type TranslatedEnvelope } from "./query-filter"; -type TranslatedEvent = TranslatedEnvelope; - -/** Query spec attached to a WS subscriber, used to filter events before - * delivery. Shape matches what the Worker's `watchRecords` handler builds; - * forwarded to the DO via trusted internal headers on the WS upgrade. */ -export interface SubscriberQuerySpec { - /** NSID of the primary collection the client is watching. */ - collection: string; - /** Space URI this subscription is scoped to. Events outside are dropped. */ - spaceUri: string; - /** Hydrated relations. Keyed by relName — value is the child collection - * NSID and the field on the child record that references the parent. */ - hydrate?: Record<string, { childCollection: string; matchField: string }>; -} - -// ---- Minimal structural typings so we don't depend on @cloudflare/workers-types -// at the library level. Callers on Workers will have proper types. -// ---------------------------------------------------------------------------- - -export interface DurableObjectId { - toString(): string; -} - -export interface DurableObjectStub { - fetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response>; -} - -export interface DurableObjectNamespace { - idFromName(name: string): DurableObjectId; - get(id: DurableObjectId): DurableObjectStub; -} - -export interface DurableObjectState { - acceptWebSocket(ws: any, tags?: string[]): void; - getWebSockets(tag?: string): any[]; -} - -// ---------------------------------------------------------------------------- -// Client adapter -// ---------------------------------------------------------------------------- - -export class DurableObjectPubSub implements PubSub { - constructor(private readonly namespace: DurableObjectNamespace) {} - - private stub(topic: string): DurableObjectStub { - return this.namespace.get(this.namespace.idFromName(topic)); - } - - async publish(event: RealtimeEvent): Promise<void> { - const res = await this.stub(event.topic).fetch("https://do/publish", { - method: "POST", - headers: { "content-type": "application/json" }, - body: JSON.stringify(event), - }); - if (!res.ok) { - // Consume the body so the edge doesn't hold a dangling response. - await res.text().catch(() => ""); - throw new Error(`DO publish failed (${res.status})`); - } - } - - /** In-Worker server-side subscribe. Browsers should hit the SSE endpoint - * directly; this is the path for an in-process consumer that wants an - * AsyncIterable (tests, bots embedded in the Worker). */ - subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { - const stub = this.stub(topic); - return { - [Symbol.asyncIterator]() { - return pullIterator(stub, signal); - }, - }; - } - - /** Forward an incoming browser WS upgrade (or SSE GET) through to the DO - * that owns this topic, attaching a query-filter spec that the DO will use - * to decide what to deliver. The Worker must verify auth + spec validity - * before calling this — the DO trusts the headers. */ - async forwardSubscribe( - topic: string, - request: Request, - opts: { - did?: string; - querySpec?: SubscriberQuerySpec; - /** Unix ms. DO replays any buffered event with ts > sinceTs before - * going live — closes the snapshot→WS race window on the client. */ - sinceTs?: number; - } = {} - ): Promise<Response> { - const headers = new Headers(request.headers); - if (opts.querySpec) { - headers.set("X-Contrail-Query-Spec", JSON.stringify(opts.querySpec)); - } - const url = new URL("https://do/subscribe"); - if (opts.did) url.searchParams.set("did", opts.did); - if (opts.sinceTs && opts.sinceTs > 0) { - url.searchParams.set("sinceTs", String(opts.sinceTs)); - } - return this.stub(topic).fetch(url.toString(), { - method: "GET", - headers - }); - } -} - -function pullIterator( - stub: DurableObjectStub, - signal?: AbortSignal -): AsyncIterator<RealtimeEvent> { - let reader: ReadableStreamDefaultReader<Uint8Array> | null = null; - let buf = ""; - const decoder = new TextDecoder(); - const ac = new AbortController(); - if (signal) { - if (signal.aborted) ac.abort(); - else signal.addEventListener("abort", () => ac.abort(), { once: true }); - } - - const open = async () => { - const res = await stub.fetch("https://do/subscribe", { - method: "GET", - headers: { accept: "text/event-stream" }, - signal: ac.signal, - }); - if (!res.ok || !res.body) throw new Error(`DO subscribe failed (${res.status})`); - reader = res.body.getReader(); - }; - - return { - async next(): Promise<IteratorResult<RealtimeEvent>> { - if (!reader) await open(); - while (true) { - // Drain buffered frames. - while (true) { - const sep = buf.indexOf("\n\n"); - if (sep < 0) break; - const frame = buf.slice(0, sep); - buf = buf.slice(sep + 2); - let data: string | null = null; - for (const line of frame.split("\n")) { - if (line.startsWith(":")) continue; - if (line.startsWith("data:")) data = line.slice(5).trim(); - } - if (data) { - try { - return { value: JSON.parse(data) as RealtimeEvent, done: false }; - } catch { - /* skip malformed */ - } - } - } - if (ac.signal.aborted) return { value: undefined, done: true }; - const r = await reader!.read(); - if (r.done) return { value: undefined, done: true }; - buf += decoder.decode(r.value, { stream: true }); - } - }, - async return(): Promise<IteratorResult<RealtimeEvent>> { - ac.abort(); - try { - await reader?.cancel(); - } catch { - /* ignore */ - } - return { value: undefined, done: true }; - }, - }; -} - -// ---------------------------------------------------------------------------- -// Durable Object class -// ---------------------------------------------------------------------------- - -/** The Durable Object implementation. Each DO instance owns the fan-out for - * exactly one topic. WebSocket connections are stored via the Hibernation - * API (`state.acceptWebSocket`) so idle rooms cost near-zero. - * - * This class intentionally avoids the `DurableObject` base class so we don't - * have to depend on @cloudflare/workers-types at the library level — users - * wire it up directly in their Worker entry. */ -/** Rolling buffer of recent events, used to close the snapshot→WS race: - * when a new subscriber connects with `?sinceTs=X`, replay any buffered - * event with `event.ts > X` before going live. Bounded by count + age so - * memory stays small. */ -const RECENT_BUFFER_MS = 15_000; -const RECENT_BUFFER_MAX = 500; - -export class RealtimePubSubDO { - private readonly recentEvents: RealtimeEvent[] = []; - - constructor( - protected readonly state: DurableObjectState, - _env?: unknown - ) {} - - private pushRecent(event: RealtimeEvent): void { - this.recentEvents.push(event); - const cutoff = Date.now() - RECENT_BUFFER_MS; - while ( - this.recentEvents.length > RECENT_BUFFER_MAX || - (this.recentEvents.length > 0 && this.recentEvents[0]!.ts < cutoff) - ) { - this.recentEvents.shift(); - } - } - - /** Worker entry delegates `fetch` to this method. */ - async fetch(request: Request): Promise<Response> { - const url = new URL(request.url); - if (request.method === "POST" && url.pathname === "/publish") { - let event: RealtimeEvent; - try { - event = (await request.json()) as RealtimeEvent; - } catch { - return new Response(JSON.stringify({ error: "InvalidRequest" }), { status: 400 }); - } - this.publishEvent(event); - return new Response("{}", { status: 200 }); - } - if (request.method === "GET" && url.pathname === "/subscribe") { - const did = url.searchParams.get("did") ?? undefined; - const sinceTsRaw = url.searchParams.get("sinceTs"); - const sinceTs = sinceTsRaw ? Number(sinceTsRaw) : 0; - // Optional query-filter spec, forwarded by the Worker after it has - // verified the caller's auth + access. Parsed once here; the parsed - // object is serialized into the WS attachment so the DO can filter - // events on publish without re-parsing. - let querySpec: SubscriberQuerySpec | undefined; - const rawSpec = request.headers.get("X-Contrail-Query-Spec"); - if (rawSpec) { - try { - querySpec = JSON.parse(rawSpec) as SubscriberQuerySpec; - } catch { - return new Response( - JSON.stringify({ error: "InvalidRequest", message: "bad X-Contrail-Query-Spec" }), - { status: 400 } - ); - } - } - - if (request.headers.get("Upgrade")?.toLowerCase() === "websocket") { - const Pair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; - if (!Pair) return new Response("websockets require Workers", { status: 426 }); - const pair = new Pair(); - this.acceptWebSocketSubscriber(pair[1], did, querySpec); - if (sinceTs > 0) this.replayRecentTo(pair[1], sinceTs); - return new Response(null, { - status: 101, - // Workers-specific init field - webSocket: pair[0], - } as ResponseInit & { webSocket: unknown }); - } - return this.openSseResponse(did, querySpec, sinceTs); - } - return new Response("not found", { status: 404 }); - } - - /** Fan-out an event to every connected subscriber (WS + SSE). - * Public so tests + advanced callers can skip the HTTP layer. - * - * If a subscriber has attached a `querySpec`, we translate the raw event - * into 0–1 watchRecords-shaped events (record.created, record.deleted, - * hydration.added, hydration.removed) and deliver only those. Otherwise - * the raw event is delivered as-is (topic-firehose behaviour for the - * `realtime.subscribe` endpoint). */ - publishEvent(event: RealtimeEvent): void { - // Buffer first so a subscriber connecting mid-publish (race-window - // replay) can pick up this event too once they provide their sinceTs. - this.pushRecent(event); - - const rawPayload = JSON.stringify(event); - const rawFrame = `event: ${event.kind}\ndata: ${rawPayload}\n\n`; - - for (const ws of this.state.getWebSockets()) { - const attachment = getAttachment(ws); - - if (attachment?.querySpec) { - const translated = translateForQuery(event, attachment); - if (translated) this.writeSubscriberState(ws, attachment, translated); - for (const msg of translated ?? []) { - try { - ws.send(JSON.stringify(msg)); - } catch { - /* ignore */ - } - } - } else { - try { - ws.send(rawPayload); - } catch { - /* ignore */ - } - } - - if ( - event.kind === "member.removed" && - attachment?.did && - event.payload.did === attachment.did - ) { - try { - ws.close(4003, "membership-revoked"); - } catch { - /* ignore */ - } - } - } - - for (const entry of this.sseControllers) { - if (entry.querySpec) { - const translated = translateForQuery(event, entry); - if (translated) this.writeSubscriberStateForSse(entry, translated); - for (const msg of translated ?? []) { - try { - entry.controller.enqueue( - this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) - ); - } catch { - /* drop */ - } - } - } else { - try { - entry.controller.enqueue(this.encoder.encode(rawFrame)); - } catch { - /* drop; cleanup happens on the subscribe-side */ - } - } - - if ( - event.kind === "member.removed" && - entry.did && - event.payload.did === entry.did - ) { - try { - entry.controller.close(); - } catch { - /* ignore */ - } - } - } - } - - /** Register a server-side WebSocket as a subscriber. Wires the DID + - * optional query spec into the hibernation attachment so this DO can - * filter and route events after going to sleep. */ - acceptWebSocketSubscriber( - serverWs: any, - did?: string, - querySpec?: SubscriberQuerySpec - ): void { - this.state.acceptWebSocket(serverWs, did ? [did] : undefined); - if (did || querySpec) { - setAttachment(serverWs, { - did, - querySpec, - parentUris: [], - childToParent: {} - }); - } - } - - /** Open an SSE subscriber; returns the streaming Response. */ - openSseResponse( - did?: string, - querySpec?: SubscriberQuerySpec, - sinceTs = 0 - ): Response { - let entry: SseEntry; - const stream = new ReadableStream<Uint8Array>({ - start: (controller) => { - entry = { - controller, - did, - querySpec, - parentUris: new Set(), - childToParent: new Map() - }; - this.sseControllers.add(entry); - controller.enqueue(this.encoder.encode(`: open\n\n`)); - if (sinceTs > 0) this.replayRecentToSse(entry, sinceTs); - }, - cancel: () => { - this.sseControllers.delete(entry); - }, - }); - return new Response(stream, { - status: 200, - headers: { - "content-type": "text/event-stream", - "cache-control": "no-cache, no-transform", - connection: "keep-alive", - }, - }); - } - - /** Replay buffered events with ts > sinceTs through this subscriber's - * query-spec filter. Called once, synchronously, on WS connect. */ - private replayRecentTo(ws: any, sinceTs: number): void { - const attachment = getAttachment(ws); - for (const event of this.recentEvents) { - if (event.ts <= sinceTs) continue; - if (attachment?.querySpec) { - const translated = translateForQuery(event, attachment); - if (translated) this.writeSubscriberState(ws, attachment, translated); - for (const msg of translated ?? []) { - try { - ws.send(JSON.stringify(msg)); - } catch { - /* ignore */ - } - } - } else { - try { - ws.send(JSON.stringify(event)); - } catch { - /* ignore */ - } - } - } - } - - private replayRecentToSse(entry: SseEntry, sinceTs: number): void { - for (const event of this.recentEvents) { - if (event.ts <= sinceTs) continue; - if (entry.querySpec) { - const translated = translateForQuery(event, entry); - if (translated) this.writeSubscriberStateForSse(entry, translated); - for (const msg of translated ?? []) { - try { - entry.controller.enqueue( - this.encoder.encode(`event: ${msg.kind}\ndata: ${JSON.stringify(msg.data)}\n\n`) - ); - } catch { - /* drop */ - } - } - } else { - try { - entry.controller.enqueue( - this.encoder.encode(`event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`) - ); - } catch { - /* drop */ - } - } - } - } - - /** Update the persisted WS attachment state after we've decided which - * events to forward. Keeps the parent/child tracking tables warm across - * hibernation. */ - private writeSubscriberState( - ws: any, - attachment: WsAttachment, - translated: TranslatedEvent[] - ): void { - let dirty = false; - for (const msg of translated) { - if (msg.kind === "record.created" && msg.data.record?.uri) { - attachment.parentUris = Array.from( - new Set([...(attachment.parentUris ?? []), msg.data.record.uri]) - ); - dirty = true; - } else if (msg.kind === "record.deleted" && msg.data.uri) { - const before = attachment.parentUris ?? []; - attachment.parentUris = before.filter((u) => u !== msg.data.uri); - if (attachment.parentUris.length !== before.length) dirty = true; - } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { - attachment.childToParent = { - ...(attachment.childToParent ?? {}), - [msg.data.child.rkey]: { - parentUri: msg.data.parentUri, - relName: msg.data.relation - } - }; - dirty = true; - } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { - const next = { ...(attachment.childToParent ?? {}) }; - if (next[msg.data.childRkey]) { - delete next[msg.data.childRkey]; - attachment.childToParent = next; - dirty = true; - } - } - } - if (dirty) setAttachment(ws, attachment); - } - - private writeSubscriberStateForSse( - entry: SseEntry, - translated: TranslatedEvent[] - ): void { - for (const msg of translated) { - if (msg.kind === "record.created" && msg.data.record?.uri) { - entry.parentUris?.add(msg.data.record.uri); - } else if (msg.kind === "record.deleted" && msg.data.uri) { - entry.parentUris?.delete(msg.data.uri); - } else if (msg.kind === "hydration.added" && msg.data.child?.rkey) { - entry.childToParent?.set(msg.data.child.rkey, { - parentUri: msg.data.parentUri, - relName: msg.data.relation - }); - } else if (msg.kind === "hydration.removed" && msg.data.childRkey) { - entry.childToParent?.delete(msg.data.childRkey); - } - } - } - - private readonly sseControllers = new Set<SseEntry>(); - private readonly encoder = new TextEncoder(); -} - -interface SseEntry { - controller: ReadableStreamDefaultController<Uint8Array>; - did: string | undefined; - querySpec?: SubscriberQuerySpec; - parentUris?: Set<string>; - childToParent?: Map<string, { parentUri: string; relName: string }>; -} - -interface WsAttachment { - did?: string; - querySpec?: SubscriberQuerySpec; - /** URIs of primary records currently in this subscriber's result set. */ - parentUris?: string[]; - /** childRkey → parent info, for routing child delete events. */ - childToParent?: Record<string, { parentUri: string; relName: string }>; -} - -function setAttachment(ws: any, attachment: WsAttachment): void { - try { - ws.serializeAttachment?.(attachment); - } catch { - /* non-hibernating socket — fall back to a direct property */ - ws.__attachment = attachment; - } -} - -function getAttachment(ws: any): WsAttachment | null { - try { - const a = ws.deserializeAttachment?.(); - if (a) return a as WsAttachment; - } catch { - /* ignore */ - } - return (ws.__attachment as WsAttachment | undefined) ?? null; -} - -// Query-spec filtering lives in ./query-filter so the Worker can reuse it for -// non-DO (InMemoryPubSub) watchRecords paths without bundling the whole DO. +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/in-memory.ts b/packages/contrail/src/core/realtime/in-memory.ts index 311c0a5..1129419 100644 --- a/packages/contrail/src/core/realtime/in-memory.ts +++ b/packages/contrail/src/core/realtime/in-memory.ts @@ -1,116 +1 @@ -import type { PubSub, RealtimeEvent } from "./types"; -import { DEFAULT_QUEUE_BOUND } from "./types"; - -/** Single-process PubSub backed by in-memory subscriber sets. - * - * Each subscriber owns a bounded queue; when full, oldest events are dropped. - * `publish` returns once every subscriber has been offered the event — it - * never awaits a subscriber's consumption, so a slow consumer can't block - * producers. The cost of that guarantee is the drop-oldest policy. */ -export class InMemoryPubSub implements PubSub { - private readonly subscribers = new Map<string, Set<Subscriber>>(); - private readonly queueBound: number; - - constructor(opts: { queueBound?: number } = {}) { - this.queueBound = opts.queueBound ?? DEFAULT_QUEUE_BOUND; - } - - async publish(event: RealtimeEvent): Promise<void> { - const set = this.subscribers.get(event.topic); - if (!set) return; - for (const sub of set) sub.push(event); - } - - subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent> { - const sub = new Subscriber(this.queueBound); - let set = this.subscribers.get(topic); - if (!set) { - set = new Set(); - this.subscribers.set(topic, set); - } - set.add(sub); - - const cleanup = () => { - sub.close(); - const s = this.subscribers.get(topic); - if (s) { - s.delete(sub); - if (s.size === 0) this.subscribers.delete(topic); - } - }; - - if (signal) { - if (signal.aborted) cleanup(); - else signal.addEventListener("abort", cleanup, { once: true }); - } - - return sub.iterate(cleanup); - } - - /** Test-only: current subscriber count for a topic. */ - subscriberCount(topic: string): number { - return this.subscribers.get(topic)?.size ?? 0; - } -} - -class Subscriber { - private readonly queue: RealtimeEvent[] = []; - private pending: ((v: RealtimeEvent | null) => void) | null = null; - private closed = false; - /** Number of events dropped because the queue was full. The consumer can - * observe a gap by comparing monotonic event timestamps; exposing the - * count on a side channel is future work. */ - public droppedCount = 0; - - constructor(private readonly bound: number) {} - - push(event: RealtimeEvent): void { - if (this.closed) return; - if (this.pending) { - const p = this.pending; - this.pending = null; - p(event); - return; - } - if (this.queue.length >= this.bound) { - this.queue.shift(); - this.droppedCount += 1; - } - this.queue.push(event); - } - - close(): void { - if (this.closed) return; - this.closed = true; - if (this.pending) { - const p = this.pending; - this.pending = null; - p(null); - } - } - - iterate(cleanup: () => void): AsyncIterable<RealtimeEvent> { - const self = this; - return { - [Symbol.asyncIterator]() { - return { - async next(): Promise<IteratorResult<RealtimeEvent>> { - if (self.queue.length > 0) { - return { value: self.queue.shift()!, done: false }; - } - if (self.closed) return { value: undefined, done: true }; - const event = await new Promise<RealtimeEvent | null>((resolve) => { - self.pending = resolve; - }); - if (event === null) return { value: undefined, done: true }; - return { value: event, done: false }; - }, - async return(): Promise<IteratorResult<RealtimeEvent>> { - cleanup(); - return { value: undefined, done: true }; - }, - }; - }, - }; - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/merge.ts b/packages/contrail/src/core/realtime/merge.ts index e0fa7df..1129419 100644 --- a/packages/contrail/src/core/realtime/merge.ts +++ b/packages/contrail/src/core/realtime/merge.ts @@ -1,77 +1 @@ -/** Merge N AsyncIterables into one, interleaving events as they arrive. - * Terminates when every source terminates, or when `signal` aborts. */ - -export function mergeAsyncIterables<T>( - sources: AsyncIterable<T>[], - signal?: AbortSignal -): AsyncIterable<T> { - if (sources.length === 0) { - return { - async *[Symbol.asyncIterator]() { - /* nothing to yield */ - }, - }; - } - - return { - [Symbol.asyncIterator]() { - const iterators = sources.map((s) => s[Symbol.asyncIterator]()); - // One in-flight next() per source, racing each other. - type Slot = { - idx: number; - promise: Promise<{ idx: number; result: IteratorResult<T> }>; - }; - const pending = new Map<number, Slot>(); - let doneCount = 0; - - const schedule = (idx: number) => { - const slot: Slot = { - idx, - promise: iterators[idx]! - .next() - .then((result) => ({ idx, result })), - }; - pending.set(idx, slot); - }; - - for (let i = 0; i < iterators.length; i++) schedule(i); - - const cleanup = () => { - for (const it of iterators) { - try { - it.return?.(); - } catch { - /* ignore */ - } - } - }; - - if (signal) { - if (signal.aborted) cleanup(); - else signal.addEventListener("abort", cleanup, { once: true }); - } - - return { - async next(): Promise<IteratorResult<T>> { - while (pending.size > 0) { - const slots = [...pending.values()]; - const { idx, result } = await Promise.race(slots.map((s) => s.promise)); - pending.delete(idx); - if (result.done) { - doneCount += 1; - if (doneCount === iterators.length) return { value: undefined, done: true }; - continue; - } - schedule(idx); - return { value: result.value, done: false }; - } - return { value: undefined, done: true }; - }, - async return(): Promise<IteratorResult<T>> { - cleanup(); - return { value: undefined, done: true }; - }, - }; - }, - }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/query-filter.ts b/packages/contrail/src/core/realtime/query-filter.ts index 1b79eb6..1129419 100644 --- a/packages/contrail/src/core/realtime/query-filter.ts +++ b/packages/contrail/src/core/realtime/query-filter.ts @@ -1,235 +1 @@ -/** Shared query-spec → event-translation logic. - * - * Used by: - * - the Durable Object's WS publish path (per-subscriber filter after hibernation) - * - the Worker's SSE / Worker-terminated WS path (in-process filter) - * - * Given a raw RealtimeEvent and a SubscriberQuerySpec, returns the list of - * `{kind, data}` envelopes to send to the subscriber, or `null` if the - * subscriber has no spec (i.e. raw-firehose mode). */ - -import type { RealtimeEvent } from "./types"; -import type { SubscriberQuerySpec } from "./durable-object"; - -export type TranslatedEnvelope = - | { - kind: "record.created"; - data: { - record: { - uri: string; - did: string; - rkey: string; - collection: string; - cid: string | null | undefined; - record: Record<string, unknown>; - time_us: number; - indexed_at: number; - space: string; - }; - }; - } - | { kind: "record.deleted"; data: { uri: string; did: string; rkey: string } } - | { - kind: "hydration.added"; - data: { - parentUri: string; - relation: string; - child: { - uri: string; - did: string; - rkey: string; - collection: string; - cid: string | null | undefined; - record: Record<string, unknown>; - space: string; - }; - }; - } - | { - kind: "hydration.removed"; - data: { - parentUri: string; - relation: string; - childRkey: string; - childDid?: string; - }; - }; - -export interface SubscriberView { - querySpec?: SubscriberQuerySpec; - parentUris?: Set<string> | string[]; - childToParent?: - | Map<string, { parentUri: string; relName: string }> - | Record<string, { parentUri: string; relName: string }>; -} - -export function translateForQuery( - event: RealtimeEvent, - sub: SubscriberView -): TranslatedEnvelope[] | null { - const spec = sub.querySpec; - if (!spec) return null; - if (event.kind !== "record.created" && event.kind !== "record.deleted") return []; - if (event.payload.space !== spec.spaceUri) return []; - - const primaryUri = `at://${event.payload.did}/${event.payload.collection}/${event.payload.rkey}`; - - if (event.payload.collection === spec.collection) { - if (event.kind === "record.created") { - return [ - { - kind: "record.created", - data: { - record: { - uri: primaryUri, - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - record: event.payload.record, - time_us: event.ts * 1000, - indexed_at: event.ts, - space: spec.spaceUri - } - } - } - ]; - } - return [ - { - kind: "record.deleted", - data: { - uri: primaryUri, - did: event.payload.did, - rkey: event.payload.rkey - } - } - ]; - } - - if (!spec.hydrate) return []; - for (const [relName, rel] of Object.entries(spec.hydrate)) { - if (rel.childCollection !== event.payload.collection) continue; - if (event.kind === "record.created") { - const parentUri = getNestedValue( - event.payload.record as Record<string, unknown>, - rel.matchField - ); - if (typeof parentUri !== "string") continue; - if (!hasParent(sub.parentUris, parentUri)) continue; - return [ - { - kind: "hydration.added", - data: { - parentUri, - relation: relName, - child: { - uri: primaryUri, - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - record: event.payload.record, - space: spec.spaceUri - } - } - } - ]; - } - const info = getChildInfo(sub.childToParent, event.payload.rkey); - if (!info || info.relName !== relName) continue; - return [ - { - kind: "hydration.removed", - data: { - parentUri: info.parentUri, - relation: relName, - childRkey: event.payload.rkey, - childDid: event.payload.did - } - } - ]; - } - return []; -} - -export function applyEnvelopesToSubscriber( - subscriber: SubscriberView, - envs: TranslatedEnvelope[] -): void { - for (const msg of envs) { - if (msg.kind === "record.created") { - ensureParentSet(subscriber).add(msg.data.record.uri); - } else if (msg.kind === "record.deleted") { - const set = subscriber.parentUris; - if (set instanceof Set) set.delete(msg.data.uri); - else if (Array.isArray(set)) { - const idx = set.indexOf(msg.data.uri); - if (idx >= 0) set.splice(idx, 1); - } - } else if (msg.kind === "hydration.added") { - ensureChildMap(subscriber).set(msg.data.child.rkey, { - parentUri: msg.data.parentUri, - relName: msg.data.relation - }); - } else if (msg.kind === "hydration.removed") { - const map = subscriber.childToParent; - if (map instanceof Map) map.delete(msg.data.childRkey); - else if (map && typeof map === "object") { - delete (map as Record<string, unknown>)[msg.data.childRkey]; - } - } - } -} - -function ensureParentSet(sub: SubscriberView): Set<string> { - if (sub.parentUris instanceof Set) return sub.parentUris; - const set = new Set<string>(sub.parentUris ?? []); - sub.parentUris = set; - return set; -} - -function ensureChildMap( - sub: SubscriberView -): Map<string, { parentUri: string; relName: string }> { - if (sub.childToParent instanceof Map) return sub.childToParent; - const map = new Map<string, { parentUri: string; relName: string }>(); - if (sub.childToParent && typeof sub.childToParent === "object") { - for (const [k, v] of Object.entries(sub.childToParent)) map.set(k, v); - } - sub.childToParent = map; - return map; -} - -function hasParent( - parents: Set<string> | string[] | undefined, - uri: string -): boolean { - if (!parents) return false; - if (parents instanceof Set) return parents.has(uri); - return parents.includes(uri); -} - -function getChildInfo( - map: - | Map<string, { parentUri: string; relName: string }> - | Record<string, { parentUri: string; relName: string }> - | undefined, - rkey: string -): { parentUri: string; relName: string } | undefined { - if (!map) return undefined; - if (map instanceof Map) return map.get(rkey); - return map[rkey]; -} - -function getNestedValue( - obj: Record<string, unknown>, - path: string -): unknown { - let cur: unknown = obj; - for (const key of path.split(".")) { - if (cur == null || typeof cur !== "object") return undefined; - cur = (cur as Record<string, unknown>)[key]; - } - return cur; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/sse.ts b/packages/contrail/src/core/realtime/sse.ts index 65d89c1..1129419 100644 --- a/packages/contrail/src/core/realtime/sse.ts +++ b/packages/contrail/src/core/realtime/sse.ts @@ -1,98 +1 @@ -/** Server-Sent Events transport. - * - * Wraps an AsyncIterable<RealtimeEvent> as a streaming Response. The caller - * (the router) has already done auth and has an AbortSignal it can use to - * tear the stream down (e.g. on `member.removed` for the subscriber's DID). */ - -import type { RealtimeEvent } from "./types"; -import { DEFAULT_KEEPALIVE_MS } from "./types"; - -export interface SseOptions { - keepaliveMs?: number; - /** Called before the stream closes. Useful for cleanup that the caller - * can't do via the signal (e.g. removing a subscriber from a set). */ - onClose?: () => void; -} - -export function sseResponse( - iter: AsyncIterable<RealtimeEvent>, - signal: AbortSignal, - opts: SseOptions = {} -): Response { - const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; - const encoder = new TextEncoder(); - - const stream = new ReadableStream<Uint8Array>({ - start(controller) { - let closed = false; - let keepalive: ReturnType<typeof setInterval> | null = null; - - const close = () => { - if (closed) return; - closed = true; - if (keepalive) clearInterval(keepalive); - try { - controller.close(); - } catch { - /* already closed */ - } - opts.onClose?.(); - }; - - signal.addEventListener("abort", close, { once: true }); - - keepalive = setInterval(() => { - if (closed) return; - try { - controller.enqueue(encoder.encode(`: keepalive\n\n`)); - } catch { - close(); - } - }, keepaliveMs); - - (async () => { - // Opening comment — helps some clients / proxies initialize promptly. - controller.enqueue(encoder.encode(`: open\n\n`)); - try { - for await (const event of iter) { - if (closed) break; - controller.enqueue(encoder.encode(frameEvent(event))); - } - } catch (err) { - if (!closed) { - try { - controller.enqueue( - encoder.encode( - `event: error\ndata: ${JSON.stringify({ - message: err instanceof Error ? err.message : String(err), - })}\n\n` - ) - ); - } catch { - /* stream already torn down */ - } - } - } finally { - close(); - } - })(); - }, - cancel() { - opts.onClose?.(); - }, - }); - - return new Response(stream, { - status: 200, - headers: { - "content-type": "text/event-stream", - "cache-control": "no-cache, no-transform", - connection: "keep-alive", - "x-accel-buffering": "no", - }, - }); -} - -function frameEvent(event: RealtimeEvent): string { - return `event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/ticket.ts b/packages/contrail/src/core/realtime/ticket.ts index 1c6e3de..1129419 100644 --- a/packages/contrail/src/core/realtime/ticket.ts +++ b/packages/contrail/src/core/realtime/ticket.ts @@ -1,179 +1 @@ -/** Subscription tickets — HMAC-signed short-lived `{topics, did, exp}` blobs. - * - * Wire format: `<payload>.<sig>` where - * payload = base64url(JSON({ topics, did, exp, iat })) - * sig = base64url(HMAC-SHA256(key, payload)) - * - * Tickets are integrity-only (not encrypted). Browsers use them because - * EventSource / WebSocket can't send Authorization headers; server-side - * consumers skip the ticket dance and send their JWT directly. */ - -export interface TicketPayload { - /** Concrete delivery topics this ticket authorizes. `community:<did>` is - * expanded to the caller's visible spaces before signing — a ticket never - * carries a community alias. */ - topics: string[]; - did: string; - /** Unix ms. */ - exp: number; - /** Unix ms — useful for debugging; ignored on verify. */ - iat: number; - /** Optional: query-scoped watchRecords spec this ticket authorizes. Present - * when the ticket was minted from a watchRecords handshake. The server - * trusts the signed spec on upgrade and forwards it to the DO. */ - querySpec?: TicketQuerySpec; -} - -export interface TicketQuerySpec { - collection: string; - /** Exactly one of `spaceUri` or `actor` is set. `spaceUri` = per-space - * watch; `actor` = cross-space watch for records authored by this DID - * (the ticket's `topics` list carries the expanded delivery topics). */ - spaceUri?: string; - actor?: string; - hydrate?: Record<string, { childCollection: string; matchField: string }>; -} - -function normalizeSecret(secret: Uint8Array | string): Uint8Array { - if (typeof secret !== "string") { - if (secret.length !== 32) { - throw new Error(`realtime ticketSecret must be 32 bytes, got ${secret.length}`); - } - return secret; - } - // 64 hex chars would also round-trip as base64 (to 48 bytes). Prefer hex - // when the input matches the hex alphabet exactly; fall back to base64. - const hex = tryHex(secret); - if (hex && hex.length === 32) return hex; - const b64 = tryBase64(secret); - if (b64 && b64.length === 32) return b64; - if (hex || b64) { - const got = (hex ?? b64)!.length; - throw new Error(`realtime ticketSecret must decode to 32 bytes, got ${got}`); - } - throw new Error("realtime ticketSecret must be a 32-byte Uint8Array or base64/hex string"); -} - -function tryBase64(s: string): Uint8Array | null { - try { - const normal = s.replace(/-/g, "+").replace(/_/g, "/"); - const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); - if (!/^[A-Za-z0-9+/]*=*$/.test(padded)) return null; - const bin = atob(padded); - const out = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); - return out; - } catch { - return null; - } -} - -function tryHex(s: string): Uint8Array | null { - if (!/^[0-9a-fA-F]+$/.test(s) || s.length % 2 !== 0) return null; - const out = new Uint8Array(s.length / 2); - for (let i = 0; i < out.length; i++) { - out[i] = parseInt(s.slice(i * 2, i * 2 + 2), 16); - } - return out; -} - -function b64urlFromBytes(bytes: Uint8Array): string { - let bin = ""; - for (let i = 0; i < bytes.length; i++) bin += String.fromCharCode(bytes[i]!); - return btoa(bin).replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); -} - -function b64urlToBytes(s: string): Uint8Array { - const normal = s.replace(/-/g, "+").replace(/_/g, "/"); - const padded = normal + "=".repeat((4 - (normal.length % 4)) % 4); - const bin = atob(padded); - const out = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); - return out; -} - -function b64urlFromString(s: string): string { - return b64urlFromBytes(new TextEncoder().encode(s)); -} - -function stringFromB64url(s: string): string { - return new TextDecoder().decode(b64urlToBytes(s)); -} - -function constantTimeEq(a: Uint8Array, b: Uint8Array): boolean { - if (a.length !== b.length) return false; - let diff = 0; - for (let i = 0; i < a.length; i++) diff |= a[i]! ^ b[i]!; - return diff === 0; -} - -export class TicketSigner { - private readonly keyPromise: Promise<CryptoKey>; - - constructor(secret: Uint8Array | string) { - const raw = normalizeSecret(secret); - this.keyPromise = crypto.subtle.importKey( - "raw", - raw as BufferSource, - { name: "HMAC", hash: "SHA-256" }, - false, - ["sign", "verify"] - ); - } - - async sign(input: { - topics: string[]; - did: string; - ttlMs: number; - querySpec?: TicketQuerySpec; - }): Promise<string> { - const now = Date.now(); - const payload: TicketPayload = { - topics: input.topics, - did: input.did, - exp: now + input.ttlMs, - iat: now, - ...(input.querySpec ? { querySpec: input.querySpec } : {}), - }; - const payloadPart = b64urlFromString(JSON.stringify(payload)); - const sig = await crypto.subtle.sign( - "HMAC", - await this.keyPromise, - new TextEncoder().encode(payloadPart) as BufferSource - ); - const sigPart = b64urlFromBytes(new Uint8Array(sig)); - return `${payloadPart}.${sigPart}`; - } - - /** Returns the decoded payload if the ticket is valid + unexpired, else null. */ - async verify(ticket: string): Promise<TicketPayload | null> { - const dot = ticket.indexOf("."); - if (dot < 0) return null; - const payloadPart = ticket.slice(0, dot); - const sigPart = ticket.slice(dot + 1); - let expectedSig: Uint8Array; - try { - expectedSig = b64urlToBytes(sigPart); - } catch { - return null; - } - const computedRaw = await crypto.subtle.sign( - "HMAC", - await this.keyPromise, - new TextEncoder().encode(payloadPart) as BufferSource - ); - const computed = new Uint8Array(computedRaw); - if (!constantTimeEq(expectedSig, computed)) return null; - let parsed: TicketPayload; - try { - parsed = JSON.parse(stringFromB64url(payloadPart)); - } catch { - return null; - } - if (!parsed || !Array.isArray(parsed.topics) || typeof parsed.did !== "string") { - return null; - } - if (typeof parsed.exp !== "number" || parsed.exp <= Date.now()) return null; - return parsed; - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/types.ts b/packages/contrail/src/core/realtime/types.ts index 05a246e..1129419 100644 --- a/packages/contrail/src/core/realtime/types.ts +++ b/packages/contrail/src/core/realtime/types.ts @@ -1,127 +1 @@ -/** Realtime module — canonical types + interfaces. See docs/realtime.md. */ - -/** Discriminated union of every event kind that flows through the PubSub. - * - * `record.created` carries the full record body so a subscriber can apply an - * insert or upsert without a follow-up `getRecord` call. Writing a new record - * to the same `(did, collection, rkey)` publishes another `record.created` — - * treat it as upsert. - * - * **Payload shape mirrors `listRecords` output** (`uri`, `did`, `space?`, - * `time_us`), so a subscriber can render a live row the same way it renders - * a fetched row. - * - * **Publisher/topic matrix (intentional trust split):** - * - `collection:<nsid>` and `actor:<did>` carry *public* record events only - * (from jetstream ingestion) — no `space`. - * - `space:<uri>` and `community:<did>` carry *space* events — `space` is - * always set. Never cross-published to public topics (privacy). */ -export type RealtimeEvent = - | { - topic: string; - kind: "record.created"; - payload: { - uri: string; - did: string; - collection: string; - rkey: string; - cid: string | null; - record: Record<string, unknown>; - time_us: number; - /** Present only for space records; absent for public records. */ - space?: string; - }; - ts: number; - } - | { - topic: string; - kind: "record.deleted"; - payload: { - uri: string; - did: string; - collection: string; - rkey: string; - /** Present only for space records; absent for public records. */ - space?: string; - }; - ts: number; - } - | { - topic: string; - kind: "member.added"; - payload: { space: string; did: string }; - ts: number; - } - | { - topic: string; - kind: "member.removed"; - payload: { space: string; did: string }; - ts: number; - }; - -export type RealtimeEventKind = RealtimeEvent["kind"]; - -/** Core pubsub abstraction. Implementations: InMemoryPubSub, DurableObjectPubSub. */ -export interface PubSub { - publish(event: RealtimeEvent): Promise<void>; - /** Stream events on the topic until the signal aborts (or the iterator is - * returned/broken out of). Implementations use a bounded per-subscriber - * queue with drop-oldest semantics — a slow subscriber can't stall publishers. */ - subscribe(topic: string, signal?: AbortSignal): AsyncIterable<RealtimeEvent>; -} - -// ---- Canonical topic strings ----------------------------------------------- -// `community:<did>` is an alias resolved at ticket-mint time to the concrete -// set of `space:<uri>` topics the caller can see; it is never a real delivery -// topic. The other three are real. - -export function spaceTopic(uri: string): string { - return `space:${uri}`; -} - -export function communityTopic(did: string): string { - return `community:${did}`; -} - -export function collectionTopic(nsid: string): string { - return `collection:${nsid}`; -} - -export function actorTopic(did: string): string { - return `actor:${did}`; -} - -export function isCommunityTopic(topic: string): boolean { - return topic.startsWith("community:"); -} - -export function parseCommunityTopic(topic: string): string | null { - return isCommunityTopic(topic) ? topic.slice("community:".length) : null; -} - -export function parseSpaceTopic(topic: string): string | null { - return topic.startsWith("space:") ? topic.slice("space:".length) : null; -} - -// ---- Config ----------------------------------------------------------------- - -export interface RealtimeConfig { - /** Backing pubsub. Default: new InMemoryPubSub() (single-process only). On - * Workers, pass `new DurableObjectPubSub(env.REALTIME)`. */ - pubsub?: PubSub; - /** HMAC secret used to sign subscription tickets. 32 bytes. Accepts raw - * Uint8Array or base64 / hex string. Envelope-encrypts nothing — tickets - * are integrity-only, not confidential. */ - ticketSecret: Uint8Array | string; - /** Ticket lifetime in ms. Default 120_000 (2 minutes). */ - ticketTtlMs?: number; - /** SSE/WS keepalive interval in ms. Default 15_000. */ - keepaliveMs?: number; - /** Per-subscriber queue bound. Default 1024. Events beyond this are dropped - * oldest-first and the subscriber receives a `lag` signal (out of band). */ - queueBound?: number; -} - -export const DEFAULT_TICKET_TTL_MS = 120_000; -export const DEFAULT_KEEPALIVE_MS = 15_000; -export const DEFAULT_QUEUE_BOUND = 1024; +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/realtime/websocket.ts b/packages/contrail/src/core/realtime/websocket.ts index 833483d..1129419 100644 --- a/packages/contrail/src/core/realtime/websocket.ts +++ b/packages/contrail/src/core/realtime/websocket.ts @@ -1,84 +1 @@ -/** WebSocket transport. - * - * Accepts a new WebSocket connection (either via `WebSocketPair` on Workers - * or a platform-provided server-side socket) and pumps events to it from an - * AsyncIterable<RealtimeEvent>. Messages are UTF-8 JSON, one event per frame. - * - * Close codes (subset, RFC 6455 + app-custom): - * - 4001: server error pumping - * - 4003: membership revoked - * - 4008: ticket/auth invalid (used by the router, not here) - */ - -import type { RealtimeEvent } from "./types"; -import { DEFAULT_KEEPALIVE_MS } from "./types"; - -export interface WebSocketLike { - send(data: string): void; - close(code?: number, reason?: string): void; - addEventListener(type: "message" | "close" | "error", listener: (ev: any) => void): void; -} - -export interface WebSocketPumpOptions { - keepaliveMs?: number; - onClose?: () => void; -} - -/** Pump events from `iter` to `ws` until the signal aborts or the iter ends. - * Caller is responsible for having already accept()ed the socket. */ -export async function pumpWebSocket( - ws: WebSocketLike, - iter: AsyncIterable<RealtimeEvent>, - signal: AbortSignal, - opts: WebSocketPumpOptions = {} -): Promise<void> { - const keepaliveMs = opts.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; - let closed = false; - - const close = (code?: number, reason?: string) => { - if (closed) return; - closed = true; - try { - ws.close(code, reason); - } catch { - /* already closed */ - } - opts.onClose?.(); - }; - - ws.addEventListener("close", () => { - closed = true; - opts.onClose?.(); - }); - ws.addEventListener("error", () => { - closed = true; - opts.onClose?.(); - }); - signal.addEventListener("abort", () => close(1000, "aborted"), { once: true }); - - const keepalive = setInterval(() => { - if (closed) return; - try { - ws.send(JSON.stringify({ kind: "$keepalive" })); - } catch { - close(); - } - }, keepaliveMs); - - try { - for await (const event of iter) { - if (closed) break; - try { - ws.send(JSON.stringify(event)); - } catch { - close(4001, "send-failed"); - break; - } - } - } catch { - close(4001, "pump-error"); - } finally { - clearInterval(keepalive); - close(); - } -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/refresh.ts b/packages/contrail/src/core/refresh.ts index aacedaa..7b2fd21 100644 --- a/packages/contrail/src/core/refresh.ts +++ b/packages/contrail/src/core/refresh.ts @@ -1,3 +1,4 @@ +import type {} from "@atcute/atproto"; /** * Fresh refresh: re-walk every known DID's PDS for every configured collection * and reconcile against what's in our DB. Unlike `backfillPending`, this diff --git a/packages/contrail/src/core/spaces/acl.ts b/packages/contrail/src/core/spaces/acl.ts index 6e0da0a..1129419 100644 --- a/packages/contrail/src/core/spaces/acl.ts +++ b/packages/contrail/src/core/spaces/acl.ts @@ -1,69 +1 @@ -import type { AppPolicy, SpaceMemberRow, SpaceRow } from "./types"; - -export type AclOp = "read" | "write" | "delete"; - -export interface AclInput { - op: AclOp; - space: SpaceRow; - callerDid: string; - /** Membership row for the caller (or null). Owner does not require a row. */ - member: SpaceMemberRow | null; - /** OAuth client_id of the app calling on caller's behalf, for app policy checks. */ - clientId?: string; - /** For per-record ops (get/delete), the record's author DID. */ - targetAuthorDid?: string; -} - -export type AclResult = - | { allow: true } - | { allow: false; reason: AclDenyReason }; - -export type AclDenyReason = - | "not-member" - | "not-own-record" - | "app-not-allowed" - | "unknown-op"; - -/** Check whether the caller's app is permitted to act in this space. */ -export function checkAppPolicy( - appPolicy: AppPolicy | null, - clientId: string | undefined -): boolean { - if (!appPolicy) return true; // no policy = allow-all - const listed = clientId ? appPolicy.apps.includes(clientId) : false; - if (appPolicy.mode === "allow") return !listed; // apps[] is a denylist - return listed; // mode === "deny": apps[] is an allowlist -} - -const isOwner = (space: SpaceRow, did: string) => space.ownerDid === did; -const hasMember = (space: SpaceRow, member: SpaceMemberRow | null, did: string) => - isOwner(space, did) || member != null; - -/** Space-level access check. - * Membership = access. Any member (including owner) can read and write. - * Delete is scoped to the caller's own records — owners don't get a bypass. - * A random member can't nuke other people's records, and neither can the - * owner. To remove a non-author record, delete the whole space. */ -export function checkAccess(input: AclInput): AclResult { - if (!checkAppPolicy(input.space.appPolicy, input.clientId)) { - return { allow: false, reason: "app-not-allowed" }; - } - - if (input.op === "read" || input.op === "write") { - return hasMember(input.space, input.member, input.callerDid) - ? { allow: true } - : { allow: false, reason: "not-member" }; - } - - if (input.op === "delete") { - if (!hasMember(input.space, input.member, input.callerDid)) { - return { allow: false, reason: "not-member" }; - } - if (input.targetAuthorDid && input.targetAuthorDid !== input.callerDid) { - return { allow: false, reason: "not-own-record" }; - } - return { allow: true }; - } - - return { allow: false, reason: "unknown-op" }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 4c1e817..1129419 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -1,177 +1 @@ -import type { Context, MiddlewareHandler } from "hono"; -import { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; -import { - CompositeDidDocumentResolver, - PlcDidDocumentResolver, - WebDidDocumentResolver, - type DidDocumentResolver, -} from "@atcute/identity-resolver"; -import type { Did, Nsid } from "@atcute/lexicons"; -import type { AuthorityConfig } from "./types"; -import { readInProcess } from "./in-process"; - -export { ServiceJwtVerifier }; - -/** Build a ServiceJwtVerifier from an AuthorityConfig, using the configured - * resolver or a default PLC+Web composite. The verifier checks that incoming - * JWTs target this authority's serviceDid (aud claim). */ -export function buildVerifier(authority: AuthorityConfig): ServiceJwtVerifier { - const resolver = - authority.resolver ?? - new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, - }); - return new ServiceJwtVerifier({ - serviceDid: authority.serviceDid as Did, - resolver, - }); -} - -export interface ServiceAuth { - issuer: string; - audience: string; - lxm: string | undefined; - /** OAuth client_id of the caller, if the JWT carries one. */ - clientId?: string; -} - -export interface ServiceAuthOptions { - serviceDid: Did; - resolver: DidDocumentResolver; -} - -/** Hono middleware that authenticates XRPC requests. Order of precedence: - * 1. In-process marker (same-module calls; see `core/spaces/in-process.ts`) - * 2. Authorization: Bearer <JWT> as an atproto service-auth token - * - * On success, attaches the claims to `c.var.serviceAuth`. Expected Nsid is - * taken from the route pattern (last segment after `/xrpc/`). */ -export function createServiceAuthMiddleware( - verifier: ServiceJwtVerifier -): MiddlewareHandler { - return async (c, next) => { - const lxm = extractLxmFromPath(c); - - const inProcess = readInProcess(c.req.raw); - if (inProcess) { - c.set("serviceAuth", { - issuer: inProcess.did, - audience: "", - lxm: lxm ?? undefined, - } satisfies ServiceAuth); - await next(); - return; - } - - const header = c.req.header("Authorization"); - if (!header || !header.startsWith("Bearer ")) { - return c.json({ error: "AuthRequired", message: "Missing bearer token" }, 401); - } - const token = header.slice(7).trim(); - - const result = await verifier.verify(token, { lxm }); - if (!result.ok) { - const err = result.error as { error?: string; description?: string } | undefined; - return c.json( - { - error: "AuthRequired", - message: err?.description ?? err?.error ?? String(result.error), - }, - 401, - ); - } - - c.set("serviceAuth", { - issuer: result.value.issuer, - audience: result.value.audience, - lxm: result.value.lxm, - } satisfies ServiceAuth); - - await next(); - }; -} - -function extractLxmFromPath(c: Context): Nsid | null { - const path = new URL(c.req.url).pathname; - const match = path.match(/\/xrpc\/([a-zA-Z0-9.-]+)/); - return (match?.[1] as Nsid) ?? null; -} - -/** Read the service auth claims set by the middleware. Throws if unset. */ -export function requireServiceAuth(c: Context): ServiceAuth { - const auth = c.get("serviceAuth") as ServiceAuth | undefined; - if (!auth) throw new Error("service auth missing; middleware not attached"); - return auth; -} - -/** Out-of-band auth check for handlers that don't always require auth. - * Returns claims on success, or null if no valid credentials are present. - * Order of precedence: in-process marker → service-auth JWT. */ -export async function verifyServiceAuthRequest( - verifier: ServiceJwtVerifier, - request: Request, - lxm?: Nsid | null -): Promise<ServiceAuth | null> { - const inProcess = readInProcess(request); - if (inProcess) { - return { - issuer: inProcess.did, - audience: "", - lxm: lxm ?? undefined, - }; - } - - const header = request.headers.get("Authorization"); - if (!header || !header.startsWith("Bearer ")) return null; - const token = header.slice(7).trim(); - const result = await verifier.verify(token, { lxm: lxm ?? null }); - if (!result.ok) return null; - return { - issuer: result.value.issuer, - audience: result.value.audience, - lxm: result.value.lxm, - }; -} - -/** Pull a space credential off the request — `X-Space-Credential: <jwt>` - * header. Returns the raw token or null. */ -export function extractSpaceCredential(request: Request): string | null { - const header = request.headers.get("X-Space-Credential"); - return header ? header.trim() : null; -} - -/** Pull a read-grant invite token off the request — query string `?inviteToken=` - * or `Authorization: Bearer atmo-invite:<token>`. Returns the raw token (not - * hashed) or null. Routes hash + look up via the adapter. */ -export function extractInviteToken(request: Request): string | null { - const url = new URL(request.url); - const q = url.searchParams.get("inviteToken"); - if (q) return q.trim(); - const header = request.headers.get("Authorization"); - if (header?.startsWith("Bearer atmo-invite:")) { - return header.slice("Bearer atmo-invite:".length).trim(); - } - return null; -} - -/** Validate a read-grant invite token against a target spaceUri. Returns true - * if the token exists, scopes to this space, has a kind that grants read - * (`read` or `read-join`), and is not expired/revoked. */ -export async function checkInviteReadGrant( - adapter: { getInvite(tokenHash: string): Promise<{ spaceUri: string; kind: string; revokedAt: number | null; expiresAt: number | null } | null> }, - rawToken: string, - spaceUri: string, - hashFn: (token: string) => Promise<string> -): Promise<boolean> { - const tokenHash = await hashFn(rawToken); - const invite = await adapter.getInvite(tokenHash); - if (!invite) return false; - if (invite.spaceUri !== spaceUri) return false; - if (invite.kind !== "read" && invite.kind !== "read-join") return false; - if (invite.revokedAt != null) return false; - if (invite.expiresAt != null && invite.expiresAt <= Date.now()) return false; - return true; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts index a81b95f..1129419 100644 --- a/packages/contrail/src/core/spaces/binding.ts +++ b/packages/contrail/src/core/spaces/binding.ts @@ -1,274 +1 @@ -/** Binding resolution: given a space URI, which DID is authorized to sign - * credentials for it, and where do we find that DID's verification key? - * - * Two layers of pluggable resolvers compose into a credential verifier: - * - * BindingResolver — `ats://<owner>/<type>/<key>` → authority DID - * KeyResolver — (DID, kid) → JsonWebKey - * - * The BindingResolver is what makes user-owned-DID-with-PDS-record work: - * given a space URI, we resolve the owner's PDS, fetch the declaration - * record, and read its `authority` field. For provisioned (no-PDS) DIDs we - * fall back to the owner DID's `#atproto_space_authority` service entry. - * And finally for the trivial case (HappyView-style "owner self-issues"), - * we return the owner DID itself. - * - * See conversation history (phase 4 design) for the rationale on why these - * three sources, in this order. */ - -import type { DidDocumentResolver } from "@atcute/identity-resolver"; -import type { Did } from "@atcute/lexicons"; -import { parseSpaceUri } from "./uri"; -import type { RecordHost } from "./types"; - -export interface BindingResolver { - /** Resolve the DID authorized to sign credentials for this space. Returns - * null if no binding could be found via this resolver — the composite - * walks down its list looking for a non-null. */ - resolveAuthority(spaceUri: string): Promise<string | null>; -} - -export interface KeyResolver { - /** Resolve `did`'s verification key for credential signing. `kid` is the - * full header `kid` value (e.g. "did:web:x.com#atproto_space_authority"), - * used to disambiguate when a DID doc lists multiple methods. */ - resolveKey(did: string, kid: string | undefined): Promise<JsonWebKey | null>; -} - -// --------------------------------------------------------------------------- -// Binding resolvers -// --------------------------------------------------------------------------- - -/** Always returns the configured authority DID. Used in-process when the - * authority and record host run in one deployment — no need to walk DID - * docs or PDSes; we know what we are. */ -export function createLocalBindingResolver(args: { - authorityDid: string; -}): BindingResolver { - const { authorityDid } = args; - return { - async resolveAuthority() { - return authorityDid; - }, - }; -} - -/** Reads the record host's local enrollment table. This is the *canonical* - * binding source on a record host: the host owner explicitly consented to a - * given authority for a given space (via the `recordHost.enroll` endpoint - * or auto-enrollment from the authority's createSpace). PDS-record / - * DID-doc resolvers are out-of-band discovery aids; the enrollment is what - * actually gates whether records get stored here. */ -export function createEnrollmentBindingResolver(args: { - recordHost: RecordHost; -}): BindingResolver { - return { - async resolveAuthority(spaceUri) { - const e = await args.recordHost.getEnrollment(spaceUri); - return e?.authorityDid ?? null; - }, - }; -} - -/** Returns the space owner DID as the authority. This is the implicit - * fallback ("HappyView path") — when no PDS record and no DID-doc service - * entry declare an issuer, the owner is taken to be its own. Whether the - * resulting credential actually verifies depends on whether the owner's DID - * doc publishes a usable signing key. */ -export function createOwnerSelfBindingResolver(): BindingResolver { - return { - async resolveAuthority(spaceUri) { - const parts = parseSpaceUri(spaceUri); - return parts ? parts.ownerDid : null; - }, - }; -} - -/** Walks the resolver list in order, returns the first non-null. Use this - * to compose [pdsRecord, didDocService, ownerSelf] etc. */ -export function createCompositeBindingResolver( - resolvers: BindingResolver[] -): BindingResolver { - return { - async resolveAuthority(spaceUri) { - for (const r of resolvers) { - const did = await r.resolveAuthority(spaceUri); - if (did) return did; - } - return null; - }, - }; -} - -/** Reads a space-declaration record from the owner's PDS at - * `at://<owner>/<type>/<key>` and returns its `authority` field if present. - * - * This is the user-owned-DID path: the user writes a record to their PDS - * authorizing some service as the space's authority, no DID-doc edits - * required. */ -export function createPdsBindingResolver(args: { - /** DID resolver, used to look up the owner's PDS endpoint. */ - resolver: DidDocumentResolver; - /** Fetch impl. Defaults to `globalThis.fetch`. */ - fetch?: typeof fetch; - /** Per-request timeout in ms. Defaults to 5000. */ - timeoutMs?: number; -}): BindingResolver { - const fetchImpl = args.fetch ?? globalThis.fetch; - const timeoutMs = args.timeoutMs ?? 5000; - - return { - async resolveAuthority(spaceUri) { - const parts = parseSpaceUri(spaceUri); - if (!parts) return null; - const pds = await pdsEndpointFor(args.resolver, parts.ownerDid); - if (!pds) return null; - - const url = new URL(`${pds}/xrpc/com.atproto.repo.getRecord`); - url.searchParams.set("repo", parts.ownerDid); - url.searchParams.set("collection", parts.type); - url.searchParams.set("rkey", parts.key); - - const ctrl = new AbortController(); - const timer = setTimeout(() => ctrl.abort(), timeoutMs); - let res: Response; - try { - res = await fetchImpl(url.toString(), { signal: ctrl.signal }); - } catch { - return null; - } finally { - clearTimeout(timer); - } - if (!res.ok) return null; - const body = (await res.json().catch(() => null)) as - | { value?: { authority?: unknown } } - | null; - const authority = body?.value?.authority; - return typeof authority === "string" && authority.startsWith("did:") ? authority : null; - }, - }; -} - -/** Reads `service[id="#atproto_space_authority"].serviceEndpoint` from the - * owner's DID doc. This is the no-PDS path — useful for provisioned space - * DIDs that exist as DID docs only. - * - * Note the service endpoint here is a *DID*, not a URL. The DID names the - * authority; the key resolver's job is to then fetch its verification key. - * For DID docs that declare a URL endpoint, we treat the URL as a - * did:web hint — caller can normalize. */ -export function createDidDocBindingResolver(args: { - resolver: DidDocumentResolver; - /** Service id to look up. Defaults to "#atproto_space_authority". */ - serviceId?: string; -}): BindingResolver { - const serviceId = args.serviceId ?? "#atproto_space_authority"; - return { - async resolveAuthority(spaceUri) { - const parts = parseSpaceUri(spaceUri); - if (!parts) return null; - let doc; - try { - doc = await args.resolver.resolve(parts.ownerDid as Did); - } catch { - return null; - } - const entry = doc.service?.find((s: { id?: string }) => s.id === serviceId); - if (!entry) return null; - const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; - if (typeof endpoint !== "string") return null; - // Endpoint may be a DID (preferred) or a URL hint. Only DIDs are - // verifiable downstream; URLs require the caller to map URL → DID. - return endpoint.startsWith("did:") ? endpoint : null; - }, - }; -} - -// --------------------------------------------------------------------------- -// Key resolvers -// --------------------------------------------------------------------------- - -/** Knows the local authority's public key directly. Returns null for any - * other DID — composite with a DID-doc resolver if you also accept - * external authorities. */ -export function createLocalKeyResolver(args: { - authorityDid: string; - publicKey: JsonWebKey; -}): KeyResolver { - return { - async resolveKey(did) { - return did === args.authorityDid ? args.publicKey : null; - }, - }; -} - -/** Resolves a DID, finds the verification method matching `kid`, returns - * its `publicKeyJwk`. */ -export function createDidDocKeyResolver(args: { - resolver: DidDocumentResolver; -}): KeyResolver { - return { - async resolveKey(did, kid) { - let doc; - try { - doc = await args.resolver.resolve(did as Did); - } catch { - return null; - } - const methods = (doc as { verificationMethod?: VerificationMethod[] }).verificationMethod; - if (!methods) return null; - // kid is "<did>#<methodId>" — we match against the method.id which DID - // docs spell as "<did>#<methodId>" too. - const method = kid - ? methods.find((m) => m.id === kid) - : methods[0]; - if (!method?.publicKeyJwk) return null; - return method.publicKeyJwk as JsonWebKey; - }, - }; -} - -/** Walks resolvers in order; returns the first non-null. */ -export function createCompositeKeyResolver( - resolvers: KeyResolver[] -): KeyResolver { - return { - async resolveKey(did, kid) { - for (const r of resolvers) { - const k = await r.resolveKey(did, kid); - if (k) return k; - } - return null; - }, - }; -} - -interface VerificationMethod { - id: string; - type?: string; - controller?: string; - publicKeyJwk?: unknown; - publicKeyMultibase?: string; -} - -// --------------------------------------------------------------------------- -// Internal: PDS endpoint lookup -// --------------------------------------------------------------------------- - -async function pdsEndpointFor( - resolver: DidDocumentResolver, - did: string -): Promise<string | null> { - let doc; - try { - doc = await resolver.resolve(did as Did); - } catch { - return null; - } - const entry = doc.service?.find( - (s: { id?: string }) => s.id === "#atproto_pds" - ); - if (!entry) return null; - const endpoint = (entry as { serviceEndpoint?: unknown }).serviceEndpoint; - return typeof endpoint === "string" ? endpoint : null; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/blob-adapter.ts b/packages/contrail/src/core/spaces/blob-adapter.ts index 632b566..1129419 100644 --- a/packages/contrail/src/core/spaces/blob-adapter.ts +++ b/packages/contrail/src/core/spaces/blob-adapter.ts @@ -1,94 +1 @@ -/** - * Bytes-only storage adapter for space blobs. Metadata (CID, mime, size, - * author, space) lives in the `spaces_blobs` table on the main StorageAdapter; - * this interface only moves bytes in and out of a backend (R2, S3, fs, …). - * - * Keys are opaque strings formed by the router as `blobKey(spaceUri, cid)`. - */ - -export interface BlobUploadMeta { - mimeType: string; - size: number; -} - -export interface BlobAdapter { - put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void>; - get(key: string): Promise<Uint8Array | null>; - /** Bulk delete. Adapters that don't support batch can implement serially. */ - delete(keys: string[]): Promise<void>; -} - -/** In-memory adapter. Useful for tests and local development. */ -export class MemoryBlobAdapter implements BlobAdapter { - private readonly store = new Map<string, Uint8Array>(); - - async put(key: string, bytes: Uint8Array): Promise<void> { - this.store.set(key, bytes.slice()); - } - - async get(key: string): Promise<Uint8Array | null> { - const v = this.store.get(key); - return v ? v.slice() : null; - } - - async delete(keys: string[]): Promise<void> { - for (const k of keys) this.store.delete(k); - } - - /** Test helper. */ - size(): number { - return this.store.size; - } -} - -/** Minimal Cloudflare R2 bucket shape — matches @cloudflare/workers-types' R2Bucket - * without forcing a types dependency here. */ -export interface R2BucketLike { - put( - key: string, - value: ArrayBuffer | ArrayBufferView | ReadableStream | Blob, - options?: { httpMetadata?: { contentType?: string }; customMetadata?: Record<string, string> } - ): Promise<unknown>; - get(key: string): Promise<{ arrayBuffer(): Promise<ArrayBuffer> } | null>; - delete(keys: string | string[]): Promise<void>; -} - -/** Cloudflare R2 adapter. Pass the `env.BLOBS` binding from your Worker. */ -export class R2BlobAdapter implements BlobAdapter { - constructor(private readonly bucket: R2BucketLike) {} - - async put(key: string, bytes: Uint8Array, meta: BlobUploadMeta): Promise<void> { - await this.bucket.put(key, bytes, { - httpMetadata: { contentType: meta.mimeType }, - }); - } - - async get(key: string): Promise<Uint8Array | null> { - const obj = await this.bucket.get(key); - if (!obj) return null; - const buf = await obj.arrayBuffer(); - return new Uint8Array(buf); - } - - async delete(keys: string[]): Promise<void> { - if (keys.length === 0) return; - await this.bucket.delete(keys); - } -} - -/** Hash a space URI to a short, filesystem/R2-safe key segment. - * Used as the first segment of a blob key so all blobs for one space - * share a common prefix (enables bulk delete on space deletion). */ -export async function spaceKeyPrefix(spaceUri: string): Promise<string> { - const bytes = new TextEncoder().encode(spaceUri); - const digest = await crypto.subtle.digest("SHA-256", bytes); - const hex = Array.from(new Uint8Array(digest), (b) => b.toString(16).padStart(2, "0")).join(""); - return hex.slice(0, 16); -} - -/** Compose an adapter key from a space URI and CID. - * Shape: `<16-hex-chars-of-sha256(spaceUri)>/<cid>`. */ -export async function blobKey(spaceUri: string, cid: string): Promise<string> { - const prefix = await spaceKeyPrefix(spaceUri); - return `${prefix}/${cid}`; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts index d5c3029..1129419 100644 --- a/packages/contrail/src/core/spaces/credentials.ts +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -1,284 +1 @@ -/** Space-credential primitives: ES256 (P-256) JWTs minted by the authority, - * verified by the record host (or any third party that can resolve the - * authority's DID document). - * - * Format is a compact JWS: - * header = { alg: "ES256", typ: "JWT", kid: "<authorityDid>#<keyId>" } - * payload = { iss, sub, space, scope, iat, exp } - * - * - `iss` is the authority DID (the signer; for phase 3 this is the local - * authority's serviceDid; phase 4 adds a binding-resolution layer that - * lets the issuer be a *different* DID from the space owner). - * - `sub` is the caller DID — the credential bearer. - * - `space` is the full `ats://<owner>/<type>/<key>` URI. - * - `scope` is "rw" or "read". - * - * We don't use a JWT library — Web Crypto's subtle covers everything (P-256 - * generate, sign, verify, JWK import/export) and saves a runtime dep. */ - -const ALG = "ES256"; -const TYP = "JWT"; -const DEFAULT_KEY_ID = "atproto_space_authority"; - -export type CredentialScope = "rw" | "read"; - -export interface CredentialClaims { - iss: string; - sub: string; - space: string; - scope: CredentialScope; - iat: number; // seconds since epoch - exp: number; // seconds since epoch -} - -export interface CredentialKeyMaterial { - /** Private key in JWK form. P-256 / ES256. */ - privateKey: JsonWebKey; - /** Public key in JWK form. Must match privateKey. */ - publicKey: JsonWebKey; - /** DID-doc verification method id. The full JWT `kid` becomes - * `<authorityDid>#<keyId>`. Defaults to "atproto_space_authority". */ - keyId?: string; -} - -/** Generate a fresh P-256 keypair as JWKs. Useful for local dev / tests; in - * production the operator generates once and stores out-of-band. */ -export async function generateAuthoritySigningKey(): Promise<CredentialKeyMaterial> { - const pair = (await crypto.subtle.generateKey( - { name: "ECDSA", namedCurve: "P-256" }, - true, - ["sign", "verify"] - )) as CryptoKeyPair; - const privateKey = (await crypto.subtle.exportKey("jwk", pair.privateKey)) as JsonWebKey; - const publicKey = (await crypto.subtle.exportKey("jwk", pair.publicKey)) as JsonWebKey; - return { privateKey, publicKey }; -} - -const enc = new TextEncoder(); -const dec = new TextDecoder(); - -function base64urlEncode(bytes: Uint8Array): string { - let s = btoa(String.fromCharCode(...bytes)); - return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); -} - -function base64urlDecode(s: string): Uint8Array { - const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); - const bin = atob(padded); - const out = new Uint8Array(bin.length); - for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); - return out; -} - -function jsonEncode(value: unknown): string { - return base64urlEncode(enc.encode(JSON.stringify(value))); -} - -function jsonDecode<T>(seg: string): T { - return JSON.parse(dec.decode(base64urlDecode(seg))) as T; -} - -async function importPrivate(jwk: JsonWebKey): Promise<CryptoKey> { - return crypto.subtle.importKey( - "jwk", - jwk, - { name: "ECDSA", namedCurve: "P-256" }, - false, - ["sign"] - ); -} - -async function importPublic(jwk: JsonWebKey): Promise<CryptoKey> { - return crypto.subtle.importKey( - "jwk", - jwk, - { name: "ECDSA", namedCurve: "P-256" }, - false, - ["verify"] - ); -} - -/** Sign a credential payload with the authority's private key. - * `iat` and `exp` are filled in by the caller (so tests can mint expired - * tokens deterministically). */ -export async function signCredential( - payload: CredentialClaims, - key: CredentialKeyMaterial -): Promise<string> { - const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; - const header = { alg: ALG, typ: TYP, kid }; - const head = jsonEncode(header); - const body = jsonEncode(payload); - const signingInput = `${head}.${body}`; - const privateKey = await importPrivate(key.privateKey); - const sig = await crypto.subtle.sign( - { name: "ECDSA", hash: "SHA-256" }, - privateKey, - enc.encode(signingInput) - ); - return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; -} - -/** Issue a credential using the current wall-clock for iat/exp. */ -export async function issueCredential( - args: Omit<CredentialClaims, "iat" | "exp"> & { ttlMs: number }, - key: CredentialKeyMaterial -): Promise<{ credential: string; expiresAt: number }> { - const now = Math.floor(Date.now() / 1000); - const expSec = now + Math.floor(args.ttlMs / 1000); - const claims: CredentialClaims = { - iss: args.iss, - sub: args.sub, - space: args.space, - scope: args.scope, - iat: now, - exp: expSec, - }; - const credential = await signCredential(claims, key); - return { credential, expiresAt: expSec * 1000 }; -} - -export type VerifyOk = { ok: true; claims: CredentialClaims }; -export type VerifyErr = { - ok: false; - reason: - | "malformed" - | "bad-alg" - | "bad-signature" - | "expired" - | "not-yet-valid" - | "wrong-space" - | "wrong-scope" - | "unknown-issuer"; -}; - -export interface VerifyOptions { - /** Optional: when set, rejects credentials whose `space` claim differs. - * Omit when verifying in middleware where the target space isn't known - * yet — handlers can do the match themselves against the verified - * claims. */ - expectedSpace?: string; - /** Optional: required scope (e.g. "rw" rejects read-only credentials on writes). */ - requiredScope?: CredentialScope; - /** Resolve a verification key for `iss`. If null, verification fails with - * unknown-issuer. */ - resolveKey: (iss: string, kid: string | undefined) => Promise<JsonWebKey | null>; - /** Time provider for tests. Returns ms since epoch. */ - now?: () => number; -} - -export async function verifyCredential( - jwt: string, - opts: VerifyOptions -): Promise<VerifyOk | VerifyErr> { - const parts = jwt.split("."); - if (parts.length !== 3) return { ok: false, reason: "malformed" }; - const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; - - let header: { alg?: string; typ?: string; kid?: string }; - let claims: CredentialClaims; - try { - header = jsonDecode(headSeg); - claims = jsonDecode(bodySeg); - } catch { - return { ok: false, reason: "malformed" }; - } - if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; - if (opts.expectedSpace !== undefined && claims.space !== opts.expectedSpace) { - return { ok: false, reason: "wrong-space" }; - } - if (opts.requiredScope === "rw" && claims.scope !== "rw") { - return { ok: false, reason: "wrong-scope" }; - } - - const nowMs = (opts.now ?? Date.now)(); - const nowSec = Math.floor(nowMs / 1000); - if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; - if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; - - const jwk = await opts.resolveKey(claims.iss, header.kid); - if (!jwk) return { ok: false, reason: "unknown-issuer" }; - - const publicKey = await importPublic(jwk); - const sigBytes = base64urlDecode(sigSeg); - const signingInput = `${headSeg}.${bodySeg}`; - const valid = await crypto.subtle.verify( - { name: "ECDSA", hash: "SHA-256" }, - publicKey, - sigBytes, - enc.encode(signingInput) - ); - if (!valid) return { ok: false, reason: "bad-signature" }; - return { ok: true, claims }; -} - -/** Header reader for handlers that want to peek at `iss` before resolving the - * key (e.g. to short-circuit DID-doc fetches for the local authority). */ -export function decodeUnverifiedClaims(jwt: string): CredentialClaims | null { - const parts = jwt.split("."); - if (parts.length !== 3) return null; - try { - return jsonDecode<CredentialClaims>(parts[1]!); - } catch { - return null; - } -} - -/** Verifier interface consumed by the record host. The record host doesn't - * care HOW credentials get verified — it only cares whether a given JWT is - * valid. Phase 3 ships an in-process verifier that knows the local - * authority's public key; phase 4 adds a binding-resolving verifier that - * consults PDS records / DID docs. */ -export interface CredentialVerifier { - /** Verify a credential's signature, expiry, and `not-before` window. Does - * NOT enforce a space match — handlers do that against the request URI. */ - verify(jwt: string): Promise<VerifyOk | VerifyErr>; -} - -/** In-process verifier for the simple deployment: the authority and record - * host run in one process and the record host has direct access to the - * authority's public key. Rejects any credential whose `iss` isn't the - * configured authority. Phase 4 has a more general - * {@link createBindingCredentialVerifier} that does proper binding lookup. */ -export function createInProcessVerifier(args: { - authorityDid: string; - publicKey: JsonWebKey; -}): CredentialVerifier { - return { - verify(jwt) { - return verifyCredential(jwt, { - resolveKey: async (iss) => (iss === args.authorityDid ? args.publicKey : null), - }); - }, - }; -} - -/** Verifier composed of a {@link BindingResolver} (which DID is authorized - * to issue for this space?) and a {@link KeyResolver} (what's that DID's - * public key?). This is the production-shape verifier — phase 4's main - * contribution. - * - * Verification flow: - * 1. Decode the JWT's claims (no signature check yet). - * 2. Ask the binding resolver: who's authorized for `claims.space`? - * 3. Confirm `claims.iss === authorizedDid`. - * 4. Ask the key resolver for that DID's verification key. - * 5. Verify signature + expiry + scope match. - */ -export function createBindingCredentialVerifier(args: { - bindings: import("./binding").BindingResolver; - keys: import("./binding").KeyResolver; -}): CredentialVerifier { - return { - async verify(jwt) { - const peek = decodeUnverifiedClaims(jwt); - if (!peek) return { ok: false, reason: "malformed" }; - const authorizedDid = await args.bindings.resolveAuthority(peek.space); - if (!authorizedDid) return { ok: false, reason: "unknown-issuer" }; - if (peek.iss !== authorizedDid) return { ok: false, reason: "unknown-issuer" }; - return verifyCredential(jwt, { - resolveKey: (iss, kid) => args.keys.resolveKey(iss, kid), - }); - }, - }; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/in-process.ts b/packages/contrail/src/core/spaces/in-process.ts index ec1d7a1..1129419 100644 --- a/packages/contrail/src/core/spaces/in-process.ts +++ b/packages/contrail/src/core/spaces/in-process.ts @@ -1,34 +1 @@ -/** In-process auth marker. - * - * For same-module callers (e.g. a SvelteKit worker that imports contrail and - * dispatches requests directly to the handler), service-auth JWTs are pure - * overhead: no network boundary is crossed, so there's nothing for the JWT to - * protect against. Instead, the caller tags the `Request` with a principal via - * a module-private WeakMap, and the auth middleware reads it back. - * - * Security note: this is unforgeable from outside the module because - * - WeakMap keys are `Request` object identities, not serialized data; - * - no HTTP request crossing a network boundary can reach into this map; - * - exploiting it requires code execution inside the same isolate, at - * which point auth is already game over. - * - * This is the strongest auth adapter contrail offers — it has no secret to - * leak. */ - -export interface InProcessPrincipal { - did: string; -} - -const PRINCIPALS = new WeakMap<Request, InProcessPrincipal>(); - -/** Tag a Request with an in-process principal. The returned Request is the - * same reference; the return value is for ergonomics. */ -export function markInProcess(req: Request, did: string): Request { - PRINCIPALS.set(req, { did }); - return req; -} - -/** Read the in-process principal for a Request, or null if unmarked. */ -export function readInProcess(req: Request): InProcessPrincipal | null { - return PRINCIPALS.get(req) ?? null; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/tid.ts b/packages/contrail/src/core/spaces/tid.ts index c419090..1129419 100644 --- a/packages/contrail/src/core/spaces/tid.ts +++ b/packages/contrail/src/core/spaces/tid.ts @@ -1,20 +1 @@ -const B32_CHARSET = "234567abcdefghijklmnopqrstuvwxyz"; - -let lastTimestamp = 0; -let clockId = Math.floor(Math.random() * 1024); - -/** Generate an atproto TID: 13-char base32-sortable (timestamp-ordered). */ -export function nextTid(): string { - let now = Date.now() * 1000; - if (now <= lastTimestamp) now = lastTimestamp + 1; - lastTimestamp = now; - - const n = BigInt(now) * 1024n + BigInt(clockId); - let s = ""; - let v = n; - for (let i = 0; i < 13; i++) { - s = B32_CHARSET[Number(v & 31n)] + s; - v >>= 5n; - } - return s; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 6561773..1129419 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -1,273 +1 @@ -import type { Database } from "../types"; -import type { DidDocumentResolver } from "@atcute/identity-resolver"; -import type { BlobAdapter } from "./blob-adapter"; -import type { CredentialKeyMaterial } from "./credentials"; - -export type AppPolicyMode = "allow" | "deny"; - -export interface AppPolicy { - mode: AppPolicyMode; - apps: string[]; -} - -export interface SpacesBlobsConfig { - /** Bytes backend (R2, S3, in-memory, …). */ - adapter: BlobAdapter; - /** Max blob size in bytes. Defaults to 2 MiB. */ - maxSize?: number; - /** MIME allowlist. If set, only these content types are accepted. */ - accept?: string[]; - /** Orphan blobs (those with no referencing record) are kept this long before - * GC can delete them, to allow upload-then-putRecord flows. - * Defaults to 24 hours. */ - gcOrphanAfterMs?: number; -} - -export const DEFAULT_BLOB_MAX_SIZE = 2 * 1024 * 1024; -export const DEFAULT_BLOB_GC_ORPHAN_AFTER_MS = 24 * 60 * 60 * 1000; - -/** Default credential lifetime. The rough spec calls for 2–4h; we pick the - * lower bound so revocation (kicked-from-space) is observable within 2h. */ -export const DEFAULT_CREDENTIAL_TTL_MS = 2 * 60 * 60 * 1000; - -/** Configuration for the **space authority** role: holds the member list, - * signs credentials, and gates space-management operations. In a fully-split - * deployment, the authority can run in a different process (or even a - * different operator) than the record host. */ -export interface AuthorityConfig { - /** NSID that identifies the kind of space this authority hosts, - * e.g. "tools.atmo.event.space". */ - type: string; - /** Service DID that service-auth tokens must target (aud claim) AND that - * signs credentials it issues (`iss` claim on emitted JWTs). */ - serviceDid: string; - /** Default app policy applied to new spaces. */ - defaultAppPolicy?: AppPolicy; - /** DID document resolver for service-auth JWT verification. - * Defaults to a composite PLC + did:web resolver if omitted. */ - resolver?: DidDocumentResolver; - /** Signing key material for issuing space credentials. When omitted, - * `<ns>.space.getCredential` returns 501 NotImplemented and the record - * host's credential-verifying middleware can't be wired up. */ - signing?: CredentialKeyMaterial; - /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ - credentialTtlMs?: number; -} - -/** Configuration for the **record host** role: stores per-space records and - * blobs and serves reads. Verifies space credentials (later phases) on - * incoming traffic. */ -export interface RecordHostConfig { - /** Blob-upload backend. When omitted, blob XRPCs are not exposed. */ - blobs?: SpacesBlobsConfig; -} - -/** Spaces config — host an authority, a record host, or both. - * Today both run in one process and most deployments will set both; the - * shape is split now so phase 5 can run them independently without churning - * every consumer's config. */ -export interface SpacesConfig { - /** Space-authority config — member list, credentials (later), space - * management. Required for any space to exist. */ - authority?: AuthorityConfig; - /** Record-host config — record + blob storage. Required for records to be - * written/read on this deployment. */ - recordHost?: RecordHostConfig; -} - -export interface SpaceRow { - uri: string; - ownerDid: string; - type: string; - key: string; - serviceDid: string; - appPolicyRef: string | null; - appPolicy: AppPolicy | null; - createdAt: number; - deletedAt: number | null; -} - -export interface SpaceMemberRow { - spaceUri: string; - did: string; - addedAt: number; - addedBy: string | null; -} - -export interface StoredRecord { - spaceUri: string; - collection: string; - authorDid: string; - rkey: string; - cid: string | null; - record: Record<string, unknown>; - createdAt: number; -} - -export interface ListOptions { - byUser?: string; - cursor?: string; - limit?: number; -} - -export interface ListResult { - records: StoredRecord[]; - cursor?: string; -} - -export interface ListSpacesOptions { - type?: string; - ownerDid?: string; - memberDid?: string; - limit?: number; - cursor?: string; -} - -export interface CollectionCount { - collection: string; - count: number; -} - -/** What a token holder can do with this invite. - * - `'join'`: must be redeemed while signed in; redeemer becomes a member. - * - `'read'`: bearer-only — token itself grants read access to the space; cannot be redeemed. - * - `'read-join'`: both — anonymous holders read; signed-in holders may also redeem to join. */ -export type InviteKind = "join" | "read" | "read-join"; - -export interface InviteRow { - tokenHash: string; - spaceUri: string; - kind: InviteKind; - expiresAt: number | null; - maxUses: number | null; - usedCount: number; - createdBy: string; - createdAt: number; - revokedAt: number | null; - note: string | null; -} - -export interface CreateInviteInput { - spaceUri: string; - tokenHash: string; - kind: InviteKind; - expiresAt: number | null; - maxUses: number | null; - createdBy: string; - note: string | null; -} - -export interface RedeemInviteResult { - spaceUri: string; -} - -export interface BlobMetaRow { - spaceUri: string; - cid: string; - mimeType: string; - size: number; - authorDid: string; - createdAt: number; -} - -/** Local cache on the record host: this space is accepted here, and `authority_did` - * is the DID authorized to sign credentials for it. Populated via the - * `recordHost.enroll` endpoint or auto-populated by the authority's - * createSpace when both roles run in one process. */ -export interface EnrollmentRow { - spaceUri: string; - authorityDid: string; - enrolledAt: number; - enrolledBy: string; -} - -export interface ListBlobsOptions { - byUser?: string; - cursor?: string; - limit?: number; -} - -export interface ListBlobsResult { - blobs: BlobMetaRow[]; - cursor?: string; -} - -/** **Space authority** interface — owner of the space's ACL state and - * (eventually) credential issuer. Holds the member list, manages invites, - * governs space lifecycle and app policy. Does NOT touch records or blobs. - * - * In a fully-split deployment this is a separate service; today the - * HostedAdapter implements both this and {@link RecordHost} against one DB. */ -export interface SpaceAuthority { - // Space lifecycle - createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow>; - getSpace(spaceUri: string): Promise<SpaceRow | null>; - listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }>; - deleteSpace(spaceUri: string): Promise<void>; - updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void>; - - // Members - addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void>; - removeMember(spaceUri: string, did: string): Promise<void>; - getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null>; - listMembers(spaceUri: string): Promise<SpaceMemberRow[]>; - /** Bulk-apply a membership diff. Used only by the community module's reconciler; - * not exposed as an XRPC endpoint. */ - applyMembershipDiff( - spaceUri: string, - adds: string[], - removes: string[], - addedBy: string | null - ): Promise<void>; - - // Invites (token primitive — issued by the authority, scoped to a space) - createInvite(input: CreateInviteInput): Promise<InviteRow>; - listInvites(spaceUri: string, options?: { includeRevoked?: boolean }): Promise<InviteRow[]>; - revokeInvite(tokenHash: string): Promise<boolean>; - /** Look up an invite without consuming it. Used to validate read-token bearer access. */ - getInvite(tokenHash: string): Promise<InviteRow | null>; - /** Atomically mark a join-capable invite as used. Returns the row if usable - * (kind allows join, not expired/revoked/exhausted), null otherwise. */ - redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null>; -} - -/** **Record host** interface — stores records and blobs for a space, plus - * the local enrollment table that decides which spaces this host accepts. - * - * Trust model: the host trusts whatever credential the authority signs, so - * long as the authority is the one named in the local enrollment for this - * space. Enrollment is the consent step — the host owner agrees to spend - * storage on a given space, scoped to a specific authority. */ -export interface RecordHost { - // Enrollment - enroll(input: EnrollmentRow): Promise<void>; - getEnrollment(spaceUri: string): Promise<EnrollmentRow | null>; - listEnrollments(options?: { authorityDid?: string; limit?: number }): Promise<EnrollmentRow[]>; - removeEnrollment(spaceUri: string): Promise<void>; - - // Records - putRecord(record: StoredRecord): Promise<void>; - getRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<StoredRecord | null>; - listRecords(spaceUri: string, collection: string, options?: ListOptions): Promise<ListResult>; - deleteRecord(spaceUri: string, collection: string, authorDid: string, rkey: string): Promise<void>; - listCollections(spaceUri: string, options?: { byUser?: string }): Promise<CollectionCount[]>; - - // Blobs (metadata only; bytes live on BlobAdapter) - putBlobMeta(row: BlobMetaRow): Promise<void>; - getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null>; - listBlobMeta(spaceUri: string, options?: ListBlobsOptions): Promise<ListBlobsResult>; - deleteBlobMeta(spaceUri: string, cid: string): Promise<void>; - /** Find blob rows older than `cutoff` whose CIDs are not referenced in any - * record JSON in this space. Capped at `limit` to bound a single GC pass. */ - findOrphanBlobs(spaceUri: string, cutoff: number, limit: number): Promise<BlobMetaRow[]>; -} - -/** Combined adapter. Used internally where a single object satisfies both - * roles (today's HostedAdapter, the community reconciler, the realtime - * publishing wrapper). Phases 5+ replace consumers of this with two - * injected interfaces. */ -export type StorageAdapter = SpaceAuthority & RecordHost; - -export interface AdapterContext { - db: Database; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/spaces/uri.ts b/packages/contrail/src/core/spaces/uri.ts index e73ec21..1129419 100644 --- a/packages/contrail/src/core/spaces/uri.ts +++ b/packages/contrail/src/core/spaces/uri.ts @@ -1,37 +1 @@ -/** Centralized space URI construction / parsing. - * - * Permissioned spaces are addressed by (ownerDid, type, key) and use the - * `ats://` scheme — distinct from atproto record URIs (`at://`) so the two - * can't be confused at any layer (logs, params, dispatch). Tracks the rough - * spec at https://dholms.leaflet.pub/3mhj6bcqats2o. - * - * Record URIs inside a space are minted by authorDid for index purposes - * (`at://<authorDid>/<collection>/<rkey>`); the spec is explicitly undecided - * about authority (user vs space owner), so we don't expose those as a - * canonical record address — they're storage-internal. */ - -export interface SpaceUriParts { - ownerDid: string; - type: string; - key: string; -} - -/** Build a space URI from its three addressing components. */ -export function buildSpaceUri(parts: SpaceUriParts): string { - return `ats://${parts.ownerDid}/${parts.type}/${parts.key}`; -} - -/** Parse a space URI into its components, or null if malformed. */ -export function parseSpaceUri(uri: string): SpaceUriParts | null { - if (!uri.startsWith("ats://")) return null; - const rest = uri.slice("ats://".length); - const [ownerDid, type, key, ...extra] = rest.split("/"); - if (!ownerDid || !type || !key || extra.length > 0) return null; - return { ownerDid, type, key }; -} - -/** Build a record URI under a given author. Used only as a secondary index key - * inside storage — not a canonical address for permissioned records. */ -export function buildRecordUri(authorDid: string, collection: string, rkey: string): string { - return `at://${authorDid}/${collection}/${rkey}`; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index 3b9dc48..1129419 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -1,513 +1 @@ -import type { SqlDialect } from "./dialect"; - -// Database interface — D1 implements this natively -export interface Database { - prepare(sql: string): Statement; - batch(stmts: Statement[]): Promise<any[]>; - dialect?: SqlDialect; -} - -export interface Statement { - bind(...values: any[]): Statement; - run(): Promise<any>; - all<T = any>(): Promise<{ results: T[] }>; - first<T = any>(): Promise<T | null>; -} - -// Config types - -export interface QueryableField { - type?: "range"; -} - -export interface RelationConfig { - /** Short name of the child collection (a key in `collections`). */ - collection: string; - field?: string; - match?: "uri" | "did"; - groupBy?: string; - /** Enable materialized count columns on the parent. Defaults to true. */ - count?: boolean; - /** Count distinct values of a field (e.g. "did" for unique users) instead of total records. */ - countDistinct?: string; - /** Pre-resolved group mappings: shortName → full token (e.g. { going: "community.lexicon.calendar.rsvp#going" }). Auto-computed from groupBy if omitted. */ - groups?: Record<string, string>; -} - -/** A forward reference: this collection's records point at another collection. */ -export interface ReferenceConfig { - /** Short name of the target collection. */ - collection: string; - /** Field on this collection's records containing the target URI. */ - field: string; -} - -export type CustomQueryHandler = ( - db: Database, - params: URLSearchParams, - config: ContrailConfig -) => Promise<Response>; - -export interface RecordSource { - joins?: string; - conditions?: string[]; - params?: (string | number)[]; -} - -export type PipelineQueryHandler = ( - db: Database, - params: URLSearchParams, - config: ContrailConfig -) => Promise<RecordSource>; - -export interface FeedConfig { - /** Short name of the follow collection. */ - follow: string; - /** Short names of target collections to fan out to. */ - targets: string[]; - /** Max feed items per user (default: 200). Oldest items are pruned after backfill. */ - maxItems?: number; -} - -export const DEFAULT_FEED_MAX_ITEMS = 200; - -export type CollectionMethod = "listRecords" | "getRecord"; -export const DEFAULT_COLLECTION_METHODS: CollectionMethod[] = [ - "listRecords", - "getRecord", -]; - -export interface CollectionConfig { - /** Full NSID of the record type this collection indexes. */ - collection: string; - /** Include this collection in Jetstream ingest / discovery (default true). - * Set false for dependent collections (auto-fetched on demand). */ - discover?: boolean; - queryable?: Record<string, QueryableField>; - relations?: Record<string, RelationConfig>; - /** Forward references: fields on this collection's records that point at another collection. */ - references?: Record<string, ReferenceConfig>; - queries?: Record<string, CustomQueryHandler>; - pipelineQueries?: Record<string, PipelineQueryHandler>; - /** FTS5 search fields. Provide an array of field names to enable full-text search. Omit or set to false to disable. */ - searchable?: string[] | false; - /** XRPC methods to emit. Defaults to ['listRecords', 'getRecord']. */ - methods?: CollectionMethod[]; - /** When spaces are enabled globally, emit a parallel spaces_records_<short> table - * so this collection can also live inside spaces. Defaults to true. */ - allowInSpaces?: boolean; -} - -export interface ProfileConfig { - /** Full NSID of the profile record type. */ - collection: string; - /** Short name used for table/endpoint naming. Defaults to the NSID's last segment. */ - shortName?: string; - rkey?: string; // defaults to "self" -} - -export const DEFAULT_PROFILES: ProfileConfig[] = [ - { collection: "app.bsky.actor.profile", shortName: "profile" }, -]; - -/** Normalize a profiles config entry (string or object) into ProfileConfig. */ -export function normalizeProfileConfig( - p: string | ProfileConfig -): ProfileConfig { - if (typeof p === "string") { - return { collection: p, shortName: deriveShortName(p) }; - } - return { ...p, shortName: p.shortName ?? deriveShortName(p.collection) }; -} - -/** Last NSID segment, used as fallback short name. */ -export function deriveShortName(nsid: string): string { - const parts = nsid.split("."); - return parts[parts.length - 1] ?? nsid; -} - -export const DEFAULT_JETSTREAMS = [ - "wss://jetstream1.us-east.bsky.network", -]; - -export const DEFAULT_RELAYS = [ - "https://relay1.us-east.bsky.network" -]; - -export interface Logger { - log(...args: any[]): void; - warn(...args: any[]): void; - error(...args: any[]): void; -} - -export interface ContrailConfig { - namespace: string; - /** Collections to index, keyed by short name. Short names become endpoint URL segments - * (`<namespace>.<short>.listRecords`) and table suffixes (`records_<short>`). */ - collections: Record<string, CollectionConfig>; - profiles?: (string | ProfileConfig)[]; - relays?: string[]; - jetstreams?: string[]; - feeds?: Record<string, FeedConfig>; - logger?: Logger; - /** Expose the notifyOfUpdate HTTP endpoint. Off by default. - * Set to `true` for open access, or a string to require `Authorization: Bearer <secret>`. */ - notify?: boolean | string; - /** Permissioned spaces configuration. When set, the service exposes space XRPCs. */ - spaces?: import("./spaces/types").SpacesConfig; - /** Community module configuration. Typed by the community package via - * declaration merging — contrail core only knows it's "something the - * community package consumes." Set when wiring community via - * `createCommunityIntegration({ ... })`. Requires `spaces.authority`. */ - community?: unknown; - /** Realtime module configuration. When set, the service exposes ticket + SSE/WS - * subscribe XRPCs, and wraps the spaces adapter to publish events after writes. */ - realtime?: import("./realtime/types").RealtimeConfig; - /** Labels module configuration. When set, contrail subscribes to the - * configured labelers, indexes their labels into a single `labels` table, - * and hydrates `record.labels` onto `listRecords` / `getRecord` / profile - * responses gated by the caller's `atproto-accept-labelers` header. */ - labels?: import("./labels/types").LabelsConfig; - /** Customize the auto-generated `<namespace>.authFull` lexicon. */ - permissionSet?: PermissionSetConfig; -} - -/** Single entry in an atproto permission-set's `permissions` array. - * See https://atproto.com/guides/permission-sets for the full schema. */ -export type PermissionEntry = - | { type: "permission"; resource: "rpc"; lxm?: string[]; aud?: string; inheritAud?: boolean } - | { type: "permission"; resource: "repo"; collection?: string[] } - | { type: "permission"; resource: "blob"; accept?: string[]; maxSize?: number } - | { type: "permission"; resource: "account"; attr?: string[] } - | { type: "permission"; resource: string; [key: string]: unknown }; - -export interface PermissionSetConfig { - /** Shown on the OAuth consent screen. Defaults to the namespace. */ - title?: string; - /** Shown on the OAuth consent screen. Defaults to a generated description. */ - description?: string; - /** Extra permission entries appended after the auto-generated rpc entry — - * e.g. repo writes for collections your app needs the user to create, or - * blob permissions for uploads. */ - additional?: PermissionEntry[]; -} - -export interface ResolvedRelation { - /** Short name of the child collection. */ - collection: string; - groupBy: string; - groups: Record<string, string>; // shortName → full token value -} - -export interface ResolvedMaps { - queryable: Record<string, Record<string, QueryableField>>; - relations: Record<string, Record<string, ResolvedRelation>>; - /** Reverse map: full record NSID → short name. */ - nsidToShort: Record<string, string>; -} - -/** Config after resolveConfig() — has computed queryable/relation maps attached. */ -export interface ResolvedContrailConfig extends ContrailConfig { - _resolved: ResolvedMaps; -} - -/** - * Resolve config: apply defaults, auto-add profile collections, compute queryable maps. - */ -export function resolveConfig(config: ContrailConfig): ResolvedContrailConfig { - const profiles = (config.profiles ?? DEFAULT_PROFILES).map( - normalizeProfileConfig - ); - const collections = { ...config.collections }; - for (const p of profiles) { - const short = p.shortName!; - if (!collections[short]) { - collections[short] = { collection: p.collection, discover: false }; - } - } - - // Auto-add follow collections from feed configs as dependent collections if they're - // not already listed. Feed config already uses short names so nothing to resolve — - // but if the user forgot to declare the follow collection, we can't auto-add it without - // knowing its NSID. In that case we warn later via validateConfig. - - const base = { - ...config, - collections, - profiles, - jetstreams: config.jetstreams ?? DEFAULT_JETSTREAMS, - relays: config.relays ?? DEFAULT_RELAYS, - logger: config.logger ?? console, - }; - - return { - ...base, - _resolved: _resolveQueryableMaps(base), - }; -} - -function _resolveQueryableMaps(config: ContrailConfig): ResolvedMaps { - const queryable: Record<string, Record<string, QueryableField>> = {}; - const relations: Record<string, Record<string, ResolvedRelation>> = {}; - const nsidToShort: Record<string, string> = {}; - - for (const [short, colConfig] of Object.entries(config.collections)) { - nsidToShort[colConfig.collection] = short; - - if (colConfig.queryable) { - queryable[short] = colConfig.queryable; - } - - if (colConfig.relations) { - for (const [relName, rel] of Object.entries(colConfig.relations)) { - if (!rel.groupBy) continue; - const groups: Record<string, string> = rel.groups ? { ...rel.groups } : {}; - if (Object.keys(groups).length > 0) { - if (!relations[short]) relations[short] = {}; - relations[short][relName] = { - collection: rel.collection, - groupBy: rel.groupBy, - groups, - }; - } - } - } - } - - return { queryable, relations, nsidToShort }; -} - -export function getFeedFollowShortNames(config: ContrailConfig): string[] { - if (!config.feeds) return []; - return [...new Set(Object.values(config.feeds).map((f) => f.follow))]; -} - -/** Alias for getFeedFollowShortNames. */ -export const getFeedFollowCollections = getFeedFollowShortNames; - -// Record types - -export interface RecordRow { - uri: string; - did: string; - collection: string; // full NSID - rkey: string; - cid: string | null; - record: string | null; - time_us: number; - indexed_at: number; - /** Set when the row originates from a per-space table. Used by the - * pipeline/hydration/response layers to route child queries to the same - * space and tag the output. */ - space?: string; -} - -export interface IngestEvent { - uri: string; - did: string; - collection: string; // full NSID - rkey: string; - operation: "create" | "update" | "delete"; - cid: string | null; - record: string | null; - time_us: number; - indexed_at: number; -} - -// Validation - -const SAFE_FIELD_NAME = /^[a-zA-Z0-9_.]+$/; -const SAFE_SHORT_NAME = /^[a-zA-Z][a-zA-Z0-9]*$/; - -export function validateFieldName(field: string): string { - if (!SAFE_FIELD_NAME.test(field)) { - throw new Error(`Invalid field name: ${field}`); - } - return field; -} - -function validateShortName(short: string): void { - if (!SAFE_SHORT_NAME.test(short)) { - throw new Error( - `Invalid collection short name: "${short}". Must be alphanumeric, starting with a letter.` - ); - } -} - -export function validateConfig(config: ContrailConfig): void { - const shortNames = new Set<string>(); - for (const [short, colConfig] of Object.entries(config.collections)) { - validateShortName(short); - if (shortNames.has(short)) { - throw new Error(`Duplicate collection short name: ${short}`); - } - shortNames.add(short); - - if (!colConfig.collection) { - throw new Error(`Collection "${short}" is missing required 'collection' field (NSID)`); - } - - for (const field of Object.keys(colConfig.queryable ?? {})) { - validateFieldName(field); - } - for (const [, rel] of Object.entries(colConfig.relations ?? {})) { - if (rel.field) validateFieldName(rel.field); - if (rel.groupBy) validateFieldName(rel.groupBy); - if (rel.countDistinct) validateFieldName(rel.countDistinct); - if (!config.collections[rel.collection]) { - throw new Error( - `Relation in "${short}" references unknown collection short name "${rel.collection}"` - ); - } - } - for (const [, ref] of Object.entries(colConfig.references ?? {})) { - validateFieldName(ref.field); - if (!config.collections[ref.collection]) { - throw new Error( - `Reference in "${short}" references unknown collection short name "${ref.collection}"` - ); - } - } - if (Array.isArray(colConfig.searchable)) { - for (const field of colConfig.searchable) { - validateFieldName(field); - } - } - } - - if (config.feeds) { - for (const [feedName, feed] of Object.entries(config.feeds)) { - if (!config.collections[feed.follow]) { - throw new Error( - `Feed "${feedName}" references unknown follow collection "${feed.follow}"` - ); - } - for (const target of feed.targets) { - if (!config.collections[target]) { - throw new Error( - `Feed "${feedName}" references unknown target collection "${target}"` - ); - } - } - } - } - - if (config.community && !config.spaces?.authority) { - throw new Error( - "Invalid config: `community` requires `spaces.authority`. Community-owned spaces reuse the spaces storage adapter." - ); - } -} - -// Helpers - -export function getNestedValue(obj: any, path: string): any { - let current = obj; - for (const key of path.split(".")) { - if (current == null) return undefined; - current = current[key]; - } - return current; -} - -const DEFAULT_RELATION_FIELD = "subject.uri"; - -export function getRelationField(rel: RelationConfig): string { - return rel.field ?? DEFAULT_RELATION_FIELD; -} - -/** Sanitize a short name for use in SQL identifiers (already-validated; kept for paranoia). */ -function sanitizeIdentifier(name: string): string { - return name.replace(/[^a-zA-Z0-9]/g, "_"); -} - -/** Total-count column name for a relation targeting the given short name. */ -export function countColumnName(childShortName: string): string { - return "count_" + sanitizeIdentifier(childShortName); -} - -/** Grouped-count column name: `count_<child-short>_<groupKey>`. */ -export function groupedCountColumnName( - childShortName: string, - groupKey: string -): string { - return `count_${sanitizeIdentifier(childShortName)}_${sanitizeIdentifier(groupKey)}`; -} - -/** Table name for a collection's records. */ -export function recordsTableName(shortName: string): string { - return "records_" + sanitizeIdentifier(shortName); -} - -/** Table name for a collection's records inside spaces. */ -export function spacesRecordsTableName(shortName: string): string { - return "spaces_records_" + sanitizeIdentifier(shortName); -} - -/** All collection short names. */ -export function getCollectionShortNames(config: ContrailConfig): string[] { - return Object.keys(config.collections); -} - -/** Alias: collection short names (same as getCollectionShortNames). */ -export const getCollectionNames = getCollectionShortNames; - -/** All indexed record NSIDs (what Jetstream filters on). */ -export function getCollectionNsids(config: ContrailConfig): string[] { - return Object.values(config.collections).map((c) => c.collection); -} - -export function getDependentShortNames(config: ContrailConfig): string[] { - return Object.entries(config.collections) - .filter(([, c]) => c.discover === false) - .map(([name]) => name); -} - -export function getDiscoverableShortNames(config: ContrailConfig): string[] { - return Object.entries(config.collections) - .filter(([, c]) => c.discover !== false) - .map(([name]) => name); -} - -/** Aliases for readability elsewhere. These return short names (new semantic). */ -export const getDependentCollections = getDependentShortNames; -export const getDiscoverableCollections = getDiscoverableShortNames; - -/** Short names of collections the user declared with `discover !== false`, mapped to NSIDs. */ -export function getDiscoverableNsids(config: ContrailConfig): string[] { - return Object.values(config.collections) - .filter((c) => c.discover !== false) - .map((c) => c.collection); -} - -export function getDependentNsids(config: ContrailConfig): string[] { - return Object.values(config.collections) - .filter((c) => c.discover === false) - .map((c) => c.collection); -} - -/** Short name for a record NSID, if known. */ -export function shortNameForNsid( - config: ContrailConfig, - nsid: string -): string | undefined { - const resolved = (config as ResolvedContrailConfig)._resolved; - if (resolved?.nsidToShort) return resolved.nsidToShort[nsid]; - for (const [short, c] of Object.entries(config.collections)) { - if (c.collection === nsid) return short; - } - return undefined; -} - -/** Full NSID for a collection short name. */ -export function nsidForShortName( - config: ContrailConfig, - short: string -): string | undefined { - return config.collections[short]?.collection; -} - -/** The methods a collection should expose via XRPC. */ -export function getCollectionMethods(cfg: CollectionConfig): CollectionMethod[] { - return cfg.methods ?? DEFAULT_COLLECTION_METHODS; -} +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 04f1a15..0738fd3 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -1,4 +1,7 @@ import { defineConfig } from "vitest/config"; +import path from "node:path"; + +const baseSrc = path.resolve(__dirname, "../contrail-base/src"); export default defineConfig({ test: { @@ -6,4 +9,13 @@ export default defineConfig({ // PostgreSQL tests share a single database and cannot run in parallel fileParallelism: false, }, + resolve: { + alias: { + // Resolve workspace-internal contrail-base subpaths to source so tests + // don't run through the dist (where tsup drops `node:` prefixes). + "@atmo-dev/contrail-base/sqlite": path.join(baseSrc, "adapters/sqlite.ts"), + "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), + "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), + }, + }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 48ffc2e..01c655a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -376,6 +376,9 @@ importers: '@atcute/xrpc-server': specifier: ^0.1.12 version: 0.1.12 + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base cac: specifier: ^7.0.0 version: 7.0.0 @@ -411,6 +414,49 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-base: + dependencies: + '@atcute/atproto': + specifier: ^3.1.10 + version: 3.1.11 + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atcute/client': + specifier: ^4.2.1 + version: 4.2.1 + '@atcute/identity': + specifier: ^1.1.4 + version: 1.1.4 + '@atcute/identity-resolver': + specifier: ^1.2.2 + version: 1.2.2(@atcute/identity@1.1.4) + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atcute/xrpc-server': + specifier: ^0.1.12 + version: 0.1.12 + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + '@types/node': + specifier: ^25.5.0 + version: 25.6.0 + '@types/pg': + specifier: ^8.20.0 + version: 8.20.0 + pg: + specifier: ^8.20.0 + version: 8.20.0 + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/contrail-community: dependencies: '@atcute/atproto': @@ -434,6 +480,9 @@ importers: '@atmo-dev/contrail': specifier: workspace:* version: link:../contrail + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base hono: specifier: ^4.12.8 version: 4.12.15 -- 2.51.2 From edf66ed0cf948b77d63648f67074a324b084956d Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 01:35:24 +0200 Subject: [PATCH 09/19] phase 7a ^^ --- packages/contrail-authority/package.json | 44 +++ packages/contrail-authority/src/adapter.ts | 317 +++++++++++++++++ packages/contrail-authority/src/index.ts | 19 ++ packages/contrail-authority/src/schema.ts | 52 +++ .../contrail-authority/tsconfig.build.json | 7 + packages/contrail-authority/tsconfig.json | 7 + packages/contrail-authority/tsup.config.ts | 11 + packages/contrail/package.json | 1 + packages/contrail/src/core/spaces/adapter.ts | 318 +----------------- packages/contrail/vitest.config.ts | 6 +- pnpm-lock.yaml | 25 ++ 11 files changed, 505 insertions(+), 302 deletions(-) create mode 100644 packages/contrail-authority/package.json create mode 100644 packages/contrail-authority/src/adapter.ts create mode 100644 packages/contrail-authority/src/index.ts create mode 100644 packages/contrail-authority/src/schema.ts create mode 100644 packages/contrail-authority/tsconfig.build.json create mode 100644 packages/contrail-authority/tsconfig.json create mode 100644 packages/contrail-authority/tsup.config.ts diff --git a/packages/contrail-authority/package.json b/packages/contrail-authority/package.json new file mode 100644 index 0000000..07c0477 --- /dev/null +++ b/packages/contrail-authority/package.json @@ -0,0 +1,44 @@ +{ + "name": "@atmo-dev/contrail-authority", + "version": "0.6.0", + "description": "Default space-authority implementation for contrail — member list, invites, app policy, credential issuance. Contrail's binary-membership ACL flavor; for ladder-style access levels see @atmo-dev/contrail-community.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-authority" + }, + "keywords": [ + "atproto", + "contrail" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/cid": "^2.4.1", + "@atcute/lexicons": "^1.2.9", + "@atmo-dev/contrail-base": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "license": "MIT" +} diff --git a/packages/contrail-authority/src/adapter.ts b/packages/contrail-authority/src/adapter.ts new file mode 100644 index 0000000..66ffdfb --- /dev/null +++ b/packages/contrail-authority/src/adapter.ts @@ -0,0 +1,317 @@ +/** Default {@link SpaceAuthority} implementation backed by a Database. + * + * Owns three tables (`spaces`, `spaces_members`, `spaces_invites`) and + * exposes the authority surface: space lifecycle, member list, invite + * storage, app-policy management. + * + * Designed for inheritance — fields are `protected` so a record-host + * adapter (or, transitionally, contrail's all-in-one HostedAdapter) can + * extend this class to add its own methods without re-implementing the + * authority side. */ + +import type { + ContrailConfig, + Database, + SpaceAuthority, + AppPolicy, + CreateInviteInput, + InviteKind, + InviteRow, + ListSpacesOptions, + SpaceMemberRow, + SpaceRow, +} from "@atmo-dev/contrail-base"; + +export function parseJson<T>(value: unknown): T | null { + if (value == null) return null; + if (typeof value === "string") { + try { + return JSON.parse(value) as T; + } catch { + return null; + } + } + return value as T; +} + +export function toNum(v: unknown): number { + return typeof v === "string" ? Number(v) : (v as number); +} + +export function mapSpaceRow(row: any): SpaceRow { + return { + uri: row.uri, + ownerDid: row.owner_did, + type: row.type, + key: row.key, + serviceDid: row.service_did, + appPolicyRef: row.app_policy_ref ?? null, + appPolicy: parseJson<AppPolicy>(row.app_policy), + createdAt: toNum(row.created_at), + deletedAt: row.deleted_at == null ? null : toNum(row.deleted_at), + }; +} + +export function mapMemberRow(row: any): SpaceMemberRow { + return { + spaceUri: row.space_uri, + did: row.did, + addedAt: toNum(row.added_at), + addedBy: row.added_by ?? null, + }; +} + +export function mapInviteRow(row: any): InviteRow { + return { + tokenHash: row.token_hash, + spaceUri: row.space_uri, + kind: (row.kind ?? "join") as InviteKind, + expiresAt: row.expires_at == null ? null : toNum(row.expires_at), + maxUses: row.max_uses == null ? null : Number(row.max_uses), + usedCount: Number(row.used_count), + createdBy: row.created_by, + createdAt: toNum(row.created_at), + revokedAt: row.revoked_at == null ? null : toNum(row.revoked_at), + note: row.note ?? null, + }; +} + +export class HostedAuthorityAdapter implements SpaceAuthority { + constructor( + protected readonly db: Database, + protected readonly config?: ContrailConfig + ) {} + + async createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow> { + const now = Date.now(); + await this.db + .prepare( + `INSERT INTO spaces (uri, owner_did, type, key, service_did, app_policy_ref, app_policy, created_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)` + ) + .bind( + space.uri, + space.ownerDid, + space.type, + space.key, + space.serviceDid, + space.appPolicyRef, + space.appPolicy ? JSON.stringify(space.appPolicy) : null, + now + ) + .run(); + return { ...space, createdAt: now, deletedAt: null }; + } + + async getSpace(spaceUri: string): Promise<SpaceRow | null> { + const row = await this.db + .prepare(`SELECT * FROM spaces WHERE uri = ? AND deleted_at IS NULL`) + .bind(spaceUri) + .first<any>(); + return row ? mapSpaceRow(row) : null; + } + + async listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }> { + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["s.deleted_at IS NULL"]; + const params: any[] = []; + let join = ""; + + if (options.type) { + clauses.push("s.type = ?"); + params.push(options.type); + } + if (options.ownerDid) { + clauses.push("s.owner_did = ?"); + params.push(options.ownerDid); + } + if (options.memberDid) { + join = "JOIN spaces_members m ON m.space_uri = s.uri"; + clauses.push("m.did = ?"); + params.push(options.memberDid); + } + if (options.cursor) { + clauses.push("s.created_at < ?"); + params.push(Number(options.cursor)); + } + + const sql = `SELECT s.* FROM spaces s ${join} + WHERE ${clauses.join(" AND ")} + ORDER BY s.created_at DESC + LIMIT ?`; + params.push(limit + 1); + + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const spaces = results.map(mapSpaceRow); + let cursor: string | undefined; + if (spaces.length > limit) { + const next = spaces.pop()!; + cursor = String(next.createdAt); + } + return { spaces, cursor }; + } + + async deleteSpace(spaceUri: string): Promise<void> { + await this.db + .prepare(`UPDATE spaces SET deleted_at = ? WHERE uri = ?`) + .bind(Date.now(), spaceUri) + .run(); + } + + async updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void> { + await this.db + .prepare(`UPDATE spaces SET app_policy = ? WHERE uri = ?`) + .bind(JSON.stringify(appPolicy), spaceUri) + .run(); + } + + async addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void> { + await this.db + .prepare( + `INSERT INTO spaces_members (space_uri, did, added_at, added_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri, did) DO NOTHING` + ) + .bind(spaceUri, did, Date.now(), addedBy) + .run(); + } + + async removeMember(spaceUri: string, did: string): Promise<void> { + await this.db + .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) + .bind(spaceUri, did) + .run(); + } + + async getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null> { + const row = await this.db + .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? AND did = ?`) + .bind(spaceUri, did) + .first<any>(); + return row ? mapMemberRow(row) : null; + } + + async listMembers(spaceUri: string): Promise<SpaceMemberRow[]> { + const { results } = await this.db + .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? ORDER BY added_at ASC`) + .bind(spaceUri) + .all<any>(); + return results.map(mapMemberRow); + } + + async applyMembershipDiff( + spaceUri: string, + adds: string[], + removes: string[], + addedBy: string | null + ): Promise<void> { + const now = Date.now(); + const stmts: any[] = []; + for (const did of adds) { + stmts.push( + this.db + .prepare( + `INSERT INTO spaces_members (space_uri, did, added_at, added_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri, did) DO NOTHING` + ) + .bind(spaceUri, did, now, addedBy) + ); + } + for (const did of removes) { + stmts.push( + this.db + .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) + .bind(spaceUri, did) + ); + } + if (stmts.length > 0) { + await this.db.batch(stmts); + } + } + + async createInvite(input: CreateInviteInput): Promise<InviteRow> { + const now = Date.now(); + await this.db + .prepare( + `INSERT INTO spaces_invites (token_hash, space_uri, kind, expires_at, max_uses, used_count, created_by, created_at, note) + VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?)` + ) + .bind( + input.tokenHash, + input.spaceUri, + input.kind, + input.expiresAt, + input.maxUses, + input.createdBy, + now, + input.note + ) + .run(); + return { + tokenHash: input.tokenHash, + spaceUri: input.spaceUri, + kind: input.kind, + expiresAt: input.expiresAt, + maxUses: input.maxUses, + usedCount: 0, + createdBy: input.createdBy, + createdAt: now, + revokedAt: null, + note: input.note, + }; + } + + async getInvite(tokenHash: string): Promise<InviteRow | null> { + const row = await this.db + .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) + .bind(tokenHash) + .first<any>(); + return row ? mapInviteRow(row) : null; + } + + async listInvites( + spaceUri: string, + options: { includeRevoked?: boolean } = {} + ): Promise<InviteRow[]> { + const sql = options.includeRevoked + ? `SELECT * FROM spaces_invites WHERE space_uri = ? ORDER BY created_at DESC` + : `SELECT * FROM spaces_invites WHERE space_uri = ? AND revoked_at IS NULL ORDER BY created_at DESC`; + const { results } = await this.db.prepare(sql).bind(spaceUri).all<any>(); + return results.map(mapInviteRow); + } + + async revokeInvite(tokenHash: string): Promise<boolean> { + const res = await this.db + .prepare(`UPDATE spaces_invites SET revoked_at = ? WHERE token_hash = ? AND revoked_at IS NULL`) + .bind(Date.now(), tokenHash) + .run(); + const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; + return Number(changes) > 0; + } + + async redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null> { + // Atomic: increment used_count only if the invite is usable right now AND + // its kind allows redemption (read-only tokens cannot be consumed for membership). + const res = await this.db + .prepare( + `UPDATE spaces_invites + SET used_count = used_count + 1 + WHERE token_hash = ? + AND kind IN ('join', 'read-join') + AND revoked_at IS NULL + AND (expires_at IS NULL OR expires_at > ?) + AND (max_uses IS NULL OR used_count < max_uses)` + ) + .bind(tokenHash, now) + .run(); + const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; + if (Number(changes) === 0) return null; + + const row = await this.db + .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) + .bind(tokenHash) + .first<any>(); + return row ? mapInviteRow(row) : null; + } +} diff --git a/packages/contrail-authority/src/index.ts b/packages/contrail-authority/src/index.ts new file mode 100644 index 0000000..638b762 --- /dev/null +++ b/packages/contrail-authority/src/index.ts @@ -0,0 +1,19 @@ +/** @atmo-dev/contrail-authority — default space-authority implementation. + * + * Owns the authority-side adapter (member list, invites, app policy, space + * lifecycle) and DDL. Route registration currently lives in + * @atmo-dev/contrail and will move here in a subsequent extraction pass. */ + +export { + HostedAuthorityAdapter, + parseJson, + toNum, + mapSpaceRow, + mapMemberRow, + mapInviteRow, +} from "./adapter"; + +export { + buildAuthoritySchema, + applyAuthoritySchema, +} from "./schema"; diff --git a/packages/contrail-authority/src/schema.ts b/packages/contrail-authority/src/schema.ts new file mode 100644 index 0000000..e9f19e9 --- /dev/null +++ b/packages/contrail-authority/src/schema.ts @@ -0,0 +1,52 @@ +/** Authority-side DDL: `spaces`, `spaces_members`, `spaces_invites`. */ + +import type { Database, SqlDialect } from "@atmo-dev/contrail-base"; +import { getDialect } from "@atmo-dev/contrail-base"; + +export function buildAuthoritySchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS spaces ( + uri TEXT PRIMARY KEY, + owner_did TEXT NOT NULL, + type TEXT NOT NULL, + key TEXT NOT NULL, + service_did TEXT NOT NULL, + app_policy_ref TEXT, + app_policy ${dialect.recordColumnType}, + created_at ${dialect.bigintType} NOT NULL, + deleted_at ${dialect.bigintType} + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_owner ON spaces(owner_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_type ON spaces(type)`, + + `CREATE TABLE IF NOT EXISTS spaces_members ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + added_at ${dialect.bigintType} NOT NULL, + added_by TEXT, + PRIMARY KEY (space_uri, did) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_members_did ON spaces_members(did)`, + + `CREATE TABLE IF NOT EXISTS spaces_invites ( + token_hash TEXT PRIMARY KEY, + space_uri TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'join', + expires_at ${dialect.bigintType}, + max_uses INTEGER, + used_count INTEGER NOT NULL DEFAULT 0, + created_by TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + revoked_at ${dialect.bigintType}, + note TEXT + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, + ]; +} + +/** SchemaModule-shaped function suitable for `initSchema({ extraSchemas: [...] })`. */ +export async function applyAuthoritySchema(db: Database): Promise<void> { + const dialect = getDialect(db); + const stmts = buildAuthoritySchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); +} diff --git a/packages/contrail-authority/tsconfig.build.json b/packages/contrail-authority/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-authority/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-authority/tsconfig.json b/packages/contrail-authority/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-authority/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-authority/tsup.config.ts b/packages/contrail-authority/tsup.config.ts new file mode 100644 index 0000000..7d6b7eb --- /dev/null +++ b/packages/contrail-authority/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["@atmo-dev/contrail-base"], +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 8409809..02a833f 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,6 +73,7 @@ "@atcute/jetstream": "^1.0.2", "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-authority": "workspace:*", "@atmo-dev/contrail-base": "workspace:*", "cac": "^7.0.0", "hono": "^4.12.8", diff --git a/packages/contrail/src/core/spaces/adapter.ts b/packages/contrail/src/core/spaces/adapter.ts index f80ba5e..50fb33d 100644 --- a/packages/contrail/src/core/spaces/adapter.ts +++ b/packages/contrail/src/core/spaces/adapter.ts @@ -1,4 +1,13 @@ -import type { ContrailConfig, Database, RelationConfig, ResolvedContrailConfig } from "../types"; +/** Contrail's all-in-one default adapter — extends the authority package's + * {@link HostedAuthorityAdapter} (which owns space lifecycle, member list, + * invites) and adds the record-host methods (records, blobs, enrollment). + * + * Phase 7a step 3 will lift the record-host methods into a separate + * HostedRecordHostAdapter, at which point this class becomes a thin + * composition / re-export. For now we keep both roles in one class so + * consumers can wire a single object that satisfies the full StorageAdapter. */ + +import type { ContrailConfig, RelationConfig, ResolvedContrailConfig } from "../types"; import { shortNameForNsid, spacesRecordsTableName, @@ -9,63 +18,19 @@ import { } from "../types"; import { getDialect } from "../dialect"; import type { - AppPolicy, BlobMetaRow, CollectionCount, - CreateInviteInput, EnrollmentRow, - InviteKind, - InviteRow, ListBlobsOptions, ListBlobsResult, ListOptions, ListResult, - ListSpacesOptions, - SpaceMemberRow, - SpaceRow, StorageAdapter, StoredRecord, } from "./types"; +import type { Database } from "../types"; import { buildRecordUri } from "./uri"; - -function parseJson<T>(value: unknown): T | null { - if (value == null) return null; - if (typeof value === "string") { - try { - return JSON.parse(value) as T; - } catch { - return null; - } - } - return value as T; -} - -function toNum(v: unknown): number { - return typeof v === "string" ? Number(v) : (v as number); -} - -function mapSpaceRow(row: any): SpaceRow { - return { - uri: row.uri, - ownerDid: row.owner_did, - type: row.type, - key: row.key, - serviceDid: row.service_did, - appPolicyRef: row.app_policy_ref ?? null, - appPolicy: parseJson<AppPolicy>(row.app_policy), - createdAt: toNum(row.created_at), - deletedAt: row.deleted_at == null ? null : toNum(row.deleted_at), - }; -} - -function mapMemberRow(row: any): SpaceMemberRow { - return { - spaceUri: row.space_uri, - did: row.did, - addedAt: toNum(row.added_at), - addedBy: row.added_by ?? null, - }; -} +import { HostedAuthorityAdapter, parseJson, toNum } from "@atmo-dev/contrail-authority"; function mapBlobMetaRow(row: any): BlobMetaRow { return { @@ -87,21 +52,6 @@ function mapEnrollmentRow(row: any): EnrollmentRow { }; } -function mapInviteRow(row: any): InviteRow { - return { - tokenHash: row.token_hash, - spaceUri: row.space_uri, - kind: (row.kind ?? "join") as InviteKind, - expiresAt: row.expires_at == null ? null : toNum(row.expires_at), - maxUses: row.max_uses == null ? null : Number(row.max_uses), - usedCount: Number(row.used_count), - createdBy: row.created_by, - createdAt: toNum(row.created_at), - revokedAt: row.revoked_at == null ? null : toNum(row.revoked_at), - note: row.note ?? null, - }; -} - /** Row mapper for per-collection spaces_records_<short> tables. * `collection` is injected by the caller (known from the table name). */ function mapRecordRow(row: any, collection: string): StoredRecord { @@ -116,12 +66,7 @@ function mapRecordRow(row: any, collection: string): StoredRecord { }; } -export class HostedAdapter implements StorageAdapter { - constructor( - private readonly db: Database, - private readonly config?: ContrailConfig - ) {} - +export class HostedAdapter extends HostedAuthorityAdapter implements StorageAdapter { /** Resolve the per-collection spaces table name, or throw if the collection * isn't configured (and therefore has no table). */ private tableFor(collection: string): string { @@ -139,238 +84,7 @@ export class HostedAdapter implements StorageAdapter { return spacesRecordsTableName(short); } - async createSpace(space: Omit<SpaceRow, "createdAt" | "deletedAt">): Promise<SpaceRow> { - const now = Date.now(); - await this.db - .prepare( - `INSERT INTO spaces (uri, owner_did, type, key, service_did, app_policy_ref, app_policy, created_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?)` - ) - .bind( - space.uri, - space.ownerDid, - space.type, - space.key, - space.serviceDid, - space.appPolicyRef, - space.appPolicy ? JSON.stringify(space.appPolicy) : null, - now - ) - .run(); - return { ...space, createdAt: now, deletedAt: null }; - } - - async getSpace(spaceUri: string): Promise<SpaceRow | null> { - const row = await this.db - .prepare(`SELECT * FROM spaces WHERE uri = ? AND deleted_at IS NULL`) - .bind(spaceUri) - .first<any>(); - return row ? mapSpaceRow(row) : null; - } - - async listSpaces(options: ListSpacesOptions): Promise<{ spaces: SpaceRow[]; cursor?: string }> { - const limit = Math.min(options.limit ?? 50, 200); - const clauses: string[] = ["s.deleted_at IS NULL"]; - const params: any[] = []; - let join = ""; - - if (options.type) { - clauses.push("s.type = ?"); - params.push(options.type); - } - if (options.ownerDid) { - clauses.push("s.owner_did = ?"); - params.push(options.ownerDid); - } - if (options.memberDid) { - join = "JOIN spaces_members m ON m.space_uri = s.uri"; - clauses.push("m.did = ?"); - params.push(options.memberDid); - } - if (options.cursor) { - clauses.push("s.created_at < ?"); - params.push(Number(options.cursor)); - } - - const sql = `SELECT s.* FROM spaces s ${join} - WHERE ${clauses.join(" AND ")} - ORDER BY s.created_at DESC - LIMIT ?`; - params.push(limit + 1); - - const { results } = await this.db.prepare(sql).bind(...params).all<any>(); - const spaces = results.map(mapSpaceRow); - let cursor: string | undefined; - if (spaces.length > limit) { - const next = spaces.pop()!; - cursor = String(next.createdAt); - } - return { spaces, cursor }; - } - - async deleteSpace(spaceUri: string): Promise<void> { - await this.db - .prepare(`UPDATE spaces SET deleted_at = ? WHERE uri = ?`) - .bind(Date.now(), spaceUri) - .run(); - } - - async updateSpaceAppPolicy(spaceUri: string, appPolicy: AppPolicy): Promise<void> { - await this.db - .prepare(`UPDATE spaces SET app_policy = ? WHERE uri = ?`) - .bind(JSON.stringify(appPolicy), spaceUri) - .run(); - } - - async addMember(spaceUri: string, did: string, addedBy: string | null): Promise<void> { - await this.db - .prepare( - `INSERT INTO spaces_members (space_uri, did, added_at, added_by) - VALUES (?, ?, ?, ?) - ON CONFLICT (space_uri, did) DO NOTHING` - ) - .bind(spaceUri, did, Date.now(), addedBy) - .run(); - } - - async removeMember(spaceUri: string, did: string): Promise<void> { - await this.db - .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) - .bind(spaceUri, did) - .run(); - } - - async getMember(spaceUri: string, did: string): Promise<SpaceMemberRow | null> { - const row = await this.db - .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? AND did = ?`) - .bind(spaceUri, did) - .first<any>(); - return row ? mapMemberRow(row) : null; - } - - async listMembers(spaceUri: string): Promise<SpaceMemberRow[]> { - const { results } = await this.db - .prepare(`SELECT * FROM spaces_members WHERE space_uri = ? ORDER BY added_at ASC`) - .bind(spaceUri) - .all<any>(); - return results.map(mapMemberRow); - } - - async applyMembershipDiff( - spaceUri: string, - adds: string[], - removes: string[], - addedBy: string | null - ): Promise<void> { - const now = Date.now(); - const stmts: any[] = []; - for (const did of adds) { - stmts.push( - this.db - .prepare( - `INSERT INTO spaces_members (space_uri, did, added_at, added_by) - VALUES (?, ?, ?, ?) - ON CONFLICT (space_uri, did) DO NOTHING` - ) - .bind(spaceUri, did, now, addedBy) - ); - } - for (const did of removes) { - stmts.push( - this.db - .prepare(`DELETE FROM spaces_members WHERE space_uri = ? AND did = ?`) - .bind(spaceUri, did) - ); - } - if (stmts.length > 0) { - await this.db.batch(stmts); - } - } - - async createInvite(input: CreateInviteInput): Promise<InviteRow> { - const now = Date.now(); - await this.db - .prepare( - `INSERT INTO spaces_invites (token_hash, space_uri, kind, expires_at, max_uses, used_count, created_by, created_at, note) - VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?)` - ) - .bind( - input.tokenHash, - input.spaceUri, - input.kind, - input.expiresAt, - input.maxUses, - input.createdBy, - now, - input.note - ) - .run(); - return { - tokenHash: input.tokenHash, - spaceUri: input.spaceUri, - kind: input.kind, - expiresAt: input.expiresAt, - maxUses: input.maxUses, - usedCount: 0, - createdBy: input.createdBy, - createdAt: now, - revokedAt: null, - note: input.note, - }; - } - - async getInvite(tokenHash: string): Promise<InviteRow | null> { - const row = await this.db - .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) - .bind(tokenHash) - .first<any>(); - return row ? mapInviteRow(row) : null; - } - - async listInvites( - spaceUri: string, - options: { includeRevoked?: boolean } = {} - ): Promise<InviteRow[]> { - const sql = options.includeRevoked - ? `SELECT * FROM spaces_invites WHERE space_uri = ? ORDER BY created_at DESC` - : `SELECT * FROM spaces_invites WHERE space_uri = ? AND revoked_at IS NULL ORDER BY created_at DESC`; - const { results } = await this.db.prepare(sql).bind(spaceUri).all<any>(); - return results.map(mapInviteRow); - } - - async revokeInvite(tokenHash: string): Promise<boolean> { - const res = await this.db - .prepare(`UPDATE spaces_invites SET revoked_at = ? WHERE token_hash = ? AND revoked_at IS NULL`) - .bind(Date.now(), tokenHash) - .run(); - const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; - return Number(changes) > 0; - } - - async redeemInvite(tokenHash: string, now: number): Promise<InviteRow | null> { - // Atomic: increment used_count only if the invite is usable right now AND - // its kind allows redemption (read-only tokens cannot be consumed for membership). - const res = await this.db - .prepare( - `UPDATE spaces_invites - SET used_count = used_count + 1 - WHERE token_hash = ? - AND kind IN ('join', 'read-join') - AND revoked_at IS NULL - AND (expires_at IS NULL OR expires_at > ?) - AND (max_uses IS NULL OR used_count < max_uses)` - ) - .bind(tokenHash, now) - .run(); - const changes = (res as any)?.changes ?? (res as any)?.meta?.changes ?? 0; - if (Number(changes) === 0) return null; - - const row = await this.db - .prepare(`SELECT * FROM spaces_invites WHERE token_hash = ?`) - .bind(tokenHash) - .first<any>(); - return row ? mapInviteRow(row) : null; - } + // ---- Enrollment ---- async enroll(input: EnrollmentRow): Promise<void> { await this.db @@ -421,6 +135,8 @@ export class HostedAdapter implements StorageAdapter { .run(); } + // ---- Records ---- + async putRecord(record: StoredRecord): Promise<void> { const table = this.tableFor(record.collection); const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); @@ -675,6 +391,8 @@ export class HostedAdapter implements StorageAdapter { return results; } + // ---- Blobs ---- + async putBlobMeta(row: BlobMetaRow): Promise<void> { const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) VALUES (?, ?, ?, ?, ?, ?) diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 0738fd3..9c32af0 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -2,6 +2,7 @@ import { defineConfig } from "vitest/config"; import path from "node:path"; const baseSrc = path.resolve(__dirname, "../contrail-base/src"); +const authoritySrc = path.resolve(__dirname, "../contrail-authority/src"); export default defineConfig({ test: { @@ -11,11 +12,12 @@ export default defineConfig({ }, resolve: { alias: { - // Resolve workspace-internal contrail-base subpaths to source so tests - // don't run through the dist (where tsup drops `node:` prefixes). + // Resolve workspace-internal contrail-* subpaths to source so tests + // don't run through the dists (where tsup drops `node:` prefixes). "@atmo-dev/contrail-base/sqlite": path.join(baseSrc, "adapters/sqlite.ts"), "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), + "@atmo-dev/contrail-authority": path.join(authoritySrc, "index.ts"), }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 01c655a..6e26e5a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -376,6 +376,9 @@ importers: '@atcute/xrpc-server': specifier: ^0.1.12 version: 0.1.12 + '@atmo-dev/contrail-authority': + specifier: workspace:* + version: link:../contrail-authority '@atmo-dev/contrail-base': specifier: workspace:* version: link:../contrail-base @@ -414,6 +417,28 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-authority: + dependencies: + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/contrail-base: dependencies: '@atcute/atproto': -- 2.51.2 From b003ad6b82da7c43a02344cd0142d3eb02cf9637 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 02:16:03 +0200 Subject: [PATCH 10/19] phase 7a pt2 --- packages/contrail-appview/package.json | 56 + .../contrail-appview/src/core/backfill.ts | 523 +++++++ packages/contrail-appview/src/core/client.ts | 1 + .../src/core/community-integration.ts | 1 + .../contrail-appview/src/core/db/index.ts | 4 + .../contrail-appview/src/core/db/records.ts | 896 ++++++++++++ .../contrail-appview/src/core/db/schema.ts | 355 +++++ packages/contrail-appview/src/core/dialect.ts | 1 + .../contrail-appview/src/core/identity.ts | 1 + .../src/core/invite/community-handler.ts | 1 + .../contrail-appview/src/core/invite/index.ts | 3 + .../src/core/invite/router.ts | 1 + .../contrail-appview/src/core/invite/token.ts | 1 + .../contrail-appview/src/core/jetstream.ts | 280 ++++ .../contrail-appview/src/core/labels/apply.ts | 64 + .../src/core/labels/hydrate.ts | Bin 0 -> 3547 bytes .../src/core/labels/resolve.ts | 134 ++ .../src/core/labels/schema.ts | 30 + .../src/core/labels/select.ts | 64 + .../src/core/labels/subscribe.ts | 315 +++++ .../contrail-appview/src/core/labels/types.ts | 1 + .../contrail-appview/src/core/persistent.ts | 243 ++++ .../src/core/realtime/durable-object.ts | 1 + .../src/core/realtime/in-memory.ts | 1 + .../src/core/realtime/index.ts | 37 + .../src/core/realtime/merge.ts | 1 + .../src/core/realtime/publishing-adapter.ts | 157 +++ .../src/core/realtime/query-filter.ts | 1 + .../src/core/realtime/resolve.ts | 92 ++ .../src/core/realtime/router.ts | 255 ++++ .../contrail-appview/src/core/realtime/sse.ts | 1 + .../src/core/realtime/ticket.ts | 1 + .../src/core/realtime/types.ts | 1 + .../src/core/realtime/websocket.ts | 1 + packages/contrail-appview/src/core/refresh.ts | 267 ++++ .../contrail-appview/src/core/router/admin.ts | 44 + .../src/core/router/collection.ts | 1225 ++++++++++++++++ .../contrail-appview/src/core/router/feed.ts | 134 ++ .../src/core/router/helpers.ts | 67 + .../src/core/router/hydrate.ts | 223 +++ .../contrail-appview/src/core/router/index.ts | 247 ++++ .../src/core/router/notify.ts | 179 +++ .../src/core/router/profiles.ts | 181 +++ packages/contrail-appview/src/core/search.ts | 31 + .../contrail-appview/src/core/spaces/acl.ts | 1 + .../src/core/spaces/adapter.ts | 505 +++++++ .../contrail-appview/src/core/spaces/auth.ts | 1 + .../src/core/spaces/binding.ts | 1 + .../src/core/spaces/blob-adapter.ts | 1 + .../src/core/spaces/blob-gc.ts | 38 + .../src/core/spaces/blob-refs.ts | 27 + .../src/core/spaces/credentials.ts | 1 + .../src/core/spaces/in-process.ts | 1 + .../src/core/spaces/router.ts | 94 ++ .../src/core/spaces/schema.ts | 103 ++ .../contrail-appview/src/core/spaces/tid.ts | 1 + .../contrail-appview/src/core/spaces/types.ts | 1 + .../contrail-appview/src/core/spaces/uri.ts | 1 + packages/contrail-appview/src/core/types.ts | 1 + packages/contrail-appview/src/index.ts | 65 + packages/contrail-appview/tsconfig.build.json | 7 + packages/contrail-appview/tsconfig.json | 7 + packages/contrail-appview/tsup.config.ts | 15 + packages/contrail-authority/src/index.ts | 8 +- .../contrail-authority/src/invite-routes.ts | 240 ++++ packages/contrail-authority/src/routes.ts | 412 ++++++ packages/contrail-record-host/package.json | 43 + packages/contrail-record-host/src/adapter.ts | 530 +++++++ packages/contrail-record-host/src/blob-gc.ts | 37 + .../contrail-record-host/src/blob-refs.ts | 27 + packages/contrail-record-host/src/index.ts | 27 + packages/contrail-record-host/src/routes.ts | 510 +++++++ packages/contrail-record-host/src/schema.ts | 51 + .../contrail-record-host/tsconfig.build.json | 7 + packages/contrail-record-host/tsconfig.json | 7 + packages/contrail-record-host/tsup.config.ts | 11 + packages/contrail/package.json | 2 + packages/contrail/src/core/backfill.ts | 524 +------ packages/contrail/src/core/client.ts | 2 +- .../src/core/community-integration.ts | 2 +- packages/contrail/src/core/db/index.ts | 5 +- packages/contrail/src/core/db/records.ts | 897 +----------- packages/contrail/src/core/db/schema.ts | 356 +---- packages/contrail/src/core/dialect.ts | 2 +- packages/contrail/src/core/identity.ts | 2 +- .../src/core/invite/community-handler.ts | 2 +- packages/contrail/src/core/invite/index.ts | 4 +- packages/contrail/src/core/invite/router.ts | 251 +--- packages/contrail/src/core/invite/token.ts | 2 +- packages/contrail/src/core/jetstream.ts | 281 +--- packages/contrail/src/core/labels/apply.ts | 65 +- packages/contrail/src/core/labels/hydrate.ts | Bin 3547 -> 44 bytes packages/contrail/src/core/labels/resolve.ts | 135 +- packages/contrail/src/core/labels/schema.ts | 31 +- packages/contrail/src/core/labels/select.ts | 65 +- .../contrail/src/core/labels/subscribe.ts | 316 +---- packages/contrail/src/core/labels/types.ts | 2 +- packages/contrail/src/core/persistent.ts | 244 +--- .../src/core/realtime/durable-object.ts | 2 +- .../contrail/src/core/realtime/in-memory.ts | 2 +- packages/contrail/src/core/realtime/index.ts | 38 +- packages/contrail/src/core/realtime/merge.ts | 2 +- .../src/core/realtime/publishing-adapter.ts | 158 +-- .../src/core/realtime/query-filter.ts | 2 +- .../contrail/src/core/realtime/resolve.ts | 93 +- packages/contrail/src/core/realtime/router.ts | 256 +--- packages/contrail/src/core/realtime/sse.ts | 2 +- packages/contrail/src/core/realtime/ticket.ts | 2 +- packages/contrail/src/core/realtime/types.ts | 2 +- .../contrail/src/core/realtime/websocket.ts | 2 +- packages/contrail/src/core/refresh.ts | 268 +--- packages/contrail/src/core/router/admin.ts | 45 +- .../contrail/src/core/router/collection.ts | 1226 +---------------- packages/contrail/src/core/router/feed.ts | 135 +- packages/contrail/src/core/router/helpers.ts | 68 +- packages/contrail/src/core/router/hydrate.ts | 224 +-- packages/contrail/src/core/router/index.ts | 248 +--- packages/contrail/src/core/router/notify.ts | 180 +-- packages/contrail/src/core/router/profiles.ts | 182 +-- packages/contrail/src/core/search.ts | 32 +- packages/contrail/src/core/spaces/acl.ts | 2 +- packages/contrail/src/core/spaces/adapter.ts | 506 +------ packages/contrail/src/core/spaces/auth.ts | 2 +- packages/contrail/src/core/spaces/binding.ts | 2 +- .../contrail/src/core/spaces/blob-adapter.ts | 2 +- packages/contrail/src/core/spaces/blob-gc.ts | 39 +- .../contrail/src/core/spaces/blob-refs.ts | 28 +- .../contrail/src/core/spaces/credentials.ts | 2 +- .../contrail/src/core/spaces/in-process.ts | 2 +- packages/contrail/src/core/spaces/router.ts | 1028 +------------- packages/contrail/src/core/spaces/schema.ts | 104 +- packages/contrail/src/core/spaces/tid.ts | 2 +- packages/contrail/src/core/spaces/types.ts | 2 +- packages/contrail/src/core/spaces/uri.ts | 2 +- packages/contrail/src/core/types.ts | 2 +- packages/contrail/tests/persistent.test.ts | 12 +- packages/contrail/tests/refresh.test.ts | 36 +- packages/contrail/vitest.config.ts | 2 + pnpm-lock.yaml | 77 ++ 139 files changed, 9068 insertions(+), 8046 deletions(-) create mode 100644 packages/contrail-appview/package.json create mode 100644 packages/contrail-appview/src/core/backfill.ts create mode 100644 packages/contrail-appview/src/core/client.ts create mode 100644 packages/contrail-appview/src/core/community-integration.ts create mode 100644 packages/contrail-appview/src/core/db/index.ts create mode 100644 packages/contrail-appview/src/core/db/records.ts create mode 100644 packages/contrail-appview/src/core/db/schema.ts create mode 100644 packages/contrail-appview/src/core/dialect.ts create mode 100644 packages/contrail-appview/src/core/identity.ts create mode 100644 packages/contrail-appview/src/core/invite/community-handler.ts create mode 100644 packages/contrail-appview/src/core/invite/index.ts create mode 100644 packages/contrail-appview/src/core/invite/router.ts create mode 100644 packages/contrail-appview/src/core/invite/token.ts create mode 100644 packages/contrail-appview/src/core/jetstream.ts create mode 100644 packages/contrail-appview/src/core/labels/apply.ts create mode 100644 packages/contrail-appview/src/core/labels/hydrate.ts create mode 100644 packages/contrail-appview/src/core/labels/resolve.ts create mode 100644 packages/contrail-appview/src/core/labels/schema.ts create mode 100644 packages/contrail-appview/src/core/labels/select.ts create mode 100644 packages/contrail-appview/src/core/labels/subscribe.ts create mode 100644 packages/contrail-appview/src/core/labels/types.ts create mode 100644 packages/contrail-appview/src/core/persistent.ts create mode 100644 packages/contrail-appview/src/core/realtime/durable-object.ts create mode 100644 packages/contrail-appview/src/core/realtime/in-memory.ts create mode 100644 packages/contrail-appview/src/core/realtime/index.ts create mode 100644 packages/contrail-appview/src/core/realtime/merge.ts create mode 100644 packages/contrail-appview/src/core/realtime/publishing-adapter.ts create mode 100644 packages/contrail-appview/src/core/realtime/query-filter.ts create mode 100644 packages/contrail-appview/src/core/realtime/resolve.ts create mode 100644 packages/contrail-appview/src/core/realtime/router.ts create mode 100644 packages/contrail-appview/src/core/realtime/sse.ts create mode 100644 packages/contrail-appview/src/core/realtime/ticket.ts create mode 100644 packages/contrail-appview/src/core/realtime/types.ts create mode 100644 packages/contrail-appview/src/core/realtime/websocket.ts create mode 100644 packages/contrail-appview/src/core/refresh.ts create mode 100644 packages/contrail-appview/src/core/router/admin.ts create mode 100644 packages/contrail-appview/src/core/router/collection.ts create mode 100644 packages/contrail-appview/src/core/router/feed.ts create mode 100644 packages/contrail-appview/src/core/router/helpers.ts create mode 100644 packages/contrail-appview/src/core/router/hydrate.ts create mode 100644 packages/contrail-appview/src/core/router/index.ts create mode 100644 packages/contrail-appview/src/core/router/notify.ts create mode 100644 packages/contrail-appview/src/core/router/profiles.ts create mode 100644 packages/contrail-appview/src/core/search.ts create mode 100644 packages/contrail-appview/src/core/spaces/acl.ts create mode 100644 packages/contrail-appview/src/core/spaces/adapter.ts create mode 100644 packages/contrail-appview/src/core/spaces/auth.ts create mode 100644 packages/contrail-appview/src/core/spaces/binding.ts create mode 100644 packages/contrail-appview/src/core/spaces/blob-adapter.ts create mode 100644 packages/contrail-appview/src/core/spaces/blob-gc.ts create mode 100644 packages/contrail-appview/src/core/spaces/blob-refs.ts create mode 100644 packages/contrail-appview/src/core/spaces/credentials.ts create mode 100644 packages/contrail-appview/src/core/spaces/in-process.ts create mode 100644 packages/contrail-appview/src/core/spaces/router.ts create mode 100644 packages/contrail-appview/src/core/spaces/schema.ts create mode 100644 packages/contrail-appview/src/core/spaces/tid.ts create mode 100644 packages/contrail-appview/src/core/spaces/types.ts create mode 100644 packages/contrail-appview/src/core/spaces/uri.ts create mode 100644 packages/contrail-appview/src/core/types.ts create mode 100644 packages/contrail-appview/src/index.ts create mode 100644 packages/contrail-appview/tsconfig.build.json create mode 100644 packages/contrail-appview/tsconfig.json create mode 100644 packages/contrail-appview/tsup.config.ts create mode 100644 packages/contrail-authority/src/invite-routes.ts create mode 100644 packages/contrail-authority/src/routes.ts create mode 100644 packages/contrail-record-host/package.json create mode 100644 packages/contrail-record-host/src/adapter.ts create mode 100644 packages/contrail-record-host/src/blob-gc.ts create mode 100644 packages/contrail-record-host/src/blob-refs.ts create mode 100644 packages/contrail-record-host/src/index.ts create mode 100644 packages/contrail-record-host/src/routes.ts create mode 100644 packages/contrail-record-host/src/schema.ts create mode 100644 packages/contrail-record-host/tsconfig.build.json create mode 100644 packages/contrail-record-host/tsconfig.json create mode 100644 packages/contrail-record-host/tsup.config.ts diff --git a/packages/contrail-appview/package.json b/packages/contrail-appview/package.json new file mode 100644 index 0000000..c76172b --- /dev/null +++ b/packages/contrail-appview/package.json @@ -0,0 +1,56 @@ +{ + "name": "@atmo-dev/contrail-appview", + "version": "0.6.0", + "description": "Public-records appview for contrail — jetstream ingestion, backfill, query layer, feeds, labels, profiles, per-collection XRPC routes.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-appview" + }, + "keywords": [ + "atproto", + "contrail", + "appview", + "jetstream" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/atproto": "^3.1.10", + "@atcute/cbor": "^2.3.2", + "@atcute/cid": "^2.4.1", + "@atcute/client": "^4.2.1", + "@atcute/identity": "^1.1.4", + "@atcute/identity-resolver": "^1.2.2", + "@atcute/jetstream": "^1.0.2", + "@atcute/lexicons": "^1.2.9", + "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-authority": "workspace:*", + "@atmo-dev/contrail-base": "workspace:*", + "@atmo-dev/contrail-record-host": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "@types/node": "^25.5.0", + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "license": "MIT" +} diff --git a/packages/contrail-appview/src/core/backfill.ts b/packages/contrail-appview/src/core/backfill.ts new file mode 100644 index 0000000..713e23e --- /dev/null +++ b/packages/contrail-appview/src/core/backfill.ts @@ -0,0 +1,523 @@ +import type {} from "@atcute/atproto"; +import { type Did } from "@atcute/lexicons"; +import { isDid, isNsid } from "@atcute/lexicons/syntax"; + +import type { Client } from "@atcute/client"; +import type { ContrailConfig, Database, IngestEvent } from "./types"; +import { getDiscoverableNsids, getDependentNsids, DEFAULT_RELAYS } from "./types"; +import { applyEvents, getLastCursor, saveCursor } from "./db"; +import { getClient, getPDS } from "./client"; + +const PAGE_SIZE = 100; +const BATCH_SIZE = 100; +const MAX_RETRIES = 5; + +const REQUEST_TIMEOUT_MS = 10_000; + +async function withRetry<T>( + fn: () => Promise<T>, + label: string, + maxRetries = 3, + timeoutMs = REQUEST_TIMEOUT_MS +): Promise<T> { + let lastError: unknown; + for (let attempt = 0; attempt <= maxRetries; attempt++) { + try { + return await Promise.race([ + fn(), + new Promise<never>((_, reject) => + setTimeout(() => reject(new Error(`Timeout: ${label}`)), timeoutMs) + ), + ]); + } catch (err) { + lastError = err; + if (attempt < maxRetries) { + const delay = Math.min(1000 * 2 ** attempt, 10000); + await new Promise((r) => setTimeout(r, delay)); + } + } + } + throw lastError; +} + +async function markFailed( + db: Database, + did: string, + collection: string, + error: string +): Promise<void> { + await db + .prepare( + "UPDATE backfills SET retries = retries + 1, last_error = ? WHERE did = ? AND collection = ?" + ) + .bind(error, did, collection) + .run(); +} + +export interface BackfillOptions { + /** Pre-resolved client — avoids redundant PDS lookups when batching by DID */ + client?: Client; + /** Skip replay detection in applyEvents (safe during initial backfill) */ + skipReplayDetection?: boolean; + /** Max retries per request (default: 3). Set to 0 for single-attempt mode. */ + maxRetries?: number; + /** Per-request timeout in ms (default: 10000). */ + requestTimeout?: number; +} + +export async function backfillUser( + db: Database, + did: string, + collection: string, + deadline: number, + config?: ContrailConfig, + options?: BackfillOptions +): Promise<number> { + if (Date.now() >= deadline) return 0; + + const status = await db + .prepare( + "SELECT completed, pds_cursor, retries FROM backfills WHERE did = ? AND collection = ?" + ) + .bind(did, collection) + .first<{ completed: number; pds_cursor: string | null; retries: number }>(); + + if (status?.completed) return 0; + + if (!status) { + await db + .prepare( + "INSERT INTO backfills (did, collection, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" + ) + .bind(did, collection) + .run(); + } + + let currentCursor: string | undefined = status?.pds_cursor ?? undefined; + const retries = options?.maxRetries ?? 3; + const timeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; + + if (!isDid(did)) { + await markFailed(db, did, collection, `Invalid DID: ${did}`); + return 0; + } + + if (!isNsid(collection)) { + await markFailed(db, did, collection, `Invalid NSID: ${collection}`); + return 0; + } + + let client = options?.client; + if (!client) { + try { + client = await withRetry( + () => getClient(did as Did, db), + `getClient(${did})`, + Math.min(retries, 1), + timeout + ); + } catch (err) { + await markFailed(db, did, collection, String(err)); + return 0; + } + } + + let totalInserted = 0; + let done = false; + + try { + while (Date.now() < deadline) { + const response = await withRetry( + () => + client!.get("com.atproto.repo.listRecords", { + params: { + repo: did as Did, + collection, + limit: PAGE_SIZE, + cursor: currentCursor, + }, + }), + `listRecords(${did}/${collection})`, + retries, + timeout + ); + if (!response.ok) { + await markFailed( + db, + did, + collection, + `listRecords status ${response.status}` + ); + return totalInserted; + } + + if (response.data.records.length === 0) { + done = true; + break; + } + + const now = Date.now(); + const events: IngestEvent[] = response.data.records.map((r) => ({ + uri: r.uri, + did, + collection, + rkey: r.uri.split("/").pop()!, + operation: "create" as const, + cid: r.cid, + record: JSON.stringify(r.value), + time_us: now * 1000, + indexed_at: now * 1000, + })); + + await applyEvents(db, events, config, { + skipReplayDetection: options?.skipReplayDetection, + skipFeedFanout: true, + }); + totalInserted += events.length; + + currentCursor = response.data.cursor ?? undefined; + + await db + .prepare( + "UPDATE backfills SET pds_cursor = ? WHERE did = ? AND collection = ?" + ) + .bind(currentCursor ?? null, did, collection) + .run(); + + if (!currentCursor) { + done = true; + break; + } + } + } catch (err) { + await markFailed(db, did, collection, String(err)); + return totalInserted; + } + + if (done) { + await db + .prepare( + "UPDATE backfills SET completed = 1 WHERE did = ? AND collection = ?" + ) + .bind(did, collection) + .run(); + } + + return totalInserted; +} + +// --- Bulk backfill (groups by DID, resolves client once) --- + +export interface BackfillProgress { + records: number; + usersComplete: number; + usersTotal: number; + usersFailed: number; +} + +export interface BackfillAllOptions { + concurrency?: number; + onProgress?: (progress: BackfillProgress) => void; +} + +export async function backfillPending( + db: Database, + config: ContrailConfig, + options?: BackfillAllOptions +): Promise<number> { + const concurrency = options?.concurrency ?? 100; + let totalBackfilled = 0; + + // Anchor the jetstream cursor to now if it hasn't been set yet, so records + // emitted during backfill are replayed once jetstream starts. + if ((await getLastCursor(db)) === null) { + await saveCursor(db, Date.now() * 1000); + } + + // Reset retries so users that hit the cap in a prior run get another chance. + await db + .prepare("UPDATE backfills SET retries = 0 WHERE completed = 0") + .run(); + + while (true) { + const pending = await db + .prepare( + "SELECT did, collection FROM backfills WHERE completed = 0 AND retries < ? ORDER BY did" + ) + .bind(MAX_RETRIES) + .all<{ did: string; collection: string }>(); + + const rows = pending.results ?? []; + if (rows.length === 0) break; + + // Group by DID so we resolve PDS once per user + const byDid = new Map<string, string[]>(); + for (const row of rows) { + const cols = byDid.get(row.did) ?? []; + cols.push(row.collection); + byDid.set(row.did, cols); + } + + const dids = [...byDid.keys()]; + + // Resolve PDS endpoints in background (populates in-memory cache) + const resolvePromise = (async () => { + for (let i = 0; i < dids.length; i += 200) { + await Promise.allSettled( + dids.slice(i, i + 200).map((did) => + getPDS(did as Did, db).catch(() => {}) + ) + ); + } + })(); + + let roundBackfilled = 0; + let usersComplete = 0; + let usersFailed = 0; + const failedDids: string[] = []; + + const FAST_TIMEOUT = 3_000; + + const emitProgress = () => + options?.onProgress?.({ + records: totalBackfilled + roundBackfilled, + usersComplete, + usersTotal: dids.length, + usersFailed, + }); + + // Fast pass: single attempt per user with short timeout + for (let i = 0; i < dids.length; i += concurrency) { + const batch = dids.slice(i, i + concurrency); + + const results = await Promise.allSettled( + batch.map(async (did) => { + let client: Client | undefined; + try { + client = await withRetry( + () => getClient(did as Did, db), + `getClient(${did})`, + 0, + FAST_TIMEOUT + ); + } catch { + failedDids.push(did); + return 0; + } + + const cols = byDid.get(did)!; + const counts = await Promise.all( + cols.map((col) => + backfillUser(db, did, col, Infinity, config, { + client, + skipReplayDetection: true, + maxRetries: 0, + requestTimeout: FAST_TIMEOUT, + }).catch(() => { + failedDids.push(did); + return 0; + }) + ) + ); + + usersComplete++; + return counts.reduce((a, b) => a + b, 0); + }) + ); + + for (const r of results) { + if (r.status === "fulfilled") roundBackfilled += r.value; + } + + emitProgress(); + } + + // Retry pass: failed DIDs get retries with backoff, still in concurrent batches + if (failedDids.length > 0) { + const uniqueFailed = [...new Set(failedDids)]; + usersComplete -= uniqueFailed.length; // don't count them yet + + for (let i = 0; i < uniqueFailed.length; i += concurrency) { + const batch = uniqueFailed.slice(i, i + concurrency); + + const results = await Promise.allSettled( + batch.map(async (did) => { + let client: Client | undefined; + try { + client = await withRetry( + () => getClient(did as Did, db), + `getClient(${did})`, + 2 + ); + } catch (err) { + for (const col of byDid.get(did)!) { + await markFailed(db, did, col, String(err)); + } + usersFailed++; + usersComplete++; + return 0; + } + + const cols = byDid.get(did)!; + const counts = await Promise.all( + cols.map((col) => + backfillUser(db, did, col, Infinity, config, { + client, + skipReplayDetection: true, + maxRetries: 2, + }) + ) + ); + usersComplete++; + return counts.reduce((a, b) => a + b, 0); + }) + ); + + for (const r of results) { + if (r.status === "fulfilled") roundBackfilled += r.value; + } + + emitProgress(); + } + } + + await resolvePromise; + totalBackfilled += roundBackfilled; + + // If nothing was backfilled this round, we're stuck + if (roundBackfilled === 0) break; + } + + return totalBackfilled; +} + +// --- Discovery --- + +interface DiscoveryPage { + repos: { did: string }[]; + cursor?: string; +} + +async function fetchPage( + relay: string, + collection: string, + cursor?: string +): Promise<DiscoveryPage | null> { + const url = new URL( + `/xrpc/com.atproto.sync.listReposByCollection`, + relay + ); + url.searchParams.set("collection", collection); + url.searchParams.set("limit", "1000"); + if (cursor) { + url.searchParams.set("cursor", cursor); + } + + try { + return await withRetry( + async () => { + const response = await fetch(url.toString()); + if (!response.ok) { + throw new Error(`HTTP ${response.status}`); + } + return (await response.json()) as DiscoveryPage; + }, + `fetchPage(${relay}, ${collection})` + ); + } catch (err) { + // Discovery page fetch failed after retries — skip this relay + return null; + } +} + +async function insertDiscoveredDIDs( + db: Database, + dids: string[], + collection: string +): Promise<void> { + if (dids.length === 0) return; + + // Use multi-row INSERT to reduce the number of statements + const CHUNK_SIZE = 50; + for (let i = 0; i < dids.length; i += CHUNK_SIZE) { + const chunk = dids.slice(i, i + CHUNK_SIZE); + const placeholders = chunk.map(() => "(?, ?, 0)").join(", "); + const bindings: string[] = []; + for (const did of chunk) { + bindings.push(did, collection); + } + await db + .prepare( + `INSERT INTO backfills (did, collection, completed) VALUES ${placeholders} ON CONFLICT DO NOTHING` + ) + .bind(...bindings) + .run(); + } +} + +async function saveDiscoveryState( + db: Database, + collection: string, + relay: string, + cursor: string | null, + completed: boolean +): Promise<void> { + await db + .prepare( + "INSERT INTO discovery (collection, relay, cursor, completed) VALUES (?, ?, ?, ?) ON CONFLICT(collection, relay) DO UPDATE SET cursor = excluded.cursor, completed = excluded.completed" + ) + .bind(collection, relay, cursor, completed ? 1 : 0) + .run(); +} + +export async function discoverDIDs( + db: Database, + config: ContrailConfig, + deadline: number +): Promise<string[]> { + const collections = getDiscoverableNsids(config); + const relays = config.relays ?? DEFAULT_RELAYS; + if (relays.length === 0 || collections.length === 0) return []; + + const discovered: string[] = []; + + for (const collection of collections) { + if (Date.now() >= deadline) break; + + let data: DiscoveryPage | null = null; + let relay: string | null = null; + + for (const r of relays) { + const row = await db + .prepare( + "SELECT cursor, completed FROM discovery WHERE collection = ? AND relay = ?" + ) + .bind(collection, r) + .first<{ cursor: string | null; completed: number }>(); + + if (row?.completed) continue; + + data = await fetchPage(r, collection, row?.cursor ?? undefined); + if (data) { + relay = r; + break; + } else { + await saveDiscoveryState(db, collection, r, null, true); + } + } + if (!data || !relay) continue; + + const dids = data.repos?.map((r) => r.did) ?? []; + await insertDiscoveredDIDs(db, dids, collection); + discovered.push(...dids); + + for (const depCollection of getDependentNsids(config)) { + await insertDiscoveredDIDs(db, dids, depCollection); + } + + const completed = !data.cursor; + await saveDiscoveryState(db, collection, relay, data.cursor ?? null, completed); + } + + return discovered; +} diff --git a/packages/contrail-appview/src/core/client.ts b/packages/contrail-appview/src/core/client.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/client.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/community-integration.ts b/packages/contrail-appview/src/core/community-integration.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/community-integration.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/db/index.ts b/packages/contrail-appview/src/core/db/index.ts new file mode 100644 index 0000000..2469f31 --- /dev/null +++ b/packages/contrail-appview/src/core/db/index.ts @@ -0,0 +1,4 @@ +export { initSchema } from "./schema"; +export { getLastCursor, saveCursor, applyEvents, lookupExistingRecords, queryRecords, queryAcrossSources, pruneFeedItems } from "./records"; +export type { QueryOptions, SortOption, ExistingRecordInfo } from "./records"; +export type { RecordSource } from "../types"; diff --git a/packages/contrail-appview/src/core/db/records.ts b/packages/contrail-appview/src/core/db/records.ts new file mode 100644 index 0000000..a45ab9b --- /dev/null +++ b/packages/contrail-appview/src/core/db/records.ts @@ -0,0 +1,896 @@ +import type { + ContrailConfig, + ResolvedContrailConfig, + RelationConfig, + Database, + Statement, + IngestEvent, + RecordRow, + RecordSource, +} from "../types"; +import { + getNestedValue, + getRelationField, + countColumnName, + groupedCountColumnName, + getFeedFollowShortNames, + recordsTableName, + spacesRecordsTableName, + shortNameForNsid, + nsidForShortName, +} from "../types"; +import { getSearchableFields, ftsTableName, buildFtsContent } from "../search"; +import { ftsQueryClause, getDialect } from "../dialect"; + +// --- Counts --- + +interface InboundRelation { + /** Short name of the parent collection. */ + parentCollection: string; + relationName: string; + rel: RelationConfig; +} + +/** Find relations that target the given short-named child collection. */ +function getInboundRelations( + config: ContrailConfig, + childShortName: string +): InboundRelation[] { + const results: InboundRelation[] = []; + for (const [colName, colConfig] of Object.entries(config.collections)) { + for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { + if (rel.collection === childShortName) { + results.push({ parentCollection: colName, relationName: relName, rel }); + } + } + } + return results; +} + +/** + * Collect recount targets from a single event into a shared map. + * The map is keyed by `parentCollection:relationName:targetValue` to deduplicate + * across the entire batch — so 50 RSVPs to the same event produce one recount, not 50. + */ +function collectCountTargets( + event: IngestEvent, + config: ContrailConfig, + existingRecordJson: string | null, + targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> +): void { + const childShort = shortNameForNsid(config, event.collection); + if (!childShort) return; + const inbound = getInboundRelations(config, childShort); + if (inbound.length === 0) return; + + const record = event.record ? JSON.parse(event.record) : null; + const existingRecord = existingRecordJson ? JSON.parse(existingRecordJson) : null; + + for (const { parentCollection, relationName, rel } of inbound) { + if (rel.count === false) continue; + + const field = getRelationField(rel); + + const values: string[] = []; + if (record) { + const t = getNestedValue(record, field); + if (t) values.push(t); + } + if (existingRecord) { + const t = getNestedValue(existingRecord, field); + if (t && !values.includes(t)) values.push(t); + } + + for (const targetValue of values) { + const key = `${parentCollection}:${relationName}:${targetValue}`; + if (!targets.has(key)) { + targets.set(key, { parentCollection, relationName, rel, targetValue }); + } + } + } +} + +/** + * Build deduplicated count UPDATE statements from collected targets. + * One UPDATE per unique parent+relation+target, regardless of how many + * events in the batch affected that target. + */ +function buildBatchCountStatements( + db: Database, + config: ContrailConfig, + targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> +): Statement[] { + const statements: Statement[] = []; + + for (const { parentCollection, relationName, rel, targetValue } of targets.values()) { + const field = getRelationField(rel); + const matchColumn = rel.match === "did" ? "did" : "uri"; + const childTable = recordsTableName(rel.collection); + const parentTable = recordsTableName(parentCollection); + + const setClauses: string[] = []; + const setBindings: (string | number)[] = []; + + const countExpr = rel.countDistinct + ? `COUNT(DISTINCT ${rel.countDistinct})` + : "COUNT(*)"; + + // Total count + const totalCol = countColumnName(rel.collection); + setClauses.push( + `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ?)` + ); + setBindings.push(targetValue); + + // Grouped counts — column names are `count_<child-short>_<group-key>`; match + // against the group's full token value in the record. + if (rel.groupBy) { + const mapping = (config as ResolvedContrailConfig)._resolved?.relations[parentCollection]?.[relationName]; + if (mapping?.groups) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + setClauses.push( + `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ? AND ${getDialect(db).jsonExtract('record', rel.groupBy)} = ?)` + ); + setBindings.push(targetValue, fullToken); + } + } + } + + if (setClauses.length > 0) { + statements.push( + db + .prepare( + `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE ${matchColumn} = ?` + ) + .bind(...setBindings, targetValue) + ); + } + } + + return statements; +} + +// --- FTS --- + +function buildFtsStatements( + db: Database, + event: IngestEvent, + config: ContrailConfig, + existingMap: Map<string, ExistingRecordInfo> +): Statement[] { + // PostgreSQL: tsvector generated column is auto-maintained, no manual FTS sync + if (getDialect(db).ftsStrategy === "generated-column") return []; + + const short = shortNameForNsid(config, event.collection); + if (!short) return []; + const colConfig = config.collections[short]; + if (!colConfig) return []; + + const fields = getSearchableFields(short, colConfig); + if (!fields || fields.length === 0) return []; + + const table = ftsTableName(short); + const stmts: Statement[] = []; + + if (event.operation === "delete") { + stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); + } else { + const record = event.record ? JSON.parse(event.record) : null; + if (!record) return []; + + const content = buildFtsContent(record, fields); + if (!content) return []; + + // Only delete existing FTS row if this is an update (record already existed) + if (existingMap.has(event.uri)) { + stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); + } + stmts.push( + db.prepare(`INSERT INTO ${table} (uri, content) VALUES (?, ?)`).bind(event.uri, content) + ); + } + + return stmts; +} + +// --- Feeds --- + +function buildFeedStatements( + db: Database, + event: IngestEvent, + config: ContrailConfig, + existingRecords: Map<string, string | null> +): Statement[] { + if (!config.feeds) return []; + + const stmts: Statement[] = []; + + const eventShort = shortNameForNsid(config, event.collection); + if (!eventShort) return []; + + for (const [, feedConfig] of Object.entries(config.feeds)) { + const followTable = recordsTableName(feedConfig.follow); + + // Target collection: fan out to followers + if (feedConfig.targets.includes(eventShort)) { + if (event.operation === "create" || event.operation === "update") { + stmts.push( + db + .prepare( + getDialect(db).insertOrIgnore( + `INSERT INTO feed_items (actor, uri, collection, time_us) + SELECT r.did, ?, ?, ? + FROM ${followTable} r + WHERE ${getDialect(db).jsonExtract('r.record', 'subject')} = ?` + ) + ) + .bind(event.uri, event.collection, event.time_us, event.did) + ); + } else if (event.operation === "delete") { + stmts.push( + db.prepare("DELETE FROM feed_items WHERE uri = ?").bind(event.uri) + ); + } + } + + // Follow collection: handle follow/unfollow + if (eventShort === feedConfig.follow) { + if (event.operation === "create") { + const record = event.record ? JSON.parse(event.record) : null; + const subject = record?.subject; + if (subject) { + for (const targetShort of feedConfig.targets) { + const targetTable = recordsTableName(targetShort); + const targetNsid = nsidForShortName(config, targetShort) ?? targetShort; + stmts.push( + db + .prepare( + getDialect(db).insertOrIgnore( + `INSERT INTO feed_items (actor, uri, collection, time_us) + SELECT ?, r.uri, ?, r.time_us + FROM ${targetTable} r + WHERE r.did = ? + ORDER BY r.time_us DESC + LIMIT 100` + ) + ) + .bind(event.did, targetNsid, subject) + ); + } + } + } else if (event.operation === "delete") { + const existingRecord = existingRecords.get(event.uri); + if (existingRecord) { + const parsed = JSON.parse(existingRecord); + const subject = parsed?.subject; + if (subject) { + for (const targetShort of feedConfig.targets) { + const targetTable = recordsTableName(targetShort); + stmts.push( + db + .prepare( + `DELETE FROM feed_items WHERE actor = ? AND uri IN ( + SELECT uri FROM ${targetTable} WHERE did = ? + )` + ) + .bind(event.did, subject) + ); + } + } + } + } + } + } + + return stmts; +} + +// --- Feed pruning --- + +export async function pruneFeedItems( + db: Database, + maxItems: number +): Promise<number> { + const result = await db + .prepare( + `DELETE FROM feed_items WHERE (actor, uri) NOT IN ( + SELECT actor, uri FROM ( + SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn + FROM feed_items + ) sub WHERE rn <= ? + )` + ) + .bind(maxItems) + .run(); + return (result as any)?.changes ?? 0; +} + +// --- Cursor --- + +export async function getLastCursor(db: Database): Promise<number | null> { + const row = await db + .prepare("SELECT time_us FROM cursor WHERE id = 1") + .first<{ time_us: number }>(); + return row ? row.time_us : null; +} + +export async function saveCursor( + db: Database, + timeUs: number +): Promise<void> { + await db + .prepare( + "INSERT INTO cursor (id, time_us) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET time_us = excluded.time_us" + ) + .bind(timeUs) + .run(); +} + +// --- Existing record lookup --- + +export interface ExistingRecordInfo { + cid: string | null; + record: string | null; + /** When the row was last written to our DB (microseconds). Populated + * whenever `lookupExistingRecords` runs, regardless of `includeRecord`. */ + indexed_at: number | null; +} + +/** + * Look up existing records for a set of events, grouped by collection. + * Returns a map of uri → { cid, record }. + * When includeRecord is false, record will always be null (saves reading large blobs). + */ +export async function lookupExistingRecords( + db: Database, + events: { uri: string; collection: string }[], + includeRecord: boolean = true, + config?: ContrailConfig +): Promise<Map<string, ExistingRecordInfo>> { + const result = new Map<string, ExistingRecordInfo>(); + if (events.length === 0) return result; + + // Group by short name (config lookup); skip events for collections not in our config. + const byShort = new Map<string, string[]>(); + for (const e of events) { + const short = config ? shortNameForNsid(config, e.collection) : e.collection; + if (!short) continue; + const uris = byShort.get(short) ?? []; + uris.push(e.uri); + byShort.set(short, uris); + } + + const selectCols = includeRecord ? "uri, cid, record, indexed_at" : "uri, cid, indexed_at"; + for (const [short, uris] of byShort) { + const table = recordsTableName(short); + for (let i = 0; i < uris.length; i += 50) { + const chunk = uris.slice(i, i + 50); + const placeholders = chunk.map(() => "?").join(","); + const rows = await db + .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri IN (${placeholders})`) + .bind(...chunk) + .all<{ + uri: string; + cid: string | null; + record?: string | null; + indexed_at: number | null; + }>(); + for (const row of rows.results ?? []) { + result.set(row.uri, { + cid: row.cid, + record: includeRecord ? (row.record ?? null) : null, + indexed_at: row.indexed_at ?? null, + }); + } + } + } + + return result; +} + +// --- Events --- + +export async function applyEvents( + db: Database, + events: IngestEvent[], + config?: ContrailConfig, + options?: { + skipReplayDetection?: boolean; + skipFeedFanout?: boolean; + /** Pre-fetched existing records — skips the internal lookup when provided */ + existing?: Map<string, ExistingRecordInfo>; + /** When provided, publish `collection:<nsid>` and `actor:<did>` realtime + * events for each applied event. Space-scoped publishing happens elsewhere + * (see `realtime/publishing-adapter.ts`); public topics carry public + * records only, which is exactly the scope of this function. */ + pubsub?: import("../realtime/types").PubSub; + } +): Promise<void> { + if (events.length === 0) return; + + const followCollections = config ? getFeedFollowShortNames(config) : []; + const hasCountingRelations = config ? Object.values(config.collections).some(c => + Object.values(c.relations ?? {}).some(r => r.count !== false) + ) : false; + const needRecordContent = followCollections.length > 0 || hasCountingRelations; + + // Use pre-fetched data or look up existing records + let existingMap: Map<string, ExistingRecordInfo>; + if (options?.existing) { + existingMap = options.existing; + } else if (config && !options?.skipReplayDetection) { + existingMap = await lookupExistingRecords(db, events, needRecordContent, config); + } else { + existingMap = new Map(); + } + + const batch: Statement[] = []; + + // Build a record-content map for feed statements (needs string values) + const existingRecordStrings = new Map<string, string | null>(); + for (const [uri, info] of existingMap) { + existingRecordStrings.set(uri, info.record); + } + + // Collect all count recount targets across the batch, deduplicated + const countTargets = new Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }>(); + + for (const e of events) { + // Event's collection is an NSID. Look up the short name from config. + // If no config or not found, treat collection string as-is (for tests that pre-populate tables). + const short = config + ? shortNameForNsid(config, e.collection) ?? (config.collections[e.collection] ? e.collection : null) + : e.collection; + if (!short) { + (config?.logger ?? console).warn( + `[ingest] drop (unknown collection in applyEvents): ${e.operation} ${e.uri} collection=${e.collection}` + ); + continue; + } + const table = recordsTableName(short); + + if (e.operation === "delete") { + batch.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(e.uri)); + } else { + batch.push( + db.prepare( + `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, time_us = excluded.time_us, indexed_at = excluded.indexed_at` + ).bind( + e.uri, + e.did, + e.rkey, + e.cid, + e.record, + e.time_us, + e.indexed_at + ) + ); + } + + if (config) { + // Collect count targets (deduplicated across the whole batch) + const existingRecordJson = existingMap.get(e.uri)?.record ?? null; + collectCountTargets(e, config, existingRecordJson, countTargets); + + // Feed fanout still needs replay detection + const existingInfo = existingMap.get(e.uri); + const isReplay = + e.operation === "delete" + ? existingInfo === undefined + : existingInfo?.cid === e.cid; + + if (!isReplay && !options?.skipFeedFanout) { + batch.push(...buildFeedStatements(db, e, config, existingRecordStrings)); + } + batch.push(...buildFtsStatements(db, e, config, existingMap)); + } + } + + // Build deduplicated count statements — one UPDATE per unique target + if (config) { + batch.push(...buildBatchCountStatements(db, config, countTargets)); + } + + await db.batch(batch); + + // Publish realtime events for public records (collection: and actor:). + // Space records publish via the wrapping adapter; this path is public-only. + if (options?.pubsub) { + const pubsub = options.pubsub; + const ts = Date.now(); + for (const e of events) { + if (e.operation === "delete") { + const payload = { + uri: e.uri, + did: e.did, + collection: e.collection, + rkey: e.rkey, + }; + await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.deleted", payload, ts }); + await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.deleted", payload, ts }); + } else { + const record = e.record ? safeParseJson(e.record) : {}; + const payload = { + uri: e.uri, + did: e.did, + collection: e.collection, + rkey: e.rkey, + cid: e.cid, + record, + time_us: e.time_us, + }; + await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.created", payload, ts }); + await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.created", payload, ts }); + } + } + } +} + +function safeParseJson(s: string): Record<string, unknown> { + try { + const v = JSON.parse(s); + return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {}; + } catch { + return {}; + } +} + +// --- Count columns --- + +/** Count column descriptor. `type` is the identifier returned in API responses and + * accepted in countFilters — we keep the full record token for grouped counts so + * callers pass e.g. "community.lexicon.calendar.rsvp#going" and filter/hydrate by it. */ +function getCountColumns( + config: ContrailConfig, + shortName: string +): { type: string; column: string }[] { + const colConfig = config.collections[shortName]; + if (!colConfig?.relations) return []; + const columns: { type: string; column: string }[] = []; + const relMap = (config as ResolvedContrailConfig)._resolved?.relations[shortName] ?? {}; + + for (const [relName, rel] of Object.entries(colConfig.relations)) { + if (rel.count === false) continue; + // Total: identifier is the child's short name; column is `count_<child-short>`. + columns.push({ type: rel.collection, column: countColumnName(rel.collection) }); + const mapping = relMap[relName]; + if (mapping) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + // Grouped: identifier is the full record token (stable across deployments); + // column is `count_<child-short>_<group-key>`. + columns.push({ + type: fullToken, + column: groupedCountColumnName(rel.collection, groupKey), + }); + } + } + } + return columns; +} + +/** For a given "count type" (short name or full group token), return the DB column. */ +function countColumnForType( + config: ContrailConfig, + shortName: string, + type: string +): string | null { + for (const col of getCountColumns(config, shortName)) { + if (col.type === type) return col.column; + } + return null; +} + +// --- Query --- + +export interface SortOption { + recordField?: string; + countType?: string; + direction: "asc" | "desc"; +} + +/** Opaque keyset cursor. `t` is the tiebreaker (time_us of the last row), + * `v` is the sort-key value (string for record fields, number for counts), + * `k` identifies the sort so we can reject mismatched cursors. */ +interface CursorPayload { + t: number; + v?: string | number; + k: "time" | string; // "time" | `field:<name>` | `count:<type>` +} + +function sortKind(sort?: SortOption): "time" | string { + if (sort?.recordField) return `field:${sort.recordField}`; + if (sort?.countType) return `count:${sort.countType}`; + return "time"; +} + +function encodeCursor(payload: CursorPayload): string { + return Buffer.from(JSON.stringify(payload), "utf8").toString("base64url"); +} + +function decodeCursor(cursor: string): CursorPayload | null { + try { + const json = Buffer.from(cursor, "base64url").toString("utf8"); + const p = JSON.parse(json); + if (typeof p?.t !== "number" || typeof p?.k !== "string") return null; + return p as CursorPayload; + } catch { + return null; + } +} + +export interface QueryOptions { + collection: string; + did?: string; + limit?: number; + cursor?: string; + filters?: Record<string, string>; + rangeFilters?: Record<string, { min?: string; max?: string }>; + countFilters?: Record<string, number>; + sort?: SortOption; + search?: string; + source?: RecordSource; + /** When set, query the per-space table (`spaces_records_<short>`) instead of the + * public `records_<short>` table, scoped to rows where `space_uri = ?`. */ + spaceUri?: string; +} + +export async function queryRecords( + db: Database, + config: ContrailConfig, + options: QueryOptions +): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { + const { + collection: collectionInput, + did, + limit: rawLimit, + cursor, + filters = {}, + rangeFilters = {}, + countFilters = {}, + sort, + search, + source, + spaceUri, + } = options; + + // Accept either the short name (canonical) or the full NSID for convenience. + const collection = + config.collections[collectionInput] + ? collectionInput + : shortNameForNsid(config, collectionInput) ?? collectionInput; + + const table = spaceUri ? spacesRecordsTableName(collection) : recordsTableName(collection); + const limit = Math.min(Math.max(1, rawLimit ?? 50), 200); + const conditions: string[] = []; + const bindings: (string | number)[] = []; + + if (spaceUri) { + conditions.push("r.space_uri = ?"); + bindings.push(spaceUri); + } + + if (source?.conditions) conditions.push(...source.conditions); + if (source?.params) bindings.push(...source.params); + + const countCols = getCountColumns(config, collection); + + if (did) { + conditions.push("r.did = ?"); + bindings.push(did); + } + + // Opaque keyset cursor encoding { t, v?, k }. Silently ignored if it doesn't + // match the current sort — callers shouldn't mix sort params with stale cursors. + const expectedKind = sortKind(sort); + if (cursor) { + const payload = decodeCursor(cursor); + if (payload && payload.k === expectedKind) { + if (sort?.recordField) { + const sortExpr = getDialect(db).jsonExtract('r.record', sort.recordField); + const cmp = sort.direction === "desc" ? "<" : ">"; + conditions.push(`(${sortExpr} ${cmp} ? OR (${sortExpr} = ? AND r.time_us < ?))`); + const v = payload.v ?? ""; + bindings.push(v as string | number, v as string | number, payload.t); + } else if (sort?.countType) { + const sortCol = countColumnForType(config, collection, sort.countType); + if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); + const cmp = sort.direction === "desc" ? "<" : ">"; + conditions.push(`(r.${sortCol} ${cmp} ? OR (r.${sortCol} = ? AND r.time_us < ?))`); + const v = Number(payload.v ?? 0); + bindings.push(v, v, payload.t); + } else { + conditions.push("r.time_us < ?"); + bindings.push(payload.t); + } + } + } + + for (const [field, value] of Object.entries(filters)) { + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} = ?`); + bindings.push(value); + } + + for (const [field, range] of Object.entries(rangeFilters)) { + if (range.min != null) { + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} >= ?`); + bindings.push(range.min); + } + if (range.max != null) { + conditions.push(`${getDialect(db).jsonExtract('r.record', field)} <= ?`); + bindings.push(range.max); + } + } + + for (const [type, minCount] of Object.entries(countFilters)) { + const col = countColumnForType(config, collection, type); + if (!col) continue; // unknown count type — skip filter + conditions.push(`r.${col} >= ?`); + bindings.push(minCount); + } + + // FTS search. Not supported in space mode yet (would need composite keying + // because the same at-URI can appear in multiple spaces). + let ftsJoin = ""; + let ftsClause: ReturnType<typeof ftsQueryClause> | null = null; + if (search && !spaceUri) { + const colConfig2 = config.collections[collection]; + const fields = colConfig2 ? getSearchableFields(collection, colConfig2) : null; + if (fields && fields.length > 0) { + ftsClause = ftsQueryClause(getDialect(db), recordsTableName(collection)); + ftsJoin = ftsClause.join; + conditions.push(ftsClause.condition); + // SECURITY: `search` is user input bound as a parameter, not interpolated. + bindings.push(search); + } + } + + const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; + + const countSelect = countCols.length > 0 + ? ", " + countCols.map(({ column }) => `r.${column}`).join(", ") + : ""; + const select = `r.uri, r.did, r.rkey, r.cid, r.record, r.time_us, r.indexed_at${countSelect}`; + + const join = [source?.joins, ftsJoin].filter(Boolean).join(" "); + + let orderBy: string; + if (sort?.recordField) { + const dir = sort.direction === "desc" ? "DESC" : "ASC"; + orderBy = `${getDialect(db).jsonExtract('r.record', sort.recordField)} ${dir}, r.time_us DESC`; + } else if (sort?.countType) { + const dir = sort.direction === "desc" ? "DESC" : "ASC"; + const sortCol = countColumnForType(config, collection, sort.countType); + if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); + orderBy = `r.${sortCol} ${dir}, r.time_us DESC`; + } else if (ftsClause) { + orderBy = `${ftsClause.orderExpr}, r.time_us DESC`; + // PG ts_rank needs the search term bound again for ORDER BY + if (getDialect(db).ftsStrategy === "generated-column" && search) { + bindings.push(search); + } + } else { + orderBy = "r.time_us DESC"; + } + + bindings.push(limit); + + const query = `SELECT ${select} FROM ${table} r ${join} ${where} ORDER BY ${orderBy} LIMIT ?`; + + const result = await db + .prepare(query) + .bind(...bindings) + .all<any>(); + + const nsid = nsidForShortName(config, collection) ?? collection; + const records = (result.results ?? []).map((row: any) => { + const rec: RecordRow & { counts?: Record<string, number> } = { + uri: row.uri, + did: row.did, + collection: nsid, + rkey: row.rkey, + cid: row.cid, + record: row.record, + time_us: row.time_us, + indexed_at: row.indexed_at, + ...(spaceUri ? { space: spaceUri } : {}), + }; + if (countCols.length > 0) { + const counts: Record<string, number> = {}; + for (const { type, column } of countCols) { + const val = row[column]; + if (val != null && val !== 0) counts[type] = val; + } + if (Object.keys(counts).length > 0) rec.counts = counts; + } + return rec; + }); + + const nextCursor = + records.length === limit + ? buildCursor(records[records.length - 1], sort, expectedKind) + : undefined; + + return { records, cursor: nextCursor }; +} + +/** Build an opaque keyset cursor from the last row of a page. */ +function buildCursor( + row: RecordRow & { counts?: Record<string, number> }, + sort: SortOption | undefined, + kind: string +): string { + const t = Number(row.time_us); + if (sort?.recordField) { + const parsed = row.record ? JSON.parse(row.record) : null; + const v = parsed ? getNestedValue(parsed, sort.recordField) : undefined; + return encodeCursor({ t, v: v == null ? "" : String(v), k: kind }); + } + if (sort?.countType) { + const v = row.counts?.[sort.countType] ?? 0; + return encodeCursor({ t, v, k: kind }); + } + return encodeCursor({ t, k: kind }); +} + +/** Compare two rows according to the active sort order. Returns negative if + * `a` should come before `b`, positive otherwise. Matches the SQL ORDER BY. */ +function compareRows( + a: RecordRow & { counts?: Record<string, number> }, + b: RecordRow & { counts?: Record<string, number> }, + sort: SortOption | undefined +): number { + const timeCmp = Number(b.time_us) - Number(a.time_us); // time_us DESC + if (sort?.recordField) { + const ar = a.record ? JSON.parse(a.record) : null; + const br = b.record ? JSON.parse(b.record) : null; + const av = ar ? getNestedValue(ar, sort.recordField) : undefined; + const bv = br ? getNestedValue(br, sort.recordField) : undefined; + const dir = sort.direction === "desc" ? -1 : 1; + const cmp = (av === bv ? 0 : (av! < bv! ? -1 : 1)) * dir; + return cmp !== 0 ? cmp : timeCmp; + } + if (sort?.countType) { + const av = a.counts?.[sort.countType] ?? 0; + const bv = b.counts?.[sort.countType] ?? 0; + const dir = sort.direction === "desc" ? -1 : 1; + const cmp = (av === bv ? 0 : (av < bv ? -1 : 1)) * dir; + return cmp !== 0 ? cmp : timeCmp; + } + return timeCmp; +} + +/** Run a listRecords query across the public table and a set of per-space tables + * in parallel, then merge according to the active sort order. The cursor is a + * shared keyset cursor — every sub-query applies the same `WHERE` keyset, so + * pagination is consistent across sources. */ +export async function queryAcrossSources( + db: Database, + config: ContrailConfig, + options: QueryOptions, + spaceUris: string[] +): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { + if (spaceUris.length === 0) { + return queryRecords(db, config, options); + } + const limit = Math.min(Math.max(1, options.limit ?? 50), 200); + const perSourceLimit = limit; // each source fetches up to `limit`; we trim after merge + + const tasks: Promise<{ records: (RecordRow & { counts?: Record<string, number> })[] }>[] = [ + queryRecords(db, config, { ...options, limit: perSourceLimit }), + ]; + for (const spaceUri of spaceUris) { + tasks.push(queryRecords(db, config, { ...options, spaceUri, limit: perSourceLimit })); + } + const results = await Promise.all(tasks); + const merged = results.flatMap((r) => r.records); + merged.sort((a, b) => compareRows(a, b, options.sort)); + const trimmed = merged.slice(0, limit); + const kind = sortKind(options.sort); + const cursor = + trimmed.length === limit ? buildCursor(trimmed[trimmed.length - 1], options.sort, kind) : undefined; + return { records: trimmed, cursor }; +} + +// --- Users --- + diff --git a/packages/contrail-appview/src/core/db/schema.ts b/packages/contrail-appview/src/core/db/schema.ts new file mode 100644 index 0000000..562a6ee --- /dev/null +++ b/packages/contrail-appview/src/core/db/schema.ts @@ -0,0 +1,355 @@ +import type { ContrailConfig, Database, ResolvedContrailConfig, ResolvedMaps } from "../types"; +import type { SqlDialect } from "../dialect"; +import { buildFtsSchema, getDialect } from "../dialect"; +import { + getRelationField, + countColumnName, + groupedCountColumnName, + recordsTableName, + spacesRecordsTableName, + resolveConfig, +} from "../types"; +import { getSearchableFields } from "../search"; +import { buildSpacesBaseSchema } from "../spaces/schema"; +import { buildLabelsSchema } from "../labels/schema"; + +function getResolved(config: ContrailConfig): ResolvedMaps { + return (config as ResolvedContrailConfig)._resolved ?? resolveConfig(config)._resolved; +} + +function buildBaseSchema(dialect: SqlDialect): string { + return ` +CREATE TABLE IF NOT EXISTS backfills ( + did TEXT NOT NULL, + collection TEXT NOT NULL, + completed INTEGER NOT NULL DEFAULT 0, + pds_cursor TEXT, + retries INTEGER NOT NULL DEFAULT 0, + last_error TEXT, + PRIMARY KEY (did, collection) +); +CREATE TABLE IF NOT EXISTS discovery ( + collection TEXT NOT NULL, + relay TEXT NOT NULL, + cursor TEXT, + completed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (collection, relay) +); +CREATE TABLE IF NOT EXISTS cursor ( + id INTEGER PRIMARY KEY CHECK (id = 1), + time_us ${dialect.bigintType} NOT NULL +); +CREATE TABLE IF NOT EXISTS identities ( + did TEXT PRIMARY KEY, + handle TEXT, + pds TEXT, + resolved_at ${dialect.bigintType} NOT NULL +); +CREATE INDEX IF NOT EXISTS idx_identities_handle ON identities(handle); +`; +} + +function sanitizeName(name: string): string { + return name.replace(/[^a-zA-Z0-9]/g, "_"); +} + +interface BuilderOpts { + /** Emit tables for the spaces variant (spaces_records_<short> with space_uri column). */ + forSpaces?: boolean; +} + +function tableFor(shortName: string, opts: BuilderOpts): string { + return opts.forSpaces ? spacesRecordsTableName(shortName) : recordsTableName(shortName); +} + +function namePrefix(opts: BuilderOpts): string { + return opts.forSpaces ? "sp_" : ""; +} + +export function buildCollectionTables( + config: ContrailConfig, + dialect: SqlDialect, + opts: BuilderOpts = {} +): string[] { + const stmts: string[] = []; + for (const [shortName, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const table = tableFor(shortName, opts); + const np = namePrefix(opts); + if (opts.forSpaces) { + stmts.push( + `CREATE TABLE IF NOT EXISTS ${table} ( + space_uri TEXT NOT NULL, + uri TEXT NOT NULL, + did TEXT NOT NULL, + rkey TEXT NOT NULL, + cid TEXT, + record ${dialect.recordColumnType}, + time_us ${dialect.bigintType} NOT NULL, + indexed_at ${dialect.bigintType} NOT NULL, + PRIMARY KEY (space_uri, did, rkey) + )` + ); + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_time ON ${table}(space_uri, time_us DESC)` + ); + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_did ON ${table}(space_uri, did)` + ); + } else { + stmts.push( + `CREATE TABLE IF NOT EXISTS ${table} ( + uri TEXT PRIMARY KEY, + did TEXT NOT NULL, + rkey TEXT NOT NULL, + cid TEXT, + record ${dialect.recordColumnType}, + time_us ${dialect.bigintType} NOT NULL, + indexed_at ${dialect.bigintType} NOT NULL + )` + ); + stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_did ON ${table}(did)`); + stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_time ON ${table}(time_us DESC)`); + } + } + return stmts; +} + +export function buildDynamicIndexes( + config: ContrailConfig, + dialect: SqlDialect, + opts: BuilderOpts = {} +): string[] { + const resolved = getResolved(config); + const indexes: string[] = []; + const np = namePrefix(opts); + for (const [collection, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const table = tableFor(collection, opts); + const queryable = resolved.queryable[collection] ?? colConfig.queryable ?? {}; + for (const field of Object.keys(queryable)) { + const idxName = `idx_${np}${sanitizeName(collection)}_${sanitizeName(field)}`; + indexes.push( + `CREATE INDEX IF NOT EXISTS ${idxName} ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', field))})` + ); + } + + for (const [, rel] of Object.entries(colConfig.relations ?? {})) { + const childShort = rel.collection; + const childConfig = config.collections[childShort]; + if (opts.forSpaces && childConfig?.allowInSpaces === false) continue; + const on = getRelationField(rel); + const childTable = tableFor(childShort, opts); + const idxName = `idx_${np}${sanitizeName(childShort)}_${sanitizeName(on)}`; + indexes.push( + `CREATE INDEX IF NOT EXISTS ${idxName} ON ${childTable}(${dialect.indexExpression(dialect.jsonExtract('record', on))})` + ); + } + } + return indexes; +} + +export function buildCountColumns(config: ContrailConfig, opts: BuilderOpts = {}): string[] { + const resolved = getResolved(config); + const stmts: string[] = []; + const addedColumns = new Map<string, Set<string>>(); + const np = namePrefix(opts); + + for (const [collection, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const table = tableFor(collection, opts); + const relMap = resolved.relations[collection] ?? {}; + + if (!addedColumns.has(table)) addedColumns.set(table, new Set()); + const tableColumns = addedColumns.get(table)!; + + for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { + if (rel.count === false) continue; + if (opts.forSpaces && config.collections[rel.collection]?.allowInSpaces === false) continue; + const totalCol = countColumnName(rel.collection); + if (!tableColumns.has(totalCol)) { + tableColumns.add(totalCol); + stmts.push( + `ALTER TABLE ${table} ADD COLUMN ${totalCol} INTEGER NOT NULL DEFAULT 0` + ); + } + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${totalCol} ON ${table}(${totalCol} DESC, time_us DESC)` + ); + + const mapping = relMap[relName]; + if (mapping) { + for (const groupKey of Object.keys(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + if (!tableColumns.has(groupCol)) { + tableColumns.add(groupCol); + stmts.push( + `ALTER TABLE ${table} ADD COLUMN ${groupCol} INTEGER NOT NULL DEFAULT 0` + ); + } + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${groupCol} ON ${table}(${groupCol} DESC, time_us DESC)` + ); + } + } + } + } + return stmts; +} + +function buildFeedTables(config: ContrailConfig, dialect: SqlDialect): string[] { + if (!config.feeds || Object.keys(config.feeds).length === 0) return []; + const stmts = [ + `CREATE TABLE IF NOT EXISTS feed_items ( + actor TEXT NOT NULL, + uri TEXT NOT NULL, + collection TEXT NOT NULL, + time_us ${dialect.bigintType} NOT NULL, + PRIMARY KEY (actor, uri) + )`, + `CREATE INDEX IF NOT EXISTS idx_feed_actor_coll_time ON feed_items(actor, collection, time_us DESC)`, + `CREATE INDEX IF NOT EXISTS idx_feed_actor_time ON feed_items(actor, time_us DESC)`, + `CREATE TABLE IF NOT EXISTS feed_backfills ( + actor TEXT NOT NULL, + feed TEXT NOT NULL, + completed INTEGER NOT NULL DEFAULT 0, + PRIMARY KEY (actor, feed) + )`, + ]; + + const followCollections = new Set(Object.values(config.feeds).map((f) => f.follow)); + for (const col of followCollections) { + const table = recordsTableName(col); + const safe = sanitizeName(col); + stmts.push( + `CREATE INDEX IF NOT EXISTS idx_${safe}_subject ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', 'subject'))})` + ); + } + + return stmts; +} + +export function buildFtsTables( + config: ContrailConfig, + dialect: SqlDialect, + opts: BuilderOpts = {} +): string[] { + const stmts: string[] = []; + for (const [collection, colConfig] of Object.entries(config.collections)) { + if (opts.forSpaces && colConfig.allowInSpaces === false) continue; + const fields = getSearchableFields(collection, colConfig); + if (!fields || fields.length === 0) continue; + const table = tableFor(collection, opts); + stmts.push(...buildFtsSchema(dialect, table, fields)); + } + return stmts; +} + +const MIGRATIONS = [ + "ALTER TABLE backfills ADD COLUMN retries INTEGER NOT NULL DEFAULT 0", + "ALTER TABLE backfills ADD COLUMN last_error TEXT", + "ALTER TABLE spaces_invites ADD COLUMN kind TEXT NOT NULL DEFAULT 'join'", +]; + +async function runMigrations(db: Database): Promise<void> { + for (const sql of MIGRATIONS) { + try { + await db.prepare(sql).run(); + } catch { + // Column already exists — ignore + } + } +} + +/** Pluggable schema applier — passed in by extension packages (community, + * third-party plugins) to install their own tables alongside contrail's. */ +export type SchemaModule = (db: Database) => Promise<void>; + +export interface InitSchemaOptions { + /** Separate DB for the spaces tables. Defaults to the main `db`. */ + spacesDb?: Database; + /** Extra schema modules to apply after contrail's own DDL. Used by the + * community package to install its tables — contrail core no longer + * imports community schema directly. */ + extraSchemas?: SchemaModule[]; +} + +async function applySpacesSchema( + target: Database, + config: ContrailConfig, + dialect: SqlDialect +): Promise<void> { + const base = buildSpacesBaseSchema(dialect); + const perCollection = buildCollectionTables(config, dialect, { forSpaces: true }); + const indexes = buildDynamicIndexes(config, dialect, { forSpaces: true }); + await target.batch([...base, ...perCollection, ...indexes].map((s) => target.prepare(s))); + + const ftsStmts = buildFtsTables(config, dialect, { forSpaces: true }); + for (const stmt of ftsStmts) { + try { await target.prepare(stmt).run(); } catch { /* FTS5 unavailable */ } + } + for (const stmt of buildCountColumns(config, { forSpaces: true })) { + try { await target.prepare(stmt).run(); } catch { /* already exists */ } + } +} + +export async function initSchema( + db: Database, + config: ContrailConfig, + options: InitSchemaOptions = {} +): Promise<void> { + const dialect = getDialect(db); + const baseStatements = buildBaseSchema(dialect).split(";") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + const collectionStatements = buildCollectionTables(config, dialect); + const indexStatements = buildDynamicIndexes(config, dialect); + const ftsStatements = buildFtsTables(config, dialect); + const feedStatements = buildFeedTables(config, dialect); + + const spacesDb = options.spacesDb; + const spacesSharesMainDb = !spacesDb || spacesDb === db; + + const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...feedStatements]; + + await db.batch(all.map((s) => db.prepare(s))); + + if (config.spaces?.authority || config.spaces?.recordHost) { + await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); + } + + // Extension schemas (e.g. community) — applied to the spacesDb when one's + // configured separately, since they typically reference space_uri. The + // caller is responsible for routing the schema to the right db; we just + // hand it the spaces-or-main DB as a sensible default. + const extensionTarget = spacesSharesMainDb ? db : spacesDb!; + for (const apply of options.extraSchemas ?? []) { + await apply(extensionTarget); + } + + if (config.labels) { + // Labels tables live on the main DB — they're keyed by at-URI / DID and + // are read alongside public records during hydration. + const labelsStmts = buildLabelsSchema(dialect); + await db.batch(labelsStmts.map((s) => db.prepare(s))); + } + + // FTS5 may not be available (e.g. node:sqlite) — skip gracefully + for (const stmt of ftsStatements) { + try { + await db.prepare(stmt).run(); + } catch { + // FTS5 not supported in this environment + } + } + await runMigrations(db); + + // Add count columns (ALTER TABLE — may already exist) + for (const stmt of buildCountColumns(config)) { + try { + await db.prepare(stmt).run(); + } catch { + // Column/index already exists — ignore + } + } +} diff --git a/packages/contrail-appview/src/core/dialect.ts b/packages/contrail-appview/src/core/dialect.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/dialect.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/identity.ts b/packages/contrail-appview/src/core/identity.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/identity.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/invite/community-handler.ts b/packages/contrail-appview/src/core/invite/community-handler.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/invite/community-handler.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/invite/index.ts b/packages/contrail-appview/src/core/invite/index.ts new file mode 100644 index 0000000..8dae0df --- /dev/null +++ b/packages/contrail-appview/src/core/invite/index.ts @@ -0,0 +1,3 @@ +export { generateInviteToken, hashInviteToken, mintInviteToken } from "./token"; +export { registerInviteRoutes } from "./router"; +export type { InviteRoutesOptions } from "./router"; diff --git a/packages/contrail-appview/src/core/invite/router.ts b/packages/contrail-appview/src/core/invite/router.ts new file mode 100644 index 0000000..920123f --- /dev/null +++ b/packages/contrail-appview/src/core/invite/router.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-authority"; diff --git a/packages/contrail-appview/src/core/invite/token.ts b/packages/contrail-appview/src/core/invite/token.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/invite/token.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/jetstream.ts b/packages/contrail-appview/src/core/jetstream.ts new file mode 100644 index 0000000..d42a5e8 --- /dev/null +++ b/packages/contrail-appview/src/core/jetstream.ts @@ -0,0 +1,280 @@ +import { JetstreamSubscription } from "@atcute/jetstream"; +import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; +import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS } from "./types"; +import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; +import { refreshStaleIdentities } from "./identity"; + +const BATCH_SIZE = 50; +const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour + +/** Mutable state that persists across ingest cycles within the same process. */ +export interface IngestState { + cachedKnownDids?: Set<string>; + schemaInitialized: boolean; + lastFeedPruneMs: number; +} + +export function createIngestState(): IngestState { + return { schemaInitialized: false, lastFeedPruneMs: 0 }; +} + +function getLogger(config: ContrailConfig): Logger { + return config.logger ?? console; +} + +export async function ingestEvents( + config: ContrailConfig, + cursor: number | null, + safetyTimeoutMs: number = 25_000, + knownDids?: Set<string> +): Promise<{ events: IngestEvent[]; lastCursor: number | null }> { + const log = getLogger(config); + const startTimeUs = Date.now() * 1000; + const deadline = Date.now() + safetyTimeoutMs; + const collected: IngestEvent[] = []; + + const collections = getCollectionNsids(config); + const dependentCollections = new Set(getDependentNsids(config)); + const urls = config.jetstreams ?? []; + + let totalCommits = 0; + let filteredUnknownDid = 0; + const filteredDidSamples = new Set<string>(); + let lastYieldedTimeUs: number | null = null; + let firstYieldedTimeUs: number | null = null; + let connectCount = 0; + const seenUris = new Map<string, number>(); // uri -> time_us of first occurrence + const duplicateUris: string[] = []; + + const subscription = new JetstreamSubscription({ + url: urls, + wantedCollections: collections, + ...(cursor !== null ? { cursor } : {}), + onConnectionOpen() { + connectCount++; + log.log( + `[ingest] connected to Jetstream #${connectCount} (url=${urls.join("|")}, cursor=${cursor ?? "none"}, wanted=${collections.join(",")})` + ); + }, + onConnectionClose(event) { + log.log( + `[ingest] disconnected from Jetstream: ${event.code} ${event.reason}` + ); + }, + onConnectionError(event) { + log.error("[ingest] Jetstream error:", event.error); + }, + }); + + for await (const event of subscription) { + if (firstYieldedTimeUs === null) firstYieldedTimeUs = event.time_us; + lastYieldedTimeUs = event.time_us; + if (event.kind === "commit") { + const { commit } = event; + totalCommits++; + + const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; + + if (dependentCollections.has(commit.collection) && knownDids) { + if (!knownDids.has(event.did)) { + filteredUnknownDid++; + if (filteredDidSamples.size < 10) filteredDidSamples.add(event.did); + continue; + } + } + + const prev = seenUris.get(uri); + if (prev !== undefined) { + duplicateUris.push(uri); + log.warn( + `[ingest] DUPLICATE in cycle: ${uri} first time_us=${prev}, again=${event.time_us}, delta=${event.time_us - prev}us` + ); + } else { + seenUris.set(uri, event.time_us); + } + + const now = Date.now(); + + collected.push({ + uri, + did: event.did, + time_us: event.time_us, + collection: commit.collection, + operation: commit.operation as "create" | "update" | "delete", + rkey: commit.rkey, + cid: commit.operation === "delete" ? null : commit.cid, + record: + commit.operation === "delete" + ? null + : JSON.stringify(commit.record), + indexed_at: now * 1000, + }); + + log.log( + `[ingest] keep: ${commit.operation} ${uri} time_us=${event.time_us}` + ); + + if (knownDids && !dependentCollections.has(commit.collection)) { + knownDids.add(event.did); + } + } + + if (event.time_us >= startTimeUs) { + log.log( + `[ingest] caught up to present, stopping (last time_us=${event.time_us}, startTimeUs=${startTimeUs})` + ); + break; + } + + if (Date.now() >= deadline) { + log.log( + `[ingest] safety timeout reached, stopping (deadline=${deadline}, collected=${collected.length})` + ); + break; + } + } + + if (filteredUnknownDid > 0) { + const sample = [...filteredDidSamples].join(", "); + log.log( + `[ingest] ${filteredUnknownDid} events filtered (unknown did). sample dids: ${sample}` + ); + } + const lastCursor = subscription.cursor || null; + + const cursorGap = + lastCursor !== null && lastYieldedTimeUs !== null + ? lastCursor - lastYieldedTimeUs + : null; + + // Detect the library's internal cursor rollback (picks a different URL → rolls + // back 10s → first event comes in BEFORE the cursor we asked it to start from). + const rolledBackUs = + cursor !== null && firstYieldedTimeUs !== null && firstYieldedTimeUs < cursor + ? cursor - firstYieldedTimeUs + : 0; + + log.log( + `[ingest] jetstream loop done. commits_seen=${totalCommits}, filtered=${filteredUnknownDid}, kept=${collected.length}, dupes=${duplicateUris.length}, connects=${connectCount}, first_yielded=${firstYieldedTimeUs ?? "none"}, last_yielded=${lastYieldedTimeUs ?? "none"}, subscription_cursor=${lastCursor ?? "none"}, cursor_gap=${cursorGap ?? "n/a"}us, rolled_back=${rolledBackUs}us` + ); + + if (cursorGap !== null && cursorGap > 1000) { + log.warn( + `[ingest] CURSOR GAP: subscription cursor is ${cursorGap}us (${Math.floor( + cursorGap / 1000 + )}ms) ahead of last yielded event — buffered events may be dropped` + ); + } + + if (connectCount > 1) { + log.warn( + `[ingest] RECONNECTED ${connectCount} times during cycle — each reconnect picks a URL at random and rolls cursor back 10s` + ); + } + + return { events: collected, lastCursor }; +} + +// Run a full ingest cycle: init schema, load cursor, ingest, apply, save cursor +export async function runIngestCycle( + db: Database, + config: ContrailConfig, + timeoutMs: number = 25_000, + state?: IngestState, + pubsub?: import("./realtime/types").PubSub +): Promise<void> { + const log = getLogger(config); + const s = state ?? createIngestState(); + + if (!s.schemaInitialized) { + await initSchema(db, config); + s.schemaInitialized = true; + } + + const cursor = await getLastCursor(db); + const collections = getCollectionNsids(config); + const nowUs = Date.now() * 1000; + const lagMs = cursor !== null ? Math.floor((nowUs - cursor) / 1000) : null; + + log.log( + `[ingest] starting cycle. cursor=${cursor ?? "none"}${ + lagMs !== null ? ` (lag=${lagMs}ms)` : "" + }, timeout=${timeoutMs}ms, collections=${collections.join(", ")}` + ); + + // Load known DIDs for filtering dependent collections + const dependentCollections = getDependentNsids(config); + let knownDids: Set<string> | undefined; + + if (dependentCollections.length > 0) { + if (s.cachedKnownDids) { + knownDids = s.cachedKnownDids; + log.log(`Using cached known DIDs (${knownDids.size} users)`); + } else { + const result = await db + .prepare("SELECT did FROM identities") + .all<{ did: string }>(); + knownDids = new Set((result.results ?? []).map((r) => r.did)); + s.cachedKnownDids = knownDids; + log.log(`Loaded ${knownDids.size} known DIDs from database`); + } + } + + const { events, lastCursor } = await ingestEvents( + config, + cursor, + timeoutMs, + knownDids + ); + + if (events.length > 0) { + const breakdown: Record<string, number> = {}; + for (const e of events) { + const key = `${e.collection}:${e.operation}`; + breakdown[key] = (breakdown[key] ?? 0) + 1; + } + log.log( + `[ingest] received ${events.length} events. breakdown=${JSON.stringify(breakdown)}` + ); + } else { + log.log(`[ingest] received 0 events from Jetstream`); + } + + for (let i = 0; i < events.length; i += BATCH_SIZE) { + const batch = events.slice(i, i + BATCH_SIZE); + await applyEvents(db, batch, config, { pubsub }); + } + + // Refresh stale/missing identities for DIDs in this batch + const uniqueDids = [...new Set(events.map((e) => e.did))]; + if (uniqueDids.length > 0) { + try { + await refreshStaleIdentities(db, uniqueDids); + } catch (err) { + log.warn(`Identity refresh failed: ${err}`); + } + } + + if (lastCursor !== null) { + await saveCursor(db, lastCursor); + log.log( + `[ingest] saved cursor=${lastCursor} (advanced ${ + cursor !== null ? lastCursor - cursor : "n/a" + }us)` + ); + } else { + log.log(`[ingest] no cursor returned from subscription; not saving`); + } + + // Prune feed items hourly + if (config.feeds && Date.now() - s.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { + const maxItems = Math.max( + ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) + ); + const pruned = await pruneFeedItems(db, maxItems); + if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); + s.lastFeedPruneMs = Date.now(); + } + + log.log(`[ingest] cycle complete. stored=${events.length}`); +} diff --git a/packages/contrail-appview/src/core/labels/apply.ts b/packages/contrail-appview/src/core/labels/apply.ts new file mode 100644 index 0000000..cbf5558 --- /dev/null +++ b/packages/contrail-appview/src/core/labels/apply.ts @@ -0,0 +1,64 @@ +import type { Database, Statement } from "../types"; + +/** Wire shape of a single `com.atproto.label.defs#label` entry. Field names + * match the spec exactly. We accept the spec's ISO-8601 strings and + * convert to unix seconds at the storage boundary. */ +export interface IncomingLabel { + src: string; + uri: string; + val: string; + cid?: string; + neg?: boolean; + exp?: string; + cts: string; + sig?: Uint8Array; +} + +/** Upsert a batch of labels. Idempotent on `(src, uri, val, cts)`. Bad rows + * (missing required fields, unparseable timestamps) are dropped silently; + * we don't want one malformed label to abort an entire labeler frame. */ +export async function applyLabels( + db: Database, + labels: IncomingLabel[], +): Promise<number> { + if (labels.length === 0) return 0; + const stmts: Statement[] = []; + let kept = 0; + for (const l of labels) { + if (!l.src || !l.uri || !l.val || !l.cts) continue; + const cts = isoToUnixSec(l.cts); + if (cts == null) continue; + const exp = l.exp ? isoToUnixSec(l.exp) : null; + stmts.push( + db + .prepare( + `INSERT INTO labels (src, uri, val, cid, neg, exp, cts, sig) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT(src, uri, val, cts) DO UPDATE SET + cid = excluded.cid, + neg = excluded.neg, + exp = excluded.exp, + sig = excluded.sig`, + ) + .bind( + l.src, + l.uri, + l.val, + l.cid ?? null, + l.neg ? 1 : 0, + exp, + cts, + l.sig ?? null, + ), + ); + kept++; + } + if (stmts.length > 0) await db.batch(stmts); + return kept; +} + +function isoToUnixSec(iso: string): number | null { + const ms = Date.parse(iso); + if (!Number.isFinite(ms)) return null; + return Math.floor(ms / 1000); +} diff --git a/packages/contrail-appview/src/core/labels/hydrate.ts b/packages/contrail-appview/src/core/labels/hydrate.ts new file mode 100644 index 0000000000000000000000000000000000000000..02b947a44ef764f5c31f64fefd468727f76fd7d1 GIT binary patch literal 3547 zcmd1IEyyn_Q7EY_NL8p-a7iplOiC<HRj5@+E6UGRP}0-W2T2txS#v>ERx2bWmLzAS zrg-KBmZla}A}cM*FD*$e($7fEDM&2>>EY7X(ozV|EK1ca&PXgsRY=P(Qb<(Ds7xtJ zEJ;mK$Vp5}%~5#NFhwC3Y-X`SLUMktUSdf>QGQ9j9#~p0B{i*B8O%)JQqWRRNXtyk zNzqNqFVY3G6pHf|lJoP5OLJ3;iWQO*^Arjai;ER9OB9MriV`!^GfEUnGK%s`(=!xG z@)e*)D+IfE7wai#>2sx4fc>ADSCU$kmYAHX-~n}<57=$hTnY-sMafnm!!q;It+^Bw zN{cd)xn+qt$lT=26nkVoC;<HPaw-)Hic*VH^GX!TGg9*uN{UKT!PYA%<fW$DTY&{Y zhJ%!Q2K(z;m>C%ADCFm6mVm+utS2vDA+@3)v#3%bvsj@xwFIg^wW0vSLYN5($wjG& zC7Jno3MHAjsl_FUxdl*N$tA@ISJrZIA^abdSPu4jP=2`rIlhM(t5BnmSDKRp@>E`G zx|Kp+X>L+#5r~^wQ2^s2DFV9)rl^(+oO0Y!OOi7nsiYX1c8XI=6!Oy)ic6ESQj<%H zbrjMvb4pT+KyjO#n3I!Qq+YC$n4Fwi017v7s!1&>)&VD+<oukR#Dd~fg#?Y_qGTOV zMCvG%CFW=*D3p{I<fIlWBvmTpB$lKWmnbA8mlP-HC?w{kD5MnS7r+#hXXF>Bf(?VJ z%Pdw%R7l86O}7O_X@WvgYDrOIGAPLP6oOJqN{jM93JOw-bfGR*$W1KJ<5JK98xjol z1vuFzr79$r=!OP)7AxczDI_Hpr7F01x)g)6M@dGiLUCelszOO(QckKuMt)98u|iUQ zNd`DNQz2<MvA8lXSs|@74{Sdymw_X*Sc6MJAtlKQp22lM=^AP~EXJZ^L87q0LlrGb zP0lY$an4L}sthg4w6{|5O)Rh}E-A{)OV@$s4QO22>2PUUDFhVd=VlhC+6001!BwJU zuIN}hJ8%-qOjFQ+22ZhGPHJ9yNrr;0t*wHALQM@kGEpQnK>=M_l&4T#Yt6-_prEg> z;F6kBT967)7^uPK;_0H0n^>ukSWu9fSfr4dr;w16nNpkpj^M;%1yJ}x0v}>nVrfZ6 zevvg;931WtNw68o`MIf((lIBqxI{r47SMWJ3gCjMM4>b<vk)G&whGaDdU|=O<qE;6 zC2((Q#)5JJsN8}mcZ4NbROPV904awXUyzfSoC-<;paRbpY#U6wUT$K6hK8nst(}6B zy^^M0R(@ulhLVnwCc?J7{PN(`WCdFV-^7v(y|kSC{2~pP#FA9Ky!>(vO$B`gLjwZ? zO<1TpW#*-1=A{=alw~HO7yOAusa9am6%?hWWmf1Y=;`StXO!k;$Lc8rl!A)gRE31p zih=}4m_jqER#IwOeo?9xIBXN*<30W2;}aB+`~)h{LGh6Uv05R&C?&NBRzX0F0!ar4 z`k?3nD?`LVQGR)`f~`Vgd17V>n!|07ia0w^#sFmvP|{aWNC<ZIadi#>75RwrK}R7u zGerkfQ0st7U>$|zl41q7Ab($I$;71q0pT95L9QUJ3Z8xn8Y<ODek!ijgsD<caP)Ig z&`7N)Q1A>^@C)_vQSc7}nX6!@V2`G%v?vp779^B36CjQRr6*7lgGIWI0whks*;fIa z9$={tqyZk60h!6!;82H_T}Aoj3I(Y}n8l%<LP%z6u|j?xq!?5vN-fUMDN9vIs)PoA zVs>U;I@l<X86_EsB?^f}MVV#bDjifBXXb%r3&9P9qSWHjoDxVumI6{$q??jjkeZj0 znpXlbFsZaep$JsqW#%aqfXmP1RBMIeg4AS%l>F4<JoOTVl+?7$yi|pf%v9Z^qExU0 zK#>h<oG5swRw~#+OCdx;hxpQ|5+V!AfS@wVJGD|r0VM%|^9HD4q5;W{ATQ;oDS#4? zCb)P~fM)A#ur&!P)kXQ`dc{S_wG0quX;CJOS(cbnn*eeGOiw{kW`2=^EwowzJ4P=( zwL~L3wGwQvf<k7Rf`%eQSxpVdRy|Piva?kHi-9<rD7uRwx^+NG!A8`A3P6a{^Gi#t z6bKZ#3bx=vQvqC`qIwxA#PrG%b4pW-H8f#C0CFbC-+8I&nxOC~$;>NFwFVWu&?+xA z1>_EJQS9vLqEL{TXQhw`$sr2h6q1*in+mE7LD8C-mYJ*osZ$g{suK$eav(`Ar&6IL zADmb8^&$GoQj3Z+^Yio+Jk!953R2C3TY0G|3W-o-bdxhvKy@p)L<f}*Mfv6G#V~_F zcBEt$C#DyrrWWf!s`ShfcpU+5C%}^$$T1*SfNg`?4l}4cRlO(`TKQz=rRym;CMV~Y z=9TE?q?V=TK)hS5paH6Hpf#GNLULkJQKdp&KFFsqa|$x^K(!Dev6rVR<b#@t$r-81 z+0aHJsM3ej(F)0+%D+6bBtsz)WDF?jLy`_CPC#B)Q-cNstd53794sY5LfbhrMZp$L zB{)HX%qcC(1g9uaaR!cWaODlsqG$`MXCQenMFH6^1&|a-b8==1Qo;cH6Pz+(_JhON zN&%$>07{EsmC!~u$Q)493@Qe0X@kT{i!wna8OUAW<~B&IEHMWr25vf*6o=${7Uye# z<dRE@H6fC<kjx17O@6r^*pXnDfS90=fC!?h0+r2*w&3O~DDNeu=IDWhY!#60NUbP< z29`#CX-PE5+R~!TSOt4~TZQOYO}&EB;tUP2RbXY{uqXnz2o>^6ORTwSxwzm>HJG`j wd6^Z#smZX`u_mky1}=u6nn48_sJ_;KDpt^f)Yy6@`JTc4!QdvDh9<~-0O=5WfdBvi literal 0 HcmV?d00001 diff --git a/packages/contrail-appview/src/core/labels/resolve.ts b/packages/contrail-appview/src/core/labels/resolve.ts new file mode 100644 index 0000000..694bfec --- /dev/null +++ b/packages/contrail-appview/src/core/labels/resolve.ts @@ -0,0 +1,134 @@ +import { + CompositeDidDocumentResolver, + PlcDidDocumentResolver, + WebDidDocumentResolver, +} from "@atcute/identity-resolver"; +import type { Did } from "@atcute/lexicons"; +import type { Database } from "../types"; + +/** Reject endpoint URLs that point to private/internal addresses or non-HTTPS. + * Mirrors the validator in core/client.ts — labeler endpoints should be + * publicly reachable for the same reasons PDS endpoints should. */ +function validateEndpointUrl(url: string): boolean { + try { + const parsed = new URL(url); + if (parsed.protocol !== "https:") return false; + const host = parsed.hostname; + if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; + if (host.startsWith("10.")) return false; + if (host.startsWith("192.168.")) return false; + if (host.startsWith("169.254.")) return false; + if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; + return true; + } catch { + return false; + } +} + +const didResolver = new CompositeDidDocumentResolver({ + methods: { + plc: new PlcDidDocumentResolver(), + web: new WebDidDocumentResolver(), + }, +}); + +/** Look up the labeler service endpoint from a DID. + * Reads the DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ +export async function resolveLabelerEndpoint(did: string): Promise<string | null> { + if (!did.startsWith("did:plc:") && !did.startsWith("did:web:")) return null; + try { + const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); + const endpoint = doc.service + ?.find((s) => s.id === "#atproto_labeler") + ?.serviceEndpoint?.toString(); + if (!endpoint) return null; + if (!validateEndpointUrl(endpoint)) return null; + return endpoint; + } catch { + return null; + } +} + +/** State row for a labeler — the per-DID equivalent of the singleton + * jetstream `cursor` table, with cached endpoint to avoid repeated DID-doc + * fetches. */ +export interface LabelerState { + did: string; + cursor: number; + endpoint: string | null; + resolved_at: number | null; +} + +const ENDPOINT_TTL_MS = 6 * 60 * 60 * 1000; // 6h, matches the recommended client cache for label-defs + +/** Get cached `(endpoint, cursor)` for a labeler. Resolves endpoint on + * cache miss or staleness; persists endpoint + resolved_at back to the DB + * so subsequent ingest cycles avoid the network round-trip. */ +export async function getLabelerState( + db: Database, + did: string, + endpointOverride: string | undefined, +): Promise<LabelerState | null> { + const row = await db + .prepare( + "SELECT did, cursor, endpoint, resolved_at FROM labeler_cursors WHERE did = ?", + ) + .bind(did) + .first<LabelerState>(); + + let endpoint = endpointOverride ?? row?.endpoint ?? null; + const stale = + !row?.resolved_at || Date.now() - row.resolved_at > ENDPOINT_TTL_MS; + + if (!endpoint || (!endpointOverride && stale)) { + endpoint = await resolveLabelerEndpoint(did); + if (!endpoint) return null; + const now = Date.now(); + await db + .prepare( + `INSERT INTO labeler_cursors (did, cursor, endpoint, resolved_at) + VALUES (?, ?, ?, ?) + ON CONFLICT(did) DO UPDATE SET endpoint = excluded.endpoint, resolved_at = excluded.resolved_at`, + ) + .bind(did, row?.cursor ?? 0, endpoint, now) + .run(); + return { + did, + cursor: row?.cursor ?? 0, + endpoint, + resolved_at: now, + }; + } + + return row ?? { did, cursor: 0, endpoint, resolved_at: null }; +} + +/** Persist the highest seen seq number for a labeler. Idempotent; + * the next ingest cycle resumes from `cursor + 1` via the `?cursor=` param. */ +export async function saveLabelerCursor( + db: Database, + did: string, + cursor: number, +): Promise<void> { + await db + .prepare( + `INSERT INTO labeler_cursors (did, cursor) + VALUES (?, ?) + ON CONFLICT(did) DO UPDATE SET cursor = excluded.cursor`, + ) + .bind(did, cursor) + .run(); +} + +/** Reset cursor to 0 — used in response to `#info { name: "OutdatedCursor" }` + * frames, which signal that the labeler's seq history was rewound. */ +export async function resetLabelerCursor(db: Database, did: string): Promise<void> { + await db + .prepare( + `INSERT INTO labeler_cursors (did, cursor) + VALUES (?, 0) + ON CONFLICT(did) DO UPDATE SET cursor = 0`, + ) + .bind(did) + .run(); +} diff --git a/packages/contrail-appview/src/core/labels/schema.ts b/packages/contrail-appview/src/core/labels/schema.ts new file mode 100644 index 0000000..486ce27 --- /dev/null +++ b/packages/contrail-appview/src/core/labels/schema.ts @@ -0,0 +1,30 @@ +import type { SqlDialect } from "../dialect"; + +/** DDL for the labels module. Single `labels` table covers record-level + * (uri starts with `at://`) and account-level (uri is a bare DID) entries — + * the spec collapses both into the same row shape. `labeler_cursors` + * mirrors the role of the singleton `cursor` table for jetstream, but + * per-labeler. */ +export function buildLabelsSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS labels ( + src TEXT NOT NULL, + uri TEXT NOT NULL, + val TEXT NOT NULL, + cid TEXT, + neg INTEGER NOT NULL DEFAULT 0, + exp ${dialect.bigintType}, + cts ${dialect.bigintType} NOT NULL, + sig BLOB, + PRIMARY KEY (src, uri, val, cts) + )`, + `CREATE INDEX IF NOT EXISTS idx_labels_uri ON labels(uri)`, + `CREATE INDEX IF NOT EXISTS idx_labels_src_cts ON labels(src, cts DESC)`, + `CREATE TABLE IF NOT EXISTS labeler_cursors ( + did TEXT PRIMARY KEY, + cursor ${dialect.bigintType} NOT NULL DEFAULT 0, + endpoint TEXT, + resolved_at ${dialect.bigintType} + )`, + ]; +} diff --git a/packages/contrail-appview/src/core/labels/select.ts b/packages/contrail-appview/src/core/labels/select.ts new file mode 100644 index 0000000..da89d9c --- /dev/null +++ b/packages/contrail-appview/src/core/labels/select.ts @@ -0,0 +1,64 @@ +import type { LabelsConfig } from "./types"; +import { DEFAULT_LABELS_MAX_PER_REQUEST } from "./types"; + +/** Pick which labelers to honor for this request. + * + * Order of precedence: + * 1. `atproto-accept-labelers` header (atproto spec) + * 2. `?labelers=` query param (fallback for SSE/WS where headers are awkward) + * 3. `config.defaults` (operator policy) + * 4. every entry in `config.sources` + * + * Each candidate DID is checked against `config.sources`. Unknowns are + * dropped — we only have rows for labelers we've subscribed to. + * + * Header values can carry `;param` modifiers (e.g. `did:plc:...;redact`); + * v1 strips and ignores those — only the bare DID is honored. */ +export interface SelectedLabelers { + /** DIDs to use for hydration this request. */ + accepted: string[]; +} + +export function selectAcceptedLabelers( + headerValue: string | null | undefined, + paramValue: string | null | undefined, + cfg: LabelsConfig, +): SelectedLabelers { + const cap = cfg.maxPerRequest ?? DEFAULT_LABELS_MAX_PER_REQUEST; + const known = new Set(cfg.sources.map((s) => s.did)); + + const fromCaller = parseLabelerList(headerValue) ?? parseLabelerList(paramValue); + + let candidates: string[]; + if (fromCaller && fromCaller.length > 0) { + candidates = fromCaller; + } else { + candidates = (cfg.defaults ?? cfg.sources.map((s) => s.did)).slice(); + } + + const accepted: string[] = []; + const seen = new Set<string>(); + for (const did of candidates) { + if (seen.has(did)) continue; + seen.add(did); + if (known.has(did)) accepted.push(did); + if (accepted.length >= cap) break; + } + + return { accepted }; +} + +/** Parse a comma-separated DID list. Returns null when the input is empty + * or undefined so callers can distinguish "absent" from "empty list" (the + * latter — `atproto-accept-labelers: ` — is technically valid and means + * "no labelers"; we treat it the same as absent for ergonomics). */ +function parseLabelerList(value: string | null | undefined): string[] | null { + if (!value) return null; + const out: string[] = []; + for (const raw of value.split(",")) { + // Drop `;param` modifiers from the spec (e.g. `;redact`). v1 ignores them. + const head = raw.split(";")[0]!.trim(); + if (head.startsWith("did:")) out.push(head); + } + return out.length > 0 ? out : null; +} diff --git a/packages/contrail-appview/src/core/labels/subscribe.ts b/packages/contrail-appview/src/core/labels/subscribe.ts new file mode 100644 index 0000000..c80ed2b --- /dev/null +++ b/packages/contrail-appview/src/core/labels/subscribe.ts @@ -0,0 +1,315 @@ +import { decodeFirst } from "@atcute/cbor"; +import type { ContrailConfig, Database, Logger } from "../types"; +import type { LabelerSource } from "./types"; +import { applyLabels, type IncomingLabel } from "./apply"; +import { + getLabelerState, + resetLabelerCursor, + saveLabelerCursor, +} from "./resolve"; + +const DEFAULT_CYCLE_TIMEOUT_MS = 25_000; +const DEFAULT_BATCH_SIZE = 100; +const DEFAULT_FLUSH_INTERVAL_MS = 5_000; + +function getLogger(config: ContrailConfig): Logger { + return config.logger ?? console; +} + +/** One catch-up cycle for every configured labeler. Designed to fit inside a + * Cloudflare Workers cron tick — we drain frames until the labeler has no + * more buffered events for us, or `timeoutMs` is reached, then save cursor + * and disconnect. Mirrors the shape of `runIngestCycle` for jetstream. */ +export async function runLabelIngestCycle( + db: Database, + config: ContrailConfig, + timeoutMs = DEFAULT_CYCLE_TIMEOUT_MS, +): Promise<void> { + if (!config.labels) return; + const log = getLogger(config); + const deadline = Date.now() + timeoutMs; + + for (const source of config.labels.sources) { + if (Date.now() >= deadline) { + log.log(`[labels] cycle deadline hit before processing ${source.did}`); + break; + } + const remaining = Math.max(2_000, deadline - Date.now()); + try { + await pumpOneLabeler(db, source, log, remaining, /* persistent */ false); + } catch (err) { + log.warn(`[labels] cycle for ${source.did} failed: ${err}`); + } + } +} + +export interface PersistentLabelsOptions { + signal?: AbortSignal; + batchSize?: number; + flushIntervalMs?: number; + logger?: Logger; +} + +/** Long-lived equivalent — keeps one socket per labeler open forever, with + * exponential backoff reconnect. Mirrors `runPersistent` for jetstream. */ +export async function runPersistentLabels( + db: Database, + config: ContrailConfig, + options: PersistentLabelsOptions = {}, +): Promise<void> { + if (!config.labels) return; + const log = options.logger ?? config.logger ?? console; + const signal = options.signal; + + const tasks = config.labels.sources.map((source) => + runOneLabelerForever(db, source, log, signal, options), + ); + await Promise.all(tasks); +} + +async function runOneLabelerForever( + db: Database, + source: LabelerSource, + log: Logger, + signal: AbortSignal | undefined, + options: PersistentLabelsOptions, +): Promise<void> { + let attempts = 0; + while (!signal?.aborted) { + try { + await pumpOneLabeler(db, source, log, /* timeoutMs */ Infinity, true, { + signal, + batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE, + flushIntervalMs: options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, + }); + attempts = 0; + } catch (err) { + if (signal?.aborted) break; + log.error(`[labels] ${source.did} stream error: ${err}`); + const delay = Math.min(1_000 * 2 ** attempts, 30_000); + attempts++; + log.log(`[labels] ${source.did} reconnecting in ${delay}ms (attempt ${attempts})`); + await new Promise((r) => setTimeout(r, delay)); + } + } +} + +interface PumpOptions { + signal?: AbortSignal; + batchSize?: number; + flushIntervalMs?: number; +} + +/** Open a `subscribeLabels` WebSocket, drain frames into a buffer, flush + * the buffer to `labels` in batches, and persist the seq cursor. Returns + * when: + * - the labeler closes the socket cleanly (caught up + no more events) + * - `timeoutMs` is reached (cron mode) + * - `signal` is aborted (persistent mode) + * - an error tears the socket down (caller may retry) */ +async function pumpOneLabeler( + db: Database, + source: LabelerSource, + log: Logger, + timeoutMs: number, + persistent: boolean, + pumpOpts: PumpOptions = {}, +): Promise<void> { + const state = await getLabelerState(db, source.did, source.endpoint); + if (!state) { + log.warn(`[labels] could not resolve labeler endpoint for ${source.did}; skipping`); + return; + } + + // First-time policy: cursor 0 = "from the beginning" if backfill is on + // (default), null = "from now" otherwise. After the first cycle we always + // resume from the saved cursor — `backfill` only flips the start point. + const isFirstRun = state.cursor === 0 && state.resolved_at === null; + const backfill = source.backfill !== false; + const startCursor = isFirstRun && !backfill ? null : state.cursor; + + const url = buildWsUrl(state.endpoint!, startCursor); + log.log(`[labels] connecting to ${source.did} (cursor=${startCursor ?? "now"})`); + + const ws = new WebSocket(url); + ws.binaryType = "arraybuffer"; + + const buffer: IncomingLabel[] = []; + let highestSeq = state.cursor; + let flushing = false; + let resolveDone!: () => void; + let rejectDone!: (err: unknown) => void; + const done = new Promise<void>((res, rej) => { + resolveDone = res; + rejectDone = rej; + }); + + const flush = async () => { + if (buffer.length === 0 || flushing) return; + flushing = true; + const batch = buffer.splice(0); + try { + const kept = await applyLabels(db, batch); + if (highestSeq > state.cursor) { + await saveLabelerCursor(db, source.did, highestSeq); + state.cursor = highestSeq; + } + log.log( + `[labels] ${source.did} flushed ${kept}/${batch.length} labels, cursor=${highestSeq}`, + ); + } catch (err) { + log.error(`[labels] ${source.did} flush failed: ${err}`); + } finally { + flushing = false; + } + }; + + const batchSize = pumpOpts.batchSize ?? DEFAULT_BATCH_SIZE; + const flushInterval = pumpOpts.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS; + const flushTimer = setInterval(() => { + flush().catch(() => {}); + }, flushInterval); + + const cleanup = () => { + clearInterval(flushTimer); + try { + ws.close(); + } catch { + /* already closed */ + } + }; + + // External abort (persistent mode) — close socket gracefully. + const abortHandler = () => { + cleanup(); + flush().finally(() => resolveDone()); + }; + pumpOpts.signal?.addEventListener("abort", abortHandler, { once: true }); + + // Cron-mode time budget — close socket gracefully when reached. + let deadlineTimer: ReturnType<typeof setTimeout> | undefined; + if (Number.isFinite(timeoutMs)) { + deadlineTimer = setTimeout(() => { + log.log(`[labels] ${source.did} cycle deadline reached, closing`); + cleanup(); + flush().finally(() => resolveDone()); + }, timeoutMs); + } + + ws.addEventListener("error", (ev) => { + cleanup(); + if (deadlineTimer) clearTimeout(deadlineTimer); + pumpOpts.signal?.removeEventListener("abort", abortHandler); + rejectDone(new Error(`WebSocket error: ${(ev as ErrorEvent)?.message ?? "unknown"}`)); + }); + + ws.addEventListener("close", () => { + if (deadlineTimer) clearTimeout(deadlineTimer); + pumpOpts.signal?.removeEventListener("abort", abortHandler); + flush().finally(() => { + clearInterval(flushTimer); + resolveDone(); + }); + }); + + ws.addEventListener("message", async (ev) => { + let bytes: Uint8Array; + if (ev.data instanceof ArrayBuffer) { + bytes = new Uint8Array(ev.data); + } else if (ev.data instanceof Uint8Array) { + bytes = ev.data; + } else { + // Binary-only protocol — text frames shouldn't arrive. + return; + } + const frame = decodeFrame(bytes); + if (!frame) return; + + if (frame.t === "#labels") { + const seq = Number(frame.payload?.seq ?? 0); + const rawLabels = Array.isArray(frame.payload?.labels) ? frame.payload.labels : []; + for (const raw of rawLabels) { + const lab = normalizeLabel(raw, source.did); + if (lab) buffer.push(lab); + } + if (Number.isFinite(seq) && seq > highestSeq) highestSeq = seq; + if (buffer.length >= batchSize) { + flush().catch(() => {}); + } + } else if (frame.t === "#info") { + const name = String(frame.payload?.name ?? ""); + log.log(`[labels] ${source.did} info: ${name}`); + if (name === "OutdatedCursor") { + // Labeler rewound its log — discard our cursor and let the next + // run start from the beginning. We don't reconnect here; the + // caller (or the persistent loop) will pick up the reset on retry. + await resetLabelerCursor(db, source.did); + cleanup(); + } + } else if (frame.op === -1) { + log.warn(`[labels] ${source.did} error frame: ${JSON.stringify(frame.payload)}`); + cleanup(); + } + }); + + // Workers WebSocket doesn't always emit `open`; just await `done` directly. + await done; +} + +function buildWsUrl(httpEndpoint: string, cursor: number | null): string { + const u = new URL("/xrpc/com.atproto.label.subscribeLabels", httpEndpoint); + // wss:// for HTTPS endpoints — the protocol on the labeler service is + // expected to be HTTPS already (validated at resolution time). + u.protocol = u.protocol === "https:" ? "wss:" : "ws:"; + if (cursor !== null) u.searchParams.set("cursor", String(cursor)); + return u.toString(); +} + +interface DecodedFrame { + op: number; + t: string | undefined; + payload: Record<string, unknown>; +} + +/** Decode an atproto subscription frame: two consecutive CBOR objects. + * Header `{ op, t? }`, payload — shape depends on `t`. Returns null on + * decode failure or non-object frames. */ +function decodeFrame(bytes: Uint8Array): DecodedFrame | null { + try { + const [header, rest] = decodeFirst(bytes); + if (!header || typeof header !== "object") return null; + const op = typeof (header as { op?: number }).op === "number" ? (header as { op: number }).op : 1; + const t = typeof (header as { t?: string }).t === "string" ? (header as { t: string }).t : undefined; + const [payload] = decodeFirst(rest); + if (!payload || typeof payload !== "object") return null; + return { op, t, payload: payload as Record<string, unknown> }; + } catch { + return null; + } +} + +/** Coerce a wire `Label` object into our `IncomingLabel` shape. Returns + * null when required fields are missing — we'd rather skip a row than + * insert one with placeholder values. */ +function normalizeLabel(raw: unknown, expectedSrc: string): IncomingLabel | null { + if (!raw || typeof raw !== "object") return null; + const r = raw as Record<string, unknown>; + const src = typeof r.src === "string" ? r.src : null; + const uri = typeof r.uri === "string" ? r.uri : null; + const val = typeof r.val === "string" ? r.val : null; + const cts = typeof r.cts === "string" ? r.cts : null; + if (!src || !uri || !val || !cts) return null; + // A labeler shouldn't emit labels under a different `src` than its own + // DID — drop them rather than poison our table with cross-issuer rows. + if (src !== expectedSrc) return null; + return { + src, + uri, + val, + cts, + cid: typeof r.cid === "string" ? r.cid : undefined, + neg: r.neg === true, + exp: typeof r.exp === "string" ? r.exp : undefined, + sig: r.sig instanceof Uint8Array ? r.sig : undefined, + }; +} diff --git a/packages/contrail-appview/src/core/labels/types.ts b/packages/contrail-appview/src/core/labels/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/labels/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/persistent.ts b/packages/contrail-appview/src/core/persistent.ts new file mode 100644 index 0000000..c1e1c18 --- /dev/null +++ b/packages/contrail-appview/src/core/persistent.ts @@ -0,0 +1,243 @@ +import type { JetstreamSubscription } from "@atcute/jetstream"; +import type { ContrailConfig, IngestEvent, Database, Logger, ResolvedContrailConfig } from "./types"; +import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS, resolveConfig } from "./types"; +import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; +import { refreshStaleIdentities } from "./identity"; +import { createIngestState } from "./jetstream"; +import type { IngestState } from "./jetstream"; + +const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; + +export interface PersistentIngestOptions { + batchSize?: number; + flushIntervalMs?: number; + signal?: AbortSignal; + /** Override subscription creation for testing */ + createSubscription?: (cursor: number | null) => JetstreamSubscription; + logger?: Logger; + /** Publish `collection:<nsid>` / `actor:<did>` events for each applied + * public record. Usually supplied by the Contrail instance. */ + pubsub?: import("./realtime/types").PubSub; +} + +function getLogger(config: ContrailConfig, options?: PersistentIngestOptions): Logger { + return options?.logger ?? config.logger ?? console; +} + +export async function runPersistent( + db: Database, + config: ContrailConfig, + options?: PersistentIngestOptions, +): Promise<void> { + // Internals (applyEvents, count updates, query planning) read `_resolved` + // and silently skip features when it's missing. The Contrail class resolves + // in its constructor; callers using this raw export must also get a resolved + // config, so do it defensively here. resolveConfig is idempotent. + if (!(config as ResolvedContrailConfig)._resolved) { + config = resolveConfig(config); + } + const log = getLogger(config, options); + const batchSize = options?.batchSize ?? 50; + const flushIntervalMs = options?.flushIntervalMs ?? 5_000; + const signal = options?.signal; + const state = createIngestState(); + + // Init schema once + if (!state.schemaInitialized) { + await initSchema(db, config); + state.schemaInitialized = true; + } + + // Load known DIDs for dependent collection filtering + const dependentCollections: Set<string> = new Set(getDependentNsids(config)); + let knownDids: Set<string> | undefined; + if (dependentCollections.size > 0) { + const result = await db + .prepare("SELECT did FROM identities") + .all<{ did: string }>(); + knownDids = new Set((result.results ?? []).map((r) => r.did)); + state.cachedKnownDids = knownDids; + log.log(`Loaded ${knownDids.size} known DIDs from database`); + } + + const collections = getCollectionNsids(config); + let reconnectAttempts = 0; + + while (!signal?.aborted) { + const cursor = await getLastCursor(db); + log.log(`Starting persistent ingestion. Cursor: ${cursor ?? "none"}, Collections: ${collections.join(", ")}`); + + try { + await streamAndFlush(db, config, cursor, { + batchSize, + flushIntervalMs, + signal, + collections, + dependentCollections, + knownDids, + state, + log, + createSubscription: options?.createSubscription, + pubsub: options?.pubsub, + }); + reconnectAttempts = 0; + } catch (err) { + if (signal?.aborted) break; + log.error(`Jetstream connection error: ${err}`); + const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30_000); + reconnectAttempts++; + log.log(`Reconnecting in ${delay}ms (attempt ${reconnectAttempts})...`); + await new Promise((r) => setTimeout(r, delay)); + } + } + + log.log("Persistent ingestion stopped"); +} + +interface StreamOptions { + batchSize: number; + flushIntervalMs: number; + signal?: AbortSignal; + collections: string[]; + dependentCollections: Set<string>; + knownDids?: Set<string>; + state: IngestState; + log: Logger; + createSubscription?: (cursor: number | null) => any; + pubsub?: import("./realtime/types").PubSub; +} + +async function streamAndFlush( + db: Database, + config: ContrailConfig, + cursor: number | null, + opts: StreamOptions, +): Promise<void> { + const { batchSize, flushIntervalMs, signal, collections, dependentCollections, knownDids, state, log } = opts; + + const subscription = opts.createSubscription + ? opts.createSubscription(cursor) + : new (await import("@atcute/jetstream")).JetstreamSubscription({ + url: config.jetstreams ?? [], + wantedCollections: collections, + ...(cursor !== null ? { cursor } : {}), + onConnectionOpen() { log.log("Connected to Jetstream"); }, + onConnectionClose(event: any) { log.log(`Disconnected: ${event.code} ${event.reason}`); }, + onConnectionError(event: any) { log.error("Jetstream error:", event.error); }, + }); + + const buffer: IngestEvent[] = []; + // Guards against overlap between the periodic timer flush and a main-loop + // batchSize-driven flush. The main loop only ever awaits flush() sequentially, + // but the setInterval callback is a second entry point on another tick. + let flushing = false; + + const flush = async () => { + if (buffer.length === 0 || flushing) return; + flushing = true; + const batch = buffer.splice(0); + + try { + await applyEvents(db, batch, config, { pubsub: opts.pubsub }); + + const lastTimeUs = Math.max(...batch.map((e) => e.time_us)); + await saveCursor(db, lastTimeUs); + + const uniqueDids = [...new Set(batch.map((e) => e.did))]; + if (uniqueDids.length > 0) { + try { + await refreshStaleIdentities(db, uniqueDids); + } catch (err) { + log.warn(`Identity refresh failed: ${err}`); + } + } + + if (config.feeds && Date.now() - state.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { + const maxItems = Math.max( + ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) + ); + const pruned = await pruneFeedItems(db, maxItems); + if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); + state.lastFeedPruneMs = Date.now(); + } + + log.log(`Flushed ${batch.length} events. Cursor: ${lastTimeUs}`); + } finally { + flushing = false; + } + }; + + // Periodic flush decoupled from the main loop. Runs even when Jetstream is + // idle, which is the whole point — without it, buffered events strand until + // the next event or abort. Errors log and retry next interval rather than + // propagate, so transient DB hiccups don't force a reconnect. + const flushTimer = setInterval(() => { + flush().catch((err) => log.error(`Timer flush failed: ${err}`)); + }, flushIntervalMs); + + const onAbort = () => { + clearInterval(flushTimer); + }; + signal?.addEventListener("abort", onAbort, { once: true }); + + const iterator = subscription[Symbol.asyncIterator](); + + try { + while (!signal?.aborted) { + // Per-iteration abort race so the handler can be removed synchronously + // after the race settles — otherwise addEventListener calls accumulate on + // the signal across the streamAndFlush lifetime. + let abortHandler!: () => void; + const abortPromise = new Promise<IteratorResult<any>>((resolve) => { + abortHandler = () => resolve({ value: undefined, done: true }); + signal?.addEventListener("abort", abortHandler, { once: true }); + }); + + let result: IteratorResult<any>; + try { + result = await Promise.race([iterator.next(), abortPromise]); + } finally { + signal?.removeEventListener("abort", abortHandler); + } + + if (result.done) break; + const event = result.value; + + if (event.kind === "commit") { + const { commit } = event; + + if (dependentCollections.has(commit.collection) && knownDids) { + if (!knownDids.has(event.did)) continue; + } + + const now = Date.now(); + const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; + + buffer.push({ + uri, + did: event.did, + time_us: event.time_us, + collection: commit.collection, + operation: commit.operation as "create" | "update" | "delete", + rkey: commit.rkey, + cid: commit.operation === "delete" ? null : commit.cid, + record: commit.operation === "delete" ? null : JSON.stringify(commit.record), + indexed_at: now * 1000, + }); + + if (knownDids && !dependentCollections.has(commit.collection)) { + knownDids.add(event.did); + } + } + + if (buffer.length >= batchSize) { + await flush(); + } + } + } finally { + clearInterval(flushTimer); + signal?.removeEventListener("abort", onAbort); + await iterator.return?.({ value: undefined, done: true }); + await flush(); + } +} diff --git a/packages/contrail-appview/src/core/realtime/durable-object.ts b/packages/contrail-appview/src/core/realtime/durable-object.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/durable-object.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/in-memory.ts b/packages/contrail-appview/src/core/realtime/in-memory.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/in-memory.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/index.ts b/packages/contrail-appview/src/core/realtime/index.ts new file mode 100644 index 0000000..05e39ec --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/index.ts @@ -0,0 +1,37 @@ +export { registerRealtimeRoutes } from "./router"; +export type { RealtimeRoutesOptions } from "./router"; +export { InMemoryPubSub } from "./in-memory"; +export { DurableObjectPubSub, RealtimePubSubDO } from "./durable-object"; +export type { + DurableObjectId, + DurableObjectNamespace, + DurableObjectStub, + DurableObjectState, +} from "./durable-object"; +export { TicketSigner } from "./ticket"; +export type { TicketPayload } from "./ticket"; +export { wrapWithPublishing } from "./publishing-adapter"; +export { sseResponse } from "./sse"; +export { pumpWebSocket } from "./websocket"; +export type { WebSocketLike } from "./websocket"; +export { mergeAsyncIterables } from "./merge"; +export { resolveTopicForCaller } from "./resolve"; +export type { TopicResolution, TopicResolutionContext, TopicResolutionError } from "./resolve"; +export type { + PubSub, + RealtimeConfig, + RealtimeEvent, + RealtimeEventKind, +} from "./types"; +export { + actorTopic, + collectionTopic, + communityTopic, + parseCommunityTopic, + parseSpaceTopic, + spaceTopic, + isCommunityTopic, + DEFAULT_KEEPALIVE_MS, + DEFAULT_QUEUE_BOUND, + DEFAULT_TICKET_TTL_MS, +} from "./types"; diff --git a/packages/contrail-appview/src/core/realtime/merge.ts b/packages/contrail-appview/src/core/realtime/merge.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/merge.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/publishing-adapter.ts b/packages/contrail-appview/src/core/realtime/publishing-adapter.ts new file mode 100644 index 0000000..c5dc61f --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/publishing-adapter.ts @@ -0,0 +1,157 @@ +/** Decorator that wraps a spaces StorageAdapter and publishes realtime events + * after successful writes. Spaces and community modules stay unaware of + * realtime; the decorator is the only integration seam. */ + +import type { StorageAdapter, SpaceMemberRow } from "../spaces/types"; +import type { PubSub, RealtimeEvent } from "./types"; +import { communityTopic, spaceTopic } from "./types"; + +export interface PublishingAdapterOptions { + /** Optional lookup: given a space's ownerDid, return true if that DID is a + * community in the local `communities` table. When provided, writes also + * publish to `community:<ownerDid>` so subscribers who expanded that alias + * at ticket-mint time receive the event. + * + * The lookup is expected to be cheap (cached in the caller) — the decorator + * calls it on every write. */ + isCommunityDid?: (did: string) => Promise<boolean> | boolean; +} + +export function wrapWithPublishing( + inner: StorageAdapter, + pubsub: PubSub, + opts: PublishingAdapterOptions = {} +): StorageAdapter { + const publishSpaceAndCommunity = async ( + spaceUri: string, + ownerDid: string | null, + build: (topic: string) => RealtimeEvent + ): Promise<void> => { + await pubsub.publish(build(spaceTopic(spaceUri))); + if (ownerDid && opts.isCommunityDid && (await opts.isCommunityDid(ownerDid))) { + await pubsub.publish(build(communityTopic(ownerDid))); + } + }; + + const ownerOf = async (spaceUri: string): Promise<string | null> => { + const s = await inner.getSpace(spaceUri); + return s?.ownerDid ?? null; + }; + + const wrapped: StorageAdapter = { + ...inner, + createSpace: inner.createSpace.bind(inner), + getSpace: inner.getSpace.bind(inner), + listSpaces: inner.listSpaces.bind(inner), + deleteSpace: inner.deleteSpace.bind(inner), + updateSpaceAppPolicy: inner.updateSpaceAppPolicy.bind(inner), + getMember: inner.getMember.bind(inner), + listMembers: inner.listMembers.bind(inner), + createInvite: inner.createInvite.bind(inner), + listInvites: inner.listInvites.bind(inner), + revokeInvite: inner.revokeInvite.bind(inner), + getInvite: inner.getInvite.bind(inner), + redeemInvite: inner.redeemInvite.bind(inner), + enroll: inner.enroll.bind(inner), + getEnrollment: inner.getEnrollment.bind(inner), + listEnrollments: inner.listEnrollments.bind(inner), + removeEnrollment: inner.removeEnrollment.bind(inner), + getRecord: inner.getRecord.bind(inner), + listRecords: inner.listRecords.bind(inner), + listCollections: inner.listCollections.bind(inner), + putBlobMeta: inner.putBlobMeta.bind(inner), + getBlobMeta: inner.getBlobMeta.bind(inner), + listBlobMeta: inner.listBlobMeta.bind(inner), + deleteBlobMeta: inner.deleteBlobMeta.bind(inner), + findOrphanBlobs: inner.findOrphanBlobs.bind(inner), + + async addMember(spaceUri, did, addedBy) { + await inner.addMember(spaceUri, did, addedBy); + const owner = await ownerOf(spaceUri); + const now = Date.now(); + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.added", + payload: { space: spaceUri, did }, + ts: now, + })); + }, + + async removeMember(spaceUri, did) { + await inner.removeMember(spaceUri, did); + const owner = await ownerOf(spaceUri); + const now = Date.now(); + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.removed", + payload: { space: spaceUri, did }, + ts: now, + })); + }, + + async applyMembershipDiff(spaceUri, adds, removes, addedBy) { + await inner.applyMembershipDiff(spaceUri, adds, removes, addedBy); + if (adds.length === 0 && removes.length === 0) return; + const owner = await ownerOf(spaceUri); + const now = Date.now(); + for (const did of adds) { + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.added", + payload: { space: spaceUri, did }, + ts: now, + })); + } + for (const did of removes) { + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "member.removed", + payload: { space: spaceUri, did }, + ts: now, + })); + } + }, + + async putRecord(record) { + await inner.putRecord(record); + const owner = await ownerOf(record.spaceUri); + const now = Date.now(); + // Space records use ms timestamps; listRecords surface uses microseconds + // (time_us). Convert here so subscribers can render a row identically. + const time_us = record.createdAt * 1000; + const uri = `at://${record.authorDid}/${record.collection}/${record.rkey}`; + await publishSpaceAndCommunity(record.spaceUri, owner, (topic) => ({ + topic, + kind: "record.created", + payload: { + uri, + did: record.authorDid, + collection: record.collection, + rkey: record.rkey, + cid: record.cid, + record: record.record, + time_us, + space: record.spaceUri, + }, + ts: now, + })); + }, + + async deleteRecord(spaceUri, collection, authorDid, rkey) { + await inner.deleteRecord(spaceUri, collection, authorDid, rkey); + const owner = await ownerOf(spaceUri); + const now = Date.now(); + const uri = `at://${authorDid}/${collection}/${rkey}`; + await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ + topic, + kind: "record.deleted", + payload: { uri, did: authorDid, collection, rkey, space: spaceUri }, + ts: now, + })); + }, + }; + return wrapped; +} + +// Keep this import hint for types that downstream code might pull from here. +export type { SpaceMemberRow }; diff --git a/packages/contrail-appview/src/core/realtime/query-filter.ts b/packages/contrail-appview/src/core/realtime/query-filter.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/query-filter.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/resolve.ts b/packages/contrail-appview/src/core/realtime/resolve.ts new file mode 100644 index 0000000..c16abe5 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/resolve.ts @@ -0,0 +1,92 @@ +/** Resolve a raw topic request (as given by a caller) to the concrete set of + * delivery topics they are authorized to subscribe to. + * + * Rules (v1): + * - `space:<uri>` → allowed iff caller is owner or a member of the space. + * - `community:<did>` → expanded to `space:<uri>` for every space in the + * community reachable by the caller (direct grants + * or via delegation). `resolveReachableSpaces` + * already has exactly this semantics. + * - `actor:<did>` → self-only in v1. + * - `collection:<nsid>` → rejected unless the deployment opts in + * (not yet implemented). */ + +import type { StorageAdapter } from "../spaces/types"; +import type { CommunityProbe } from "../community-integration"; +import { spaceTopic, parseCommunityTopic, parseSpaceTopic } from "./types"; + +export interface TopicResolutionContext { + /** May be null when the deployment has no spaces module — in that case + * `space:` and `community:` topics are NotSupported. Public topics + * (`collection:`, `actor:`) still resolve. */ + spaces: StorageAdapter | null; + /** May be null if the community module is not enabled. */ + community: CommunityProbe | null; +} + +export interface TopicResolution { + ok: true; + topics: string[]; +} + +export interface TopicResolutionError { + ok: false; + error: "Forbidden" | "InvalidRequest" | "NotFound" | "NotSupported"; + reason: string; +} + +export async function resolveTopicForCaller( + rawTopic: string, + callerDid: string, + ctx: TopicResolutionContext +): Promise<TopicResolution | TopicResolutionError> { + // space:<uri> + const spaceUri = parseSpaceTopic(rawTopic); + if (spaceUri) { + if (!ctx.spaces) { + return { ok: false, error: "NotSupported", reason: "spaces-module-disabled" }; + } + const space = await ctx.spaces.getSpace(spaceUri); + if (!space) return { ok: false, error: "NotFound", reason: "space-not-found" }; + if (space.ownerDid === callerDid) return { ok: true, topics: [rawTopic] }; + const member = await ctx.spaces.getMember(spaceUri, callerDid); + if (!member) return { ok: false, error: "Forbidden", reason: "not-member" }; + return { ok: true, topics: [rawTopic] }; + } + + // community:<did> + const communityDid = parseCommunityTopic(rawTopic); + if (communityDid) { + if (!ctx.community || !ctx.spaces) { + return { ok: false, error: "NotSupported", reason: "community-module-disabled" }; + } + const row = await ctx.community.getCommunity(communityDid); + if (!row) return { ok: false, error: "NotFound", reason: "community-not-found" }; + const reachable = await ctx.community.resolveReachableSpaces(callerDid); + // Filter to spaces owned by THIS community — reachable may include spaces + // from other communities via cross-community delegation. + const ownedList = await ctx.spaces.listSpaces({ ownerDid: communityDid, limit: 1000 }); + const owned: Set<string> = new Set(ownedList.spaces.map((s) => s.uri)); + const topics: string[] = []; + for (const uri of reachable) { + if (owned.has(uri)) topics.push(spaceTopic(uri)); + } + if (topics.length === 0) { + return { ok: false, error: "Forbidden", reason: "no-reachable-spaces-in-community" }; + } + return { ok: true, topics }; + } + + // actor:<did> — public stream of records authored by this DID. + // Any caller can subscribe (parallels listRecords with an `actor` filter). + if (rawTopic.startsWith("actor:")) { + return { ok: true, topics: [rawTopic] }; + } + + // collection:<nsid> — public firehose for this collection. + if (rawTopic.startsWith("collection:")) { + return { ok: true, topics: [rawTopic] }; + } + + return { ok: false, error: "InvalidRequest", reason: "unknown-topic" }; +} diff --git a/packages/contrail-appview/src/core/realtime/router.ts b/packages/contrail-appview/src/core/realtime/router.ts new file mode 100644 index 0000000..d10e45e --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/router.ts @@ -0,0 +1,255 @@ +/** Realtime XRPC routes: ticket mint + subscribe (SSE | WS). */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { ContrailConfig } from "../types"; +import type { ServiceAuth } from "../spaces/auth"; +import type { StorageAdapter } from "../spaces/types"; +import type { CommunityProbe } from "../community-integration"; +import { InMemoryPubSub } from "./in-memory"; +import { TicketSigner } from "./ticket"; +import { sseResponse } from "./sse"; +import { pumpWebSocket, type WebSocketLike } from "./websocket"; +import { mergeAsyncIterables } from "./merge"; +import { resolveTopicForCaller } from "./resolve"; +import type { PubSub, RealtimeEvent } from "./types"; +import { DEFAULT_TICKET_TTL_MS, DEFAULT_KEEPALIVE_MS } from "./types"; + +export interface RealtimeRoutesOptions { + /** Auth middleware for `<ns>.realtime.ticket` and for JWT-based bot + * subscriptions to private topics. Null when no JWT verifier is available + * (deployments without a spaces config) — in that case, private-topic + * subscribe paths return NotSupported and public topics still work without + * auth. */ + authMiddleware: MiddlewareHandler | null; + pubsub?: PubSub; +} + +/** Public topics: subscribable without any auth. Mirrors listRecords + * semantics — no JWT means "public records only". */ +function isPublicTopic(topic: string): boolean { + return topic.startsWith("collection:") || topic.startsWith("actor:"); +} + +/** WebSocketPair exists on Cloudflare Workers; on Node/Bun it's absent. + * When absent, a platform-provided WebSocket accept hook is used instead. */ +interface WebSocketPairCtor { + new (): { 0: WebSocketLike & { accept?: () => void }; 1: WebSocketLike & { accept?: () => void } }; +} + +export function registerRealtimeRoutes( + app: Hono, + config: ContrailConfig, + spaces: StorageAdapter | null, + community: CommunityProbe | null, + options: RealtimeRoutesOptions +): void { + const cfg = config.realtime; + if (!cfg) return; + + const pubsub: PubSub = options.pubsub ?? cfg.pubsub ?? new InMemoryPubSub({ queueBound: cfg.queueBound }); + const signer = new TicketSigner(cfg.ticketSecret); + const ticketTtl = cfg.ticketTtlMs ?? DEFAULT_TICKET_TTL_MS; + const keepaliveMs = cfg.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + + const NS = `${config.namespace}.realtime`; + + // POST /<ns>.realtime.ticket — { topic } → { ticket, topics, expiresAt } + // Ticket-minting exists so browsers (which can't set Authorization on + // EventSource) can subscribe to *private* topics. Public topics + // (collection:, actor:) don't need tickets — subscribe with `?topic=` directly. + if (options.authMiddleware) { + const authMw = options.authMiddleware; + app.post(`/xrpc/${NS}.ticket`, authMw, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { topic?: string } | null; + if (!body?.topic) { + return c.json({ error: "InvalidRequest", message: "topic required" }, 400); + } + const resolved = await resolveTopicForCaller(body.topic, sa.issuer, { spaces, community }); + if (!resolved.ok) { + const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; + return c.json({ error: resolved.error, reason: resolved.reason }, status); + } + const ticket = await signer.sign({ + topics: resolved.topics, + did: sa.issuer, + ttlMs: ticketTtl, + }); + return c.json({ + ticket, + topics: resolved.topics, + expiresAt: Date.now() + ticketTtl, + }); + }); + } + + // GET /<ns>.realtime.subscribe — SSE or WS. + // + // Three access paths, all land on the same stream: + // - `?topic=collection:<nsid>` or `?topic=actor:<did>` — *public*, no auth. + // Mirrors listRecords semantics. + // - `?ticket=<jwt>` — presented by browsers, minted via `.ticket` after + // a JWT-authenticated call. Only used for private topics. + // - `Authorization: Bearer <jwt>` + `?topic=space:<uri>` — server-side + // bots can skip the ticket dance and go straight to subscribe. + app.get(`/xrpc/${NS}.subscribe`, async (c) => { + const url = new URL(c.req.url); + const ticketParam = url.searchParams.get("ticket"); + const collectionFilter = url.searchParams.get("collection"); + const topicParam = url.searchParams.get("topic"); + + let callerDid: string | null = null; + let topics: string[]; + + if (ticketParam) { + const payload = await signer.verify(ticketParam); + if (!payload) { + return c.json({ error: "AuthRequired", reason: "invalid-or-expired-ticket" }, 401); + } + callerDid = payload.did; + topics = payload.topics; + // Optional: narrow to topics the query explicitly requests. + if (topicParam) { + if (!payload.topics.includes(topicParam)) { + return c.json({ error: "Forbidden", reason: "topic-not-in-ticket" }, 403); + } + topics = [topicParam]; + } + } else if (topicParam && isPublicTopic(topicParam)) { + // Public subscribe — no auth required. Jetstream ingestion publishes + // record events to collection:/actor: topics directly. + topics = [topicParam]; + } else { + // JWT path for private-topic bots. If no auth middleware is available + // (deployment has no spaces config), private topics aren't offered. + if (!options.authMiddleware) { + return c.json( + { + error: "InvalidRequest", + reason: "private-topic-without-auth", + message: + "Subscribing to space:/community: topics requires a JWT verifier; only public topics (collection:, actor:) are available on this deployment.", + }, + 400 + ); + } + let authed = false; + await options.authMiddleware(c, async () => { + authed = true; + }); + if (!authed) return c.res; // middleware already responded with 401 + const sa = getAuth(c); + callerDid = sa.issuer; + if (!topicParam) { + return c.json({ error: "InvalidRequest", message: "topic required" }, 400); + } + const resolved = await resolveTopicForCaller(topicParam, callerDid, { spaces, community }); + if (!resolved.ok) { + const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; + return c.json({ error: resolved.error, reason: resolved.reason }, status); + } + topics = resolved.topics; + } + + if (topics.length === 0) { + return c.json({ error: "InvalidRequest", reason: "no-topics" }, 400); + } + + // Build the merged iterable, with an inline filter that closes the stream + // on a matching `member.removed` event (self-kick on revocation). + const ac = new AbortController(); + const signals: AbortSignal[] = [ac.signal]; + const reqSignal = c.req.raw.signal; + if (reqSignal) signals.push(reqSignal); + const combined = anySignal(signals); + + const sources = topics.map((t) => pubsub.subscribe(t, combined)); + const merged = withSelfKickAndFilter( + mergeAsyncIterables(sources, combined), + callerDid, + collectionFilter, + ac + ); + + // Content negotiation: Upgrade: websocket → WS, else SSE. + if (c.req.header("Upgrade")?.toLowerCase() === "websocket") { + const Pair = (globalThis as unknown as { WebSocketPair?: WebSocketPairCtor }) + .WebSocketPair; + if (!Pair) { + return c.json( + { error: "NotSupported", reason: "websockets-require-worker-or-ws-adapter" }, + 426 + ); + } + const pair = new Pair(); + const clientWs = pair[0]; + const serverWs = pair[1]; + serverWs.accept?.(); + // Pump in the background; don't await. + void pumpWebSocket(serverWs, merged, combined, { keepaliveMs }); + return new Response(null, { + status: 101, + // Hono/undici-compat: some runtimes honor `webSocket` on the init. + // @ts-expect-error - Workers-specific init field + webSocket: clientWs, + }); + } + + return sseResponse(merged, combined, { keepaliveMs }); + }); +} + +// ============================================================================ +// Helpers +// ============================================================================ + +function getAuth(c: Context): ServiceAuth { + const a = c.get("serviceAuth") as ServiceAuth | undefined; + if (!a) throw new Error("service auth not set"); + return a; +} + +/** Merge multiple AbortSignals into one. Aborts when any source aborts. */ +function anySignal(signals: AbortSignal[]): AbortSignal { + const ac = new AbortController(); + for (const s of signals) { + if (s.aborted) { + ac.abort(); + return ac.signal; + } + s.addEventListener("abort", () => ac.abort(), { once: true }); + } + return ac.signal; +} + +/** Wrap an iterable: drop events that don't pass the collection filter (if + * any), and close the outer controller as soon as we see a `member.removed` + * for the caller's own DID. `callerDid` may be null on public subscriptions + * (anonymous) — in that case self-kick is not applicable. */ +function withSelfKickAndFilter( + source: AsyncIterable<RealtimeEvent>, + callerDid: string | null, + collectionFilter: string | null, + ac: AbortController +): AsyncIterable<RealtimeEvent> { + return { + async *[Symbol.asyncIterator]() { + for await (const event of source) { + if (event.kind === "member.removed" && event.payload.did === callerDid) { + // Deliver the kick event so the client sees why, then close. + yield event; + ac.abort(); + return; + } + if ( + collectionFilter && + (event.kind === "record.created" || event.kind === "record.deleted") && + event.payload.collection !== collectionFilter + ) { + continue; + } + yield event; + } + }, + }; +} diff --git a/packages/contrail-appview/src/core/realtime/sse.ts b/packages/contrail-appview/src/core/realtime/sse.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/sse.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/ticket.ts b/packages/contrail-appview/src/core/realtime/ticket.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/ticket.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/types.ts b/packages/contrail-appview/src/core/realtime/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/realtime/websocket.ts b/packages/contrail-appview/src/core/realtime/websocket.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/realtime/websocket.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/refresh.ts b/packages/contrail-appview/src/core/refresh.ts new file mode 100644 index 0000000..7b2fd21 --- /dev/null +++ b/packages/contrail-appview/src/core/refresh.ts @@ -0,0 +1,267 @@ +import type {} from "@atcute/atproto"; +/** + * Fresh refresh: re-walk every known DID's PDS for every configured collection + * and reconcile against what's in our DB. Unlike `backfillPending`, this + * ignores the `backfills` state machine — it's a "check what we might have + * missed" pass, not a resumable bulk load. + * + * Two categories of delta are counted: + * - missing — the PDS has a record we don't + * - staleUpdates — we have the same URI but a different CID, *and* our + * copy's `indexed_at` is older than `ignoreWindowMs` + * + * The ignore window exists because Jetstream can run ~seconds behind the + * PDS; without the window, "in-sync but racy" writes would show up as + * misses every run. Records inside the window are still applied (they + * might be legit updates), just not counted toward stats. + * + * Typical uses: + * - dev: "I ran backfillAll on Monday, haven't touched it for a week, + * how much did jetstream miss?" + * - prod: "we had jetstream outage yesterday, what did we drop?" + */ +import { type Did, type Nsid } from "@atcute/lexicons"; +import { isDid, isNsid } from "@atcute/lexicons/syntax"; + +import type { Client } from "@atcute/client"; +import type { ContrailConfig, Database, IngestEvent } from "./types.js"; +import { applyEvents, lookupExistingRecords } from "./db/records.js"; +import { getClient } from "./client.js"; + +const PAGE_SIZE = 100; +const REQUEST_TIMEOUT_MS = 10_000; + +async function withTimeout<T>(fn: () => Promise<T>, ms: number): Promise<T> { + return Promise.race([ + fn(), + new Promise<never>((_, rej) => + setTimeout(() => rej(new Error(`timeout after ${ms}ms`)), ms) + ), + ]); +} + +export interface CollectionStats { + /** Record exists on PDS but was absent from our DB. */ + missing: number; + /** Record exists in our DB with a different CID than the PDS, and our + * copy was written before the ignore window. */ + staleUpdates: number; + /** Record is present and matches (same CID, or within ignore window). */ + inSync: number; +} + +export interface RefreshProgress { + usersComplete: number; + usersTotal: number; + usersFailed: number; + recordsScanned: number; +} + +export interface RefreshResult { + /** Per-NSID stats. */ + byCollection: Record<string, CollectionStats>; + /** Sum across every NSID. */ + total: CollectionStats; + usersScanned: number; + usersFailed: number; + /** Effective ignore window used for classification, in ms. */ + ignoreWindowMs: number; + /** Wall-clock runtime, in ms. */ + elapsedMs: number; +} + +export interface RefreshOptions { + /** How many DIDs to fan out against in parallel. Default: 50. */ + concurrency?: number; + /** Records whose local `indexed_at` is within this window of `now` are + * still upserted but excluded from `staleUpdates` counts — guards + * against jetstream being briefly behind the PDS. Default: 60_000 ms. */ + ignoreWindowMs?: number; + /** Override which NSIDs to walk. Default: every `config.collections[*].collection`. */ + nsids?: string[]; + /** Optional progress callback (fires per completed DID). */ + onProgress?: (p: RefreshProgress) => void; + /** Max attempts per listRecords request. Default: 3. */ + maxRetries?: number; + /** Per-request timeout in ms. Default: 10000. */ + requestTimeout?: number; +} + +function emptyStats(): CollectionStats { + return { missing: 0, staleUpdates: 0, inSync: 0 }; +} + +export async function refresh( + db: Database, + config: ContrailConfig, + options?: RefreshOptions +): Promise<RefreshResult> { + const concurrency = options?.concurrency ?? 50; + const ignoreWindowMs = options?.ignoreWindowMs ?? 60_000; + const requestTimeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; + const maxRetries = options?.maxRetries ?? 3; + const startedAt = Date.now(); + + // Default to every configured collection NSID. Profiles are already + // included because `resolveConfig` adds them to `config.collections`. + const nsids = + options?.nsids ?? + Object.values(config.collections).map((c) => c.collection); + + const byCollection: Record<string, CollectionStats> = {}; + for (const nsid of nsids) byCollection[nsid] = emptyStats(); + const total: CollectionStats = emptyStats(); + + // Known DIDs = every author we've ever written for. `backfills` is a + // superset (it also includes failed/pending users that we never got + // records from), which is actually what we want — if we tried and + // failed before, we might succeed now. + const didRows = await db + .prepare("SELECT DISTINCT did FROM backfills") + .all<{ did: string }>(); + const dids = (didRows.results ?? []) + .map((r) => r.did) + .filter((d) => isDid(d)); + + const usersTotal = dids.length; + let usersComplete = 0; + let usersFailed = 0; + let recordsScanned = 0; + + const ignoreBeforeUs = (Date.now() - ignoreWindowMs) * 1000; + + const processDid = async (did: string): Promise<void> => { + let client: Client; + try { + client = await withTimeout( + () => getClient(did as Did, db), + requestTimeout + ); + } catch { + usersFailed++; + return; + } + + for (const nsid of nsids) { + if (!isNsid(nsid)) continue; + let cursor: string | undefined; + while (true) { + let pageRecords: Array<{ uri: string; cid: string; value: unknown }>; + let nextCursor: string | undefined; + try { + // Retry listRecords: transient PDS failures are expected during refresh + let attempt = 0; + // eslint-disable-next-line no-constant-condition + while (true) { + try { + const res = await withTimeout( + () => + client.get("com.atproto.repo.listRecords", { + params: { + repo: did as Did, + collection: nsid as Nsid, + limit: PAGE_SIZE, + cursor, + }, + }), + requestTimeout + ); + if (!res.ok) { + // 400s on a collection the user doesn't have are fine; stop + // paging this collection for this user. + pageRecords = []; + nextCursor = undefined; + break; + } + pageRecords = res.data.records; + nextCursor = res.data.cursor ?? undefined; + break; + } catch (err) { + if (attempt >= maxRetries) throw err; + attempt++; + await new Promise((r) => setTimeout(r, 500 * 2 ** attempt)); + } + } + } catch { + // Give up on this collection for this user; keep going. + break; + } + + if (pageRecords.length === 0) break; + + const now = Date.now(); + const events: IngestEvent[] = pageRecords.map((r) => ({ + uri: r.uri, + did, + collection: nsid, + rkey: r.uri.split("/").pop()!, + operation: "create" as const, + cid: r.cid, + record: JSON.stringify(r.value), + time_us: now * 1000, + indexed_at: now * 1000, + })); + + const existing = await lookupExistingRecords( + db, + events.map((e) => ({ uri: e.uri, collection: e.collection })), + false, + config + ); + + for (const ev of events) { + const ex = existing.get(ev.uri); + if (!ex) { + byCollection[nsid].missing++; + total.missing++; + } else if (ex.cid !== ev.cid) { + const inWindow = + ex.indexed_at !== null && ex.indexed_at >= ignoreBeforeUs; + if (inWindow) { + byCollection[nsid].inSync++; + total.inSync++; + } else { + byCollection[nsid].staleUpdates++; + total.staleUpdates++; + } + } else { + byCollection[nsid].inSync++; + total.inSync++; + } + } + + // Upsert everything — even records "inside the ignore window" + // might genuinely have a new CID; we just don't count them as a + // miss-signal. Skip feed fanout since this is a catch-up, not a + // user-visible write. + await applyEvents(db, events, config, { skipFeedFanout: true }); + recordsScanned += events.length; + + cursor = nextCursor; + if (!cursor) break; + } + } + + usersComplete++; + options?.onProgress?.({ + usersComplete, + usersTotal, + usersFailed, + recordsScanned, + }); + }; + + for (let i = 0; i < dids.length; i += concurrency) { + const batch = dids.slice(i, i + concurrency); + await Promise.allSettled(batch.map(processDid)); + } + + return { + byCollection, + total, + usersScanned: usersComplete, + usersFailed, + ignoreWindowMs, + elapsedMs: Date.now() - startedAt, + }; +} diff --git a/packages/contrail-appview/src/core/router/admin.ts b/packages/contrail-appview/src/core/router/admin.ts new file mode 100644 index 0000000..a3b754b --- /dev/null +++ b/packages/contrail-appview/src/core/router/admin.ts @@ -0,0 +1,44 @@ +import type { Hono } from "hono"; +import type { ContrailConfig, Database } from "../types"; +import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../types"; +import { getLastCursor } from "../db"; + +export function registerAdminRoutes( + app: Hono, + db: Database, + config: ContrailConfig +): void { + const ns = config.namespace; + + app.get(`/xrpc/${ns}.getCursor`, async (c) => { + const cursor = await getLastCursor(db); + if (cursor === null) return c.json({ cursor: null }); + + const dateMs = Math.floor(cursor / 1000); + return c.json({ + time_us: cursor, + date: new Date(dateMs).toISOString(), + seconds_ago: Math.floor((Date.now() - dateMs) / 1000), + }); + }); + + app.get(`/xrpc/${ns}.getOverview`, async (c) => { + const collections: { collection: string; records: number; unique_users: number }[] = []; + + for (const short of getCollectionShortNames(config)) { + const table = recordsTableName(short); + const nsid = nsidForShortName(config, short) ?? short; + const row = await db + .prepare(`SELECT COUNT(*) as records, COUNT(DISTINCT did) as unique_users FROM ${table}`) + .first<{ records: number; unique_users: number }>(); + if (row) { + collections.push({ collection: nsid, records: row.records, unique_users: row.unique_users }); + } + } + + return c.json({ + total_records: collections.reduce((sum, col) => sum + col.records, 0), + collections, + }); + }); +} diff --git a/packages/contrail-appview/src/core/router/collection.ts b/packages/contrail-appview/src/core/router/collection.ts new file mode 100644 index 0000000..6d43d00 --- /dev/null +++ b/packages/contrail-appview/src/core/router/collection.ts @@ -0,0 +1,1225 @@ +import type { Context, Hono } from "hono"; +import type { ContrailConfig, ResolvedContrailConfig, Database, RecordRow, QueryableField, RecordSource, RelationConfig } from "../types"; +import { + getCollectionShortNames, + countColumnName, + groupedCountColumnName, + recordsTableName, + nsidForShortName, + getCollectionMethods, +} from "../types"; +import { queryRecords, queryAcrossSources } from "../db"; +import type { SortOption } from "../db/records"; +import { backfillUser } from "../backfill"; +import { resolveHydrates, resolveReferences, parseHydrateParams } from "./hydrate"; +import { resolveProfiles, collectDids } from "./profiles"; +import { resolveActor } from "../identity"; +import type { FormattedRecord } from "./helpers"; +import { formatRecord, parseIntParam, fieldToParam } from "./helpers"; +import { selectAcceptedLabelers } from "../labels/select"; +import { hydrateLabels } from "../labels/hydrate"; +import { verifyServiceAuthRequest, extractInviteToken, checkInviteReadGrant } from "../spaces/auth"; +import { checkAccess } from "../spaces/acl"; +import { hashInviteToken } from "../invite/token"; +import type { SpacesContext } from "."; +import type { Nsid } from "@atcute/lexicons"; +import type { RealtimeEvent } from "../realtime/types"; +import { sseResponse } from "../realtime/sse"; +import { spaceTopic, communityTopic, parseSpaceTopic } from "../realtime/types"; +import type { SubscriberQuerySpec } from "../realtime/durable-object"; +import { DurableObjectPubSub } from "../realtime/durable-object"; +import { TicketSigner, type TicketQuerySpec } from "../realtime/ticket"; +import { resolveTopicForCaller } from "../realtime/resolve"; +import { mergeAsyncIterables } from "../realtime/merge"; +import type { CommunityProbe } from "../community-integration"; +import { getRelationField, getNestedValue } from "../types"; + +/** Scope of a watch stream. + * - `space`: single permissioned space — one `space:<uri>` topic. + * - `actor`: records authored by `actor` across multiple spaces — the + * resolver expanded these to a per-caller subset of space topics (plus + * `actor:<did>` for public records). Events outside `allowedSpaces` + * are filtered out. */ +type WatchScope = + | { kind: "space"; spaceUri: string } + | { + kind: "actor"; + actor: string; + /** Concrete pubsub topics to subscribe to (from resolveTopicForCaller). */ + topics: string[]; + /** Space URIs the caller can see. Events with `space` outside this + * set are dropped. Undefined `space` on an event (public record) + * is allowed only when `actor` topic is in `topics`. */ + allowedSpaces: Set<string>; + }; + +/** Shared implementation of the watchRecords snapshot+live loop. Called by + * both transport branches (SSE and Worker-terminated WS). The caller owns + * the actual socket/stream and provides a `send(kind, data)` closure. */ +async function runQueryStream(opts: { + send: (kind: string, data: unknown) => void; + abort: AbortController; + scope: WatchScope; + callerDid: string | undefined; + params: URLSearchParams; + db: Database; + config: ContrailConfig; + collection: string; + colNsid: string; + pubsub: import("../realtime/types").PubSub; + relations: Record<string, import("../types").RelationConfig>; + references: Record<string, import("../types").ReferenceConfig>; + childCollectionMap: Map< + string, + { relName: string; matchField: string; matchMode: "uri" | "did" } + >; +}): Promise<void> { + const { + send, + abort, + scope, + callerDid, + params, + db, + config, + collection, + colNsid, + pubsub, + relations, + references, + childCollectionMap + } = opts; + + // Predicate: does this event belong in the caller's scope? + const inScope = (space: string | undefined): boolean => { + if (scope.kind === "space") return space === scope.spaceUri; + if (space == null) return false; // actor mode: require space for now (app topic) + return scope.allowedSpaces.has(space); + }; + + const hydrateSpec = parseHydrateParams(params, relations, references); + const trackHydration = Object.keys(hydrateSpec.relations).length > 0; + const parentUris = new Set<string>(); + const parentDids = new Set<string>(); + const childToParent = new Map<string, { parentUri: string; relName: string }>(); + + const primaryUri = (payload: { uri: string }) => payload.uri; + + const handleChildEvent = (event: RealtimeEvent) => { + if (!trackHydration) return; + if (event.kind !== "record.created" && event.kind !== "record.deleted") return; + const meta = childCollectionMap.get(event.payload.collection); + if (!meta) return; + if (!(hydrateSpec.relations as Record<string, number>)[meta.relName]) return; + if (!inScope(event.payload.space)) return; + // Actor mode: additionally require the record's author match our actor + // (the caller might share spaces with other authors — we only surface + // records by the actor under watch). + if (scope.kind === "actor" && event.payload.did !== scope.actor) return; + + if (event.kind === "record.created") { + const matched = getNestedValue(event.payload.record, meta.matchField); + if (matched == null) return; + const parent = + meta.matchMode === "did" + ? parentDids.has(String(matched)) + ? `at://${String(matched)}/${colNsid}/_` + : null + : parentUris.has(String(matched)) + ? String(matched) + : null; + if (!parent) return; + childToParent.set(event.payload.rkey, { + parentUri: parent, + relName: meta.relName + }); + send("hydration.added", { + parentUri: parent, + relation: meta.relName, + child: { + uri: primaryUri(event.payload), + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + value: event.payload.record, + space: event.payload.space + } + }); + } else { + const info = childToParent.get(event.payload.rkey); + if (!info) return; + childToParent.delete(event.payload.rkey); + send("hydration.removed", { + parentUri: info.parentUri, + relation: info.relName, + childRkey: event.payload.rkey, + childDid: event.payload.did + }); + } + }; + + const handleLive = (event: RealtimeEvent) => { + if (abort.signal.aborted) return; + if (event.kind === "member.removed" && event.payload.did === callerDid) { + send("member.removed", event.payload); + abort.abort(); + return; + } + if (event.kind !== "record.created" && event.kind !== "record.deleted") return; + if (!inScope(event.payload.space)) return; + if (scope.kind === "actor" && event.payload.did !== scope.actor) return; + + if (event.payload.collection !== colNsid) { + handleChildEvent(event); + return; + } + + const nowUs = event.ts * 1000; + const uri = primaryUri(event.payload); + if (event.kind === "record.created") { + parentUris.add(uri); + parentDids.add(event.payload.did); + send("record.created", { + record: { + uri, + did: event.payload.did, + rkey: event.payload.rkey, + collection: event.payload.collection, + cid: event.payload.cid, + value: event.payload.record, + time_us: nowUs, + indexed_at: event.ts, + space: event.payload.space + } + }); + } else { + parentUris.delete(uri); + send("record.deleted", { + uri, + did: event.payload.did, + rkey: event.payload.rkey + }); + } + }; + + // Subscribe: one topic for space-scoped, merge across all topics for + // actor-scoped. `mergeAsyncIterables` exists for exactly this case. + let iter: AsyncIterable<RealtimeEvent>; + if (scope.kind === "space") { + iter = pubsub.subscribe(spaceTopic(scope.spaceUri), abort.signal); + } else { + const sources = scope.topics.map((t) => pubsub.subscribe(t, abort.signal)); + iter = mergeAsyncIterables(sources, abort.signal); + } + + const buffered: RealtimeEvent[] = []; + let snapshotDone = false; + + const pump = (async () => { + try { + for await (const event of iter) { + if (abort.signal.aborted) break; + if (!snapshotDone) buffered.push(event); + else handleLive(event); + } + } catch { + /* aborted or errored */ + } + })(); + + try { + send( + "snapshot.start", + scope.kind === "space" + ? { spaceUri: scope.spaceUri, collection: colNsid } + : { actor: scope.actor, collection: colNsid } + ); + const snapshotSpaces = + scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); + const result = await runPipeline(db, config, collection, params, undefined, snapshotSpaces); + for (const record of result.records) { + if (abort.signal.aborted) break; + if (typeof record.uri === "string") parentUris.add(record.uri); + if (typeof record.did === "string") parentDids.add(record.did); + for (const [relName] of Object.entries(hydrateSpec.relations)) { + const hydratedGroups = (record as Record<string, unknown>)[relName]; + if (!hydratedGroups) continue; + const flat: Array<{ rkey?: string }> = Array.isArray(hydratedGroups) + ? (hydratedGroups as Array<{ rkey?: string }>) + : (Object.values(hydratedGroups as Record<string, unknown>).flat() as Array<{ + rkey?: string; + }>); + for (const child of flat) { + if (child?.rkey) { + childToParent.set(child.rkey, { + parentUri: record.uri as string, + relName + }); + } + } + } + send("snapshot.record", { record }); + } + send("snapshot.end", { cursor: result.cursor }); + snapshotDone = true; + for (const event of buffered) handleLive(event); + } catch (err) { + send("error", { + message: err instanceof Error ? err.message : String(err) + }); + abort.abort(); + } + + await pump.catch(() => {}); +} + +export async function runPipeline( + db: Database, + config: ContrailConfig, + collection: string, + params: URLSearchParams, + source?: RecordSource, + spaceUris?: string[], + /** Optional headers from the originating request — used for label + * hydration (`atproto-accept-labelers`). Other entry points pass nothing + * and labels are gated by `?labelers=` / config defaults. */ + headers?: Headers +): Promise<{ records: FormattedRecord[]; cursor?: string; profiles?: any[]; labelersApplied?: string[] }> { + const colConfig = config.collections[collection]; + if (!colConfig) throw new Error(`Unknown collection: ${collection}`); + + const relations = colConfig.relations ?? {}; + const references = colConfig.references ?? {}; + const queryableFields: Record<string, QueryableField> = + (config as ResolvedContrailConfig)._resolved?.queryable[collection] ?? colConfig.queryable ?? {}; + + const limit = parseIntParam(params.get("limit"), 50); + const cursor = params.get("cursor") || undefined; + const actor = params.get("actor") || params.get("did") || undefined; + const wantProfiles = params.get("profiles") === "true"; + + let did: string | undefined; + if (actor) { + const resolved = await resolveActor(db, actor); + if (!resolved) throw new Error("Could not resolve actor"); + did = resolved; + // backfillUser expects the record NSID (for PDS calls), not the short name. + const nsid = nsidForShortName(config, collection) ?? collection; + await backfillUser(db, did, nsid, Date.now() + 3_000, config, { + maxRetries: 0, + requestTimeout: 3_000, + }); + } + + const filters: Record<string, string> = {}; + const rangeFilters: Record<string, { min?: string; max?: string }> = {}; + for (const [field, fieldConfig] of Object.entries(queryableFields)) { + const param = fieldToParam(field); + if (fieldConfig.type === "range") { + const min = params.get(`${param}Min`); + const max = params.get(`${param}Max`); + if (min || max) { + rangeFilters[field] = {}; + if (min) rangeFilters[field].min = min; + if (max) rangeFilters[field].max = max; + } + } else { + const value = params.get(param); + if (value) filters[field] = value; + } + } + + const countFilters: Record<string, number> = {}; + const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; + for (const [relName, rel] of Object.entries(relations)) { + const totalMin = parseIntParam(params.get(`${relName}CountMin`)); + if (totalMin != null) countFilters[rel.collection] = totalMin; + const mapping = relMap[relName]; + if (mapping) { + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + for (const [shortName, fullToken] of Object.entries(mapping.groups)) { + const val = parseIntParam(params.get(`${relName}${capitalize(shortName)}CountMin`)); + if (val != null) countFilters[fullToken] = val; + } + } + } + + let sort: SortOption | undefined; + const sortParam = params.get("sort"); + if (sortParam) { + const orderParam = params.get("order"); + + const fieldEntry = Object.entries(queryableFields).find( + ([field]) => fieldToParam(field) === sortParam + ); + if (fieldEntry) { + const defaultDir = fieldEntry[1].type === "range" ? "desc" : "asc"; + const direction = orderParam === "asc" ? "asc" as const : orderParam === "desc" ? "desc" as const : defaultDir as "asc" | "desc"; + sort = { recordField: fieldEntry[0], direction }; + } else { + const direction = orderParam === "asc" ? "asc" as const : "desc" as const; + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + for (const [relName, rel] of Object.entries(relations)) { + if (sortParam === `${relName}Count`) { + sort = { countType: rel.collection, direction }; + break; + } + const mapping = relMap[relName]; + if (mapping) { + for (const [shortName, fullToken] of Object.entries(mapping.groups)) { + if (sortParam === `${relName}${capitalize(shortName)}Count`) { + sort = { countType: fullToken, direction }; + break; + } + } + if (sort) break; + } + } + } + } + + const search = params.get("search") || undefined; + const spaceUri = params.get("spaceUri") || undefined; + + const queryOpts = { + collection, + did, + limit, + cursor, + filters, + rangeFilters, + countFilters, + sort, + search, + source, + spaceUri, + }; + const result = spaceUris && spaceUris.length > 0 && !spaceUri + ? await queryAcrossSources(db, config, queryOpts, spaceUris) + : await queryRecords(db, config, queryOpts); + + const rows = result.records; + const hydrateRequested = parseHydrateParams(params, relations, references); + const hydrates = await resolveHydrates( + db, + relations, + hydrateRequested.relations, + rows, + config + ); + const refs = await resolveReferences( + db, + references, + hydrateRequested.references, + rows, + config + ); + + const formattedRecords: FormattedRecord[] = rows.map((row) => { + const formatted = formatRecord(row); + flattenCounts(formatted, row.counts, relations); + const h = hydrates[row.uri]; + if (h) { + for (const [relName, groups] of Object.entries(h)) { + formatted[relName] = groups; + } + } + const r = refs[row.uri]; + if (r) { + for (const [refName, record] of Object.entries(r)) { + formatted[refName] = record; + } + } + return formatted; + }); + + const allDids = collectDids(rows, hydrates); + const profileMap = wantProfiles + ? await resolveProfiles(db, config, allDids) + : undefined; + + let labelersApplied: string[] | undefined; + if (config.labels) { + const sel = selectAcceptedLabelers( + headers?.get("atproto-accept-labelers") ?? null, + params.get("labelers"), + config.labels, + ); + if (sel.accepted.length > 0) { + const subjects: string[] = [ + ...formattedRecords.map((r) => r.uri), + ...allDids, + ]; + const cidByUri = new Map<string, string | null>(); + for (const r of formattedRecords) cidByUri.set(r.uri, r.cid); + const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); + for (const fr of formattedRecords) { + const ls = labelsByUri[fr.uri]; + if (ls && ls.length > 0) fr.labels = ls; + } + if (profileMap) { + for (const entries of Object.values(profileMap)) { + for (const entry of entries) { + const ls = labelsByUri[entry.did]; + if (ls && ls.length > 0) entry.labels = ls; + } + } + } + labelersApplied = sel.accepted; + } + } + + return { + records: formattedRecords, + cursor: result.cursor, + ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), + ...(labelersApplied ? { labelersApplied } : {}), + }; +} + +/** Serialize a runPipeline result as JSON, echoing + * `atproto-content-labelers` when labels were applied. The result's + * `labelersApplied` field never appears in the response body — it's a + * side channel for the route to read and turn into a header. */ +function jsonWithLabelers(c: Context, result: { labelersApplied?: string[] } & Record<string, unknown>) { + const { labelersApplied, ...body } = result; + if (labelersApplied && labelersApplied.length > 0) { + c.header("atproto-content-labelers", labelersApplied.join(",")); + } + return c.json(body); +} + +export function registerCollectionRoutes( + app: Hono, + db: Database, + config: ContrailConfig, + spacesCtx?: SpacesContext | null, + options: { + pubsub?: import("../realtime/types").PubSub | null; + community?: CommunityProbe | null; + } = {} +): void { + const ns = config.namespace; + const pubsub = options.pubsub ?? null; + const community = options.community ?? null; + + /** When a per-collection endpoint receives `?spaceUri=...`, verify the JWT, + * resolve membership, run the space ACL, and return the caller DID if allowed. + * Returns null if the spaces subsystem isn't available; the handler should + * then treat the spaceUri as invalid. + * Throws by returning a Response (caller checks via `instanceof Response`). */ + async function gateSpaceAccess( + c: Context, + spaceUri: string, + op: "read" + ): Promise<Response | { callerDid?: string; clientId?: string; viaInviteToken?: boolean }> { + if (!spacesCtx) { + return c.json( + { error: "InvalidRequest", message: "spaces not configured on this service" }, + 501 + ); + } + + // Read-token path: anonymous bearer access via `?inviteToken=...` (or + // `Authorization: Bearer atmo-invite:<token>`). Token must exist, be + // unexpired/unrevoked, scoped to this space, and have a kind that grants + // read (`read` or `read-join`). Token kind cannot grant write — caller must + // separately redeem to become a member for any non-read op. + if (op === "read") { + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant( + spacesCtx.adapter, + rawToken, + spaceUri, + hashInviteToken + ); + if (ok) { + const space = await spacesCtx.adapter.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + return { viaInviteToken: true }; + } + return c.json( + { error: "Forbidden", reason: "invalid-invite-token" }, + 403 + ); + } + } + + const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; + const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); + if (!auth) { + return c.json( + { error: "AuthRequired", message: "spaceUri requires a valid service-auth JWT or read-grant invite token" }, + 401 + ); + } + const space = await spacesCtx.adapter.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await spacesCtx.adapter.getMember(spaceUri, auth.issuer); + const result = checkAccess({ + op, + space, + callerDid: auth.issuer, + member, + clientId: auth.clientId, + }); + if (!result.allow) { + return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + return { callerDid: auth.issuer, clientId: auth.clientId }; + } + + for (const collection of getCollectionShortNames(config)) { + const colConfig = config.collections[collection]; + const methods = getCollectionMethods(colConfig); + + if (methods.includes("listRecords")) { + app.get(`/xrpc/${ns}.${collection}.listRecords`, async (c) => { + const params = new URL(c.req.url).searchParams; + const spaceUri = params.get("spaceUri") || undefined; + + if (spaceUri) { + const gated = await gateSpaceAccess(c, spaceUri, "read"); + if (gated instanceof Response) return gated; + // Route through runPipeline with a single-element space list so the + // full filter / sort / hydrate / reference surface works on per-space + // queries too, not just on the cross-space union path. + try { + const result = await runPipeline(db, config, collection, params, undefined, [spaceUri], c.req.raw.headers); + return jsonWithLabelers(c, result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + } + + // Union path: when the caller is authenticated, fold in records from + // spaces they're a member of. Anonymous callers just get public results. + let spaceUris: string[] | undefined; + const hasAuthHeader = !!c.req.header("Authorization"); + if (spacesCtx) { + const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; + const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); + if (auth) { + const { spaces } = await spacesCtx.adapter.listSpaces({ + memberDid: auth.issuer, + limit: 200, + }); + spaceUris = spaces.map((s) => s.uri); + } else if (hasAuthHeader) { + // Had an auth header but it was invalid — reject rather than + // silently downgrading to public results. + return c.json( + { error: "AuthRequired", message: "invalid service-auth JWT" }, + 401 + ); + } + } + + try { + const result = await runPipeline(db, config, collection, params, undefined, spaceUris, c.req.raw.headers); + return jsonWithLabelers(c, result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + }); + + // Streaming variant — same query shape, SSE'd forever. Opted in via the + // presence of the realtime module; no explicit method config needed. + if (pubsub && spacesCtx) { + const colNsid = colConfig.collection; + const relations = colConfig.relations ?? {}; + const references = colConfig.references ?? {}; + // Map child-NSID → { relName, matchField } so we can route child + // events to hydration deltas without re-parsing config per event. + const childCollectionMap = new Map< + string, + { relName: string; matchField: string; matchMode: "uri" | "did" } + >(); + for (const [relName, rel] of Object.entries(relations)) { + const childNsid = nsidForShortName(config, rel.collection) ?? rel.collection; + childCollectionMap.set(childNsid, { + relName, + matchField: getRelationField(rel), + matchMode: rel.match ?? "uri" + }); + } + + // TicketSigner for watch-scoped tickets. Minted on `mode=ws` handshake + // so the subsequent WS upgrade can auth with just `?ticket=...` (no + // cookie or JWT needed — enables cross-origin + stateless clients). + const ticketSigner = config.realtime?.ticketSecret + ? new TicketSigner(config.realtime.ticketSecret) + : null; + const ticketTtl = config.realtime?.ticketTtlMs ?? 120_000; + + app.get(`/xrpc/${ns}.${collection}.watchRecords`, async (c) => { + const params = new URL(c.req.url).searchParams; + const spaceUri = params.get("spaceUri"); + const actorParam = params.get("actor"); + + if (!spaceUri && !actorParam) { + return c.json( + { error: "InvalidRequest", message: "spaceUri or actor required" }, + 400 + ); + } + + // Resolve the caller and their scope. Two parallel paths: + // - space-scoped: single `space:<uri>` topic, per-space ACL gate. + // - actor-scoped: caller's reachable spaces in the actor's + // community (v1 only supports community DIDs as the actor). + // Events are delivered via N `space:<uri>` topics and filtered + // to `did === actor`. + let callerDid: string | undefined; + let scope: WatchScope; + let scopeTopics: string[]; // for ticket signing + let ticketSpec: TicketQuerySpec | null = null; + + const providedTicket = params.get("ticket"); + if (providedTicket && ticketSigner) { + const payload = await ticketSigner.verify(providedTicket); + if (payload?.querySpec && payload.querySpec.collection === colNsid) { + const ts = payload.querySpec; + if (spaceUri && ts.spaceUri === spaceUri) { + if (payload.topics.includes(spaceTopic(spaceUri))) { + callerDid = payload.did; + ticketSpec = { + collection: ts.collection, + spaceUri: ts.spaceUri, + ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) + }; + } + } else if (actorParam && ts.actor === actorParam) { + callerDid = payload.did; + ticketSpec = { + collection: ts.collection, + actor: ts.actor, + ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) + }; + } + } + } + + const hydrateSpec = parseHydrateParams(params, relations, references); + const hydrateForSpec = Object.keys(hydrateSpec.relations).length > 0 + ? Object.fromEntries( + Object.entries(hydrateSpec.relations).map(([relName]) => { + const rel = relations[relName]!; + const childNsid = + nsidForShortName(config, rel.collection) ?? rel.collection; + return [ + relName, + { childCollection: childNsid, matchField: getRelationField(rel) } + ]; + }) + ) + : undefined; + + if (spaceUri) { + if (!ticketSpec) { + const gated = await gateSpaceAccess(c, spaceUri, "read"); + if (gated instanceof Response) return gated; + callerDid = "callerDid" in gated ? gated.callerDid : undefined; + } + scope = { kind: "space", spaceUri }; + scopeTopics = [spaceTopic(spaceUri)]; + } else { + // Actor-scoped path — v1 only supports community DIDs. + const actor = actorParam!; + if (!community || !spacesCtx) { + return c.json( + { error: "NotSupported", reason: "community-module-disabled" }, + 400 + ); + } + const isCommunity = !!(await community.getCommunity(actor)); + if (!isCommunity) { + return c.json( + { error: "InvalidRequest", reason: "actor-must-be-community-did", message: "cross-space watch currently only supports community DIDs as actor" }, + 400 + ); + } + + if (!ticketSpec) { + // Verify the caller via the same JWT/in-process path used for + // per-space queries, then resolve the community topic to the + // caller's accessible space topics. + const nsidLxm = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; + const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsidLxm); + if (!auth) { + return c.json( + { error: "AuthRequired", message: "service-auth JWT or in-process principal required" }, + 401 + ); + } + callerDid = auth.issuer; + } + const resolved = await resolveTopicForCaller(communityTopic(actor), callerDid!, { + spaces: spacesCtx.adapter, + community + }); + if (!resolved.ok) { + const status = + resolved.error === "NotFound" ? 404 : + resolved.error === "Forbidden" ? 403 : 400; + return c.json({ error: resolved.error, reason: resolved.reason }, status); + } + const allowedSpaces = new Set<string>(); + for (const t of resolved.topics) { + const uri = parseSpaceTopic(t); + if (uri) allowedSpaces.add(uri); + } + scope = { kind: "actor", actor, topics: resolved.topics, allowedSpaces }; + scopeTopics = resolved.topics; + } + + const querySpec: TicketQuerySpec = ticketSpec ?? { + collection: colNsid, + ...(spaceUri ? { spaceUri } : { actor: actorParam! }), + ...(hydrateForSpec ? { hydrate: hydrateForSpec } : {}) + }; + + // Upgrade-to-WS path — forward directly to the DO with the spec, + // so the DO terminates the socket and hibernates when idle. + // Requires snapshot to be fetched separately (see `mode=ws` JSON + // handshake below) or accepted as lossy-on-connect for a plain WS + // upgrade. + const isUpgrade = c.req.header("Upgrade")?.toLowerCase() === "websocket"; + const isWsMode = params.get("mode") === "ws"; + + if (isWsMode && !isUpgrade) { + // Handshake: return snapshot + a ticket the client uses to + // upgrade. Ticket carries the (did, topics, querySpec) signed + // so the WS-upgrade route skips any other auth. + try { + const sinceTs = Date.now(); + const snapshotSpaces = + scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); + const result = await runPipeline( + db, + config, + collection, + params, + undefined, + snapshotSpaces, + c.req.raw.headers + ); + let ticket: string | undefined; + if (ticketSigner && callerDid) { + ticket = await ticketSigner.sign({ + topics: scopeTopics, + did: callerDid, + ttlMs: ticketTtl, + querySpec + }); + } + const wsUrl = (() => { + const u = new URL(c.req.url); + u.searchParams.delete("mode"); + if (ticket) u.searchParams.set("ticket", ticket); + u.searchParams.set("sinceTs", String(sinceTs)); + return u.pathname + u.search; + })(); + return c.json({ + transport: "ws", + snapshot: { records: result.records, cursor: result.cursor }, + querySpec, + ticket, + ticketTtlMs: ticketTtl, + sinceTs, + wsUrl + }); + } catch (err) { + return c.json( + { error: "SnapshotFailed", message: err instanceof Error ? err.message : String(err) }, + 500 + ); + } + } + + if (isUpgrade && pubsub instanceof DurableObjectPubSub && scope.kind === "space") { + // Forward the WS upgrade to the DO. The DO owns the socket from + // here and hibernates when idle. Replays any events buffered + // since the handshake `sinceTs` so the client closes the gap. + // + // Actor-scoped queries fall through to the worker-terminated + // path below — the DO binding is single-topic today; extending + // it to fan out over N topics is future work. + const sinceTsParam = params.get("sinceTs"); + const sinceTs = sinceTsParam ? Number(sinceTsParam) : 0; + return pubsub.forwardSubscribe(spaceTopic(scope.spaceUri), c.req.raw, { + did: callerDid, + querySpec: { + collection: querySpec.collection, + spaceUri: scope.spaceUri, + ...(querySpec.hydrate ? { hydrate: querySpec.hydrate } : {}) + }, + sinceTs: Number.isFinite(sinceTs) ? sinceTs : 0 + }); + } + + const ac = new AbortController(); + const reqSignal = c.req.raw.signal; + if (reqSignal) { + if (reqSignal.aborted) ac.abort(); + else reqSignal.addEventListener("abort", () => ac.abort(), { once: true }); + } + + // Worker-terminated WebSocket — used when pubsub isn't DO-backed + // (dev InMemoryPubSub). Same query-filter loop as SSE; different + // transport. Runs in the same isolate so no cost benefit, but + // matches the prod protocol. + if (isUpgrade) { + const WsPair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; + if (!WsPair) { + return c.json( + { error: "NotSupported", reason: "websockets-require-workers-runtime" }, + 426 + ); + } + const pair = new WsPair(); + const clientWs = pair[0] as WebSocket; + const serverWs = pair[1] as WebSocket & { accept?: () => void }; + serverWs.accept?.(); + + const sendWs = (kind: string, data: unknown) => { + try { + serverWs.send(JSON.stringify({ kind, data })); + } catch { + ac.abort(); + } + }; + serverWs.addEventListener?.("close", () => ac.abort()); + serverWs.addEventListener?.("error", () => ac.abort()); + + void runQueryStream({ + send: sendWs, + abort: ac, + scope, + callerDid, + params, + db, + config, + collection, + colNsid, + pubsub, + relations, + references, + childCollectionMap + }).finally(() => { + try { + serverWs.close(); + } catch { + /* ignore */ + } + }); + + return new Response(null, { + status: 101, + webSocket: clientWs + } as ResponseInit & { webSocket: unknown }); + } + + // SSE fallback. + const encoder = new TextEncoder(); + const stream = new ReadableStream<Uint8Array>({ + start(controller) { + let closed = false; + const close = () => { + if (closed) return; + closed = true; + try { + controller.close(); + } catch { + /* already closed */ + } + }; + ac.signal.addEventListener("abort", close, { once: true }); + + const send = (kind: string, data: unknown) => { + if (closed) return; + try { + controller.enqueue( + encoder.encode(`event: ${kind}\ndata: ${JSON.stringify(data)}\n\n`) + ); + } catch { + close(); + } + }; + + const keepalive = setInterval(() => { + if (closed) return; + try { + controller.enqueue(encoder.encode(`: keepalive\n\n`)); + } catch { + close(); + } + }, 15_000); + ac.signal.addEventListener( + "abort", + () => clearInterval(keepalive), + { once: true } + ); + + void runQueryStream({ + send, + abort: ac, + scope, + callerDid, + params, + db, + config, + collection, + colNsid, + pubsub, + relations, + references, + childCollectionMap + }).finally(() => close()); + }, + cancel() { + ac.abort(); + }, + }); + + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + }, + }); + }); + } + } + + if (!methods.includes("getRecord")) { + // Skip getRecord + custom queries unless listRecords-only was explicitly requested. + for (const [queryName, handler] of Object.entries(colConfig.queries ?? {})) { + app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { + const params = new URL(c.req.url).searchParams; + return handler(db, params, config); + }); + } + continue; + } + + app.get(`/xrpc/${ns}.${collection}.getRecord`, async (c) => { + const uri = c.req.query("uri"); + if (!uri) return c.json({ error: "uri parameter required" }, 400); + + // Spaces path — `?spaceUri=` routes to the per-space store + ACL gate. + const spaceUri = c.req.query("spaceUri") || undefined; + if (spaceUri) { + const gated = await gateSpaceAccess(c, spaceUri, "read"); + if (gated instanceof Response) return gated; + + // Parse author + rkey from the record uri `at://<did>/<collection>/<rkey>` + const m = uri.match(/^at:\/\/([^/]+)\/[^/]+\/([^/]+)$/); + if (!m) return c.json({ error: "InvalidRequest", message: "uri must be at://<did>/<collection>/<rkey>" }, 400); + const authorDid = m[1]; + const rkey = m[2]; + + const nsid = colConfig.collection; + const record = await spacesCtx!.adapter.getRecord(spaceUri, nsid, authorDid, rkey); + if (!record) return c.json({ error: "NotFound" }, 404); + return c.json({ record }); + } + + const relations = colConfig.relations ?? {}; + const references = colConfig.references ?? {}; + const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; + + const table = recordsTableName(collection); + const countCols = getRelationCountColumns(relations, relMap); + const selectCols = `uri, did, rkey, cid, record, time_us, indexed_at${countCols.length > 0 ? ", " + countCols.map(c => c.column).join(", ") : ""}`; + const row = await db + .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri = ?`) + .bind(uri) + .first<any>(); + + if (!row) return c.json({ error: "Record not found" }, 404); + + const nsid = nsidForShortName(config, collection) ?? collection; + const formatted = formatRecord({ ...row, collection: nsid }); + const counts = extractCounts(row, relations); + if (counts) flattenCounts(formatted, counts, relations); + + const params = new URL(c.req.url).searchParams; + const wantProfilesSingle = params.get("profiles") === "true"; + + const hydrateRequested = parseHydrateParams(params, relations, references); + const hydrates = await resolveHydrates( + db, + relations, + hydrateRequested.relations, + [row], + config + ); + const refs = await resolveReferences( + db, + references, + hydrateRequested.references, + [row], + config + ); + const h = hydrates[row.uri]; + if (h) { + for (const [relName, groups] of Object.entries(h)) { + (formatted as any)[relName] = groups; + } + } + const r = refs[row.uri]; + if (r) { + for (const [refName, record] of Object.entries(r)) { + (formatted as any)[refName] = record; + } + } + + const allDids = collectDids([row], hydrates); + const profileMap = wantProfilesSingle + ? await resolveProfiles(db, config, allDids) + : undefined; + + let labelersApplied: string[] | undefined; + if (config.labels) { + const sel = selectAcceptedLabelers( + c.req.raw.headers.get("atproto-accept-labelers"), + params.get("labelers"), + config.labels, + ); + if (sel.accepted.length > 0) { + const subjects: string[] = [row.uri, ...allDids]; + const cidByUri = new Map<string, string | null>([[row.uri, row.cid]]); + const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); + const ls = labelsByUri[row.uri]; + if (ls && ls.length > 0) (formatted as Record<string, unknown>).labels = ls; + if (profileMap) { + for (const entries of Object.values(profileMap)) { + for (const entry of entries) { + const els = labelsByUri[entry.did]; + if (els && els.length > 0) entry.labels = els; + } + } + } + labelersApplied = sel.accepted; + } + } + if (labelersApplied) { + c.header("atproto-content-labelers", labelersApplied.join(",")); + } + + return c.json({ + ...formatted, + ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), + }); + }); + + for (const [queryName, handler] of Object.entries( + colConfig.queries ?? {} + )) { + app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { + const params = new URL(c.req.url).searchParams; + return handler(db, params, config); + }); + } + + for (const [queryName, handler] of Object.entries( + colConfig.pipelineQueries ?? {} + )) { + app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { + const params = new URL(c.req.url).searchParams; + try { + const source = await handler(db, params, config); + const result = await runPipeline(db, config, collection, params, source, undefined, c.req.raw.headers); + return jsonWithLabelers(c, result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + }); + } + } +} + +function getRelationCountColumns( + relations: Record<string, RelationConfig>, + relMap: Record<string, any> +): { column: string }[] { + const cols: { column: string }[] = []; + for (const [relName, rel] of Object.entries(relations)) { + if (rel.count === false) continue; + cols.push({ column: countColumnName(rel.collection) }); + const mapping = relMap[relName]; + if (mapping?.groups) { + for (const groupKey of Object.keys(mapping.groups as Record<string, string>)) { + cols.push({ column: groupedCountColumnName(rel.collection, groupKey) }); + } + } + } + return cols; +} + +function extractCounts( + row: any, + relations: Record<string, any> +): Record<string, number> | undefined { + const counts: Record<string, number> = {}; + + for (const [, rel] of Object.entries(relations)) { + if (rel.count === false) continue; + const totalCol = countColumnName(rel.collection); + const val = row[totalCol]; + if (val != null && val !== 0) counts[rel.collection] = val; + + if (rel.groups) { + for (const [groupKey, fullToken] of Object.entries(rel.groups as Record<string, string>)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + const gval = row[groupCol]; + if (gval != null && gval !== 0) counts[fullToken] = gval; + } + } + } + + return Object.keys(counts).length > 0 ? counts : undefined; +} + +function flattenCounts( + formatted: FormattedRecord, + counts: Record<string, number> | undefined, + relations: Record<string, any> +): void { + if (!counts) return; + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + + const collectionToRelName: Record<string, string> = {}; + const tokenToField: Record<string, string> = {}; + for (const [relName, rel] of Object.entries(relations)) { + collectionToRelName[rel.collection] = relName; + if (rel.groups) { + for (const [shortName, fullToken] of Object.entries(rel.groups as Record<string, string>)) { + tokenToField[fullToken] = `${relName}${capitalize(shortName)}Count`; + } + } + } + + for (const [type, count] of Object.entries(counts)) { + if (collectionToRelName[type]) { + formatted[`${collectionToRelName[type]}Count`] = count; + } else if (tokenToField[type]) { + formatted[tokenToField[type]] = count; + } + } +} diff --git a/packages/contrail-appview/src/core/router/feed.ts b/packages/contrail-appview/src/core/router/feed.ts new file mode 100644 index 0000000..0430292 --- /dev/null +++ b/packages/contrail-appview/src/core/router/feed.ts @@ -0,0 +1,134 @@ +import type { Hono } from "hono"; +import type { ContrailConfig, Database, FeedConfig } from "../types"; +import { getDialect } from "../dialect"; +import { DEFAULT_FEED_MAX_ITEMS, recordsTableName } from "../types"; +import { resolveActor } from "../identity"; +import { backfillUser } from "../backfill"; +import { runPipeline } from "./collection"; + +async function maybeBackfillFeed( + db: Database, + config: ContrailConfig, + actor: string, + feedName: string, + feedConfig: FeedConfig +): Promise<void> { + const status = await db + .prepare("SELECT completed FROM feed_backfills WHERE actor = ? AND feed = ?") + .bind(actor, feedName) + .first<{ completed: number }>(); + + if (status?.completed) return; + + // Ensure the user's follow records are backfilled first + await backfillUser(db, actor, feedConfig.follow, Date.now() + 3_000, config, { + maxRetries: 0, + requestTimeout: 3_000, + }); + + // Mark as in-progress (idempotent) + await db + .prepare( + "INSERT INTO feed_backfills (actor, feed, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" + ) + .bind(actor, feedName) + .run(); + + const maxItems = feedConfig.maxItems ?? DEFAULT_FEED_MAX_ITEMS; + + // Populate feed from existing records by followed users + const followTable = recordsTableName(feedConfig.follow); + for (const targetCol of feedConfig.targets) { + const targetTable = recordsTableName(targetCol); + await db + .prepare( + getDialect(db).insertOrIgnore( + `INSERT INTO feed_items (actor, uri, collection, time_us) + SELECT ?, r.uri, ?, r.time_us + FROM ${targetTable} r + WHERE r.did IN ( + SELECT ${getDialect(db).jsonExtract('f.record', 'subject')} + FROM ${followTable} f + WHERE f.did = ? + ) + ORDER BY r.time_us DESC + LIMIT ?` + ) + ) + .bind(actor, targetCol, actor, maxItems) + .run(); + } + + // Prune oldest items beyond the cap + await db + .prepare( + `DELETE FROM feed_items WHERE actor = ? AND uri NOT IN ( + SELECT uri FROM feed_items WHERE actor = ? ORDER BY time_us DESC LIMIT ? + )` + ) + .bind(actor, actor, maxItems) + .run(); + + await db + .prepare("UPDATE feed_backfills SET completed = 1 WHERE actor = ? AND feed = ?") + .bind(actor, feedName) + .run(); +} + +export function registerFeedRoutes( + app: Hono, + db: Database, + config: ContrailConfig +): void { + if (!config.feeds) return; + + const ns = config.namespace; + + app.get(`/xrpc/${ns}.getFeed`, async (c) => { + const params = new URL(c.req.url).searchParams; + const feedName = params.get("feed"); + const actor = params.get("actor"); + + if (!feedName || !actor) { + return c.json({ error: "feed and actor parameters required" }, 400); + } + + const feedConfig = config.feeds![feedName]; + if (!feedConfig) { + return c.json({ error: "Unknown feed" }, 404); + } + + const did = await resolveActor(db, actor); + if (!did) return c.json({ error: "Could not resolve actor" }, 400); + + await maybeBackfillFeed(db, config, did, feedName, feedConfig); + + const collection = params.get("collection") || feedConfig.targets[0]; + if (!feedConfig.targets.includes(collection)) { + return c.json({ error: "Collection not in feed targets" }, 400); + } + + // Strip feed-specific params so runPipeline doesn't misinterpret them + // (e.g. "actor" in feeds means "whose feed", not "filter by record creator") + const pipelineParams = new URLSearchParams(params); + pipelineParams.delete("feed"); + pipelineParams.delete("actor"); + pipelineParams.delete("collection"); + + const source = { + joins: "JOIN feed_items f ON r.uri = f.uri", + conditions: ["f.actor = ?"], + params: [did], + }; + + try { + const result = await runPipeline(db, config, collection, pipelineParams, source); + return c.json(result); + } catch (e: any) { + if (e.message === "Could not resolve actor") { + return c.json({ error: e.message }, 400); + } + throw e; + } + }); +} diff --git a/packages/contrail-appview/src/core/router/helpers.ts b/packages/contrail-appview/src/core/router/helpers.ts new file mode 100644 index 0000000..7ff372e --- /dev/null +++ b/packages/contrail-appview/src/core/router/helpers.ts @@ -0,0 +1,67 @@ +import type { Database, RecordRow } from "../types"; + +export interface FormattedRecord { + uri: string; + cid: string | null; + value: unknown; + did: string; + collection: string; + rkey: string; + time_us: number; + [key: string]: unknown; +} + +export function formatRecord(row: RecordRow): FormattedRecord { + let value: unknown = null; + if (row.record) { + try { + value = JSON.parse(row.record); + } catch { + value = row.record; + } + } + return { + uri: row.uri, + cid: row.cid, + value, + did: row.did, + collection: row.collection, + rkey: row.rkey, + time_us: row.time_us, + ...(row.space ? { space: row.space } : {}), + }; +} + +export function parseIntParam( + value: string | null | undefined, + defaultValue?: number +): number | undefined { + if (!value) return defaultValue; + const parsed = parseInt(value, 10); + return isNaN(parsed) ? defaultValue : parsed; +} + +export function fieldToParam(field: string): string { + return field.replace(/\.(\w)/g, (_, c) => c.toUpperCase()); +} + +const BATCH_SIZE = 50; + +export async function batchedInQuery<T>( + db: Database, + sql: string, + prefixBindings: (string | number)[], + inValues: string[] +): Promise<T[]> { + const results: T[] = []; + for (let i = 0; i < inValues.length; i += BATCH_SIZE) { + const chunk = inValues.slice(i, i + BATCH_SIZE); + const query = sql.replace("__IN__", chunk.map(() => "?").join(",")); + const rows = await db + .prepare(query) + .bind(...prefixBindings, ...chunk) + .all<T>(); + results.push(...(rows.results ?? [])); + } + return results; +} diff --git a/packages/contrail-appview/src/core/router/hydrate.ts b/packages/contrail-appview/src/core/router/hydrate.ts new file mode 100644 index 0000000..8c1ae63 --- /dev/null +++ b/packages/contrail-appview/src/core/router/hydrate.ts @@ -0,0 +1,223 @@ +import type { RelationConfig, ReferenceConfig, RecordRow, Database, ContrailConfig } from "../types"; +import { getDialect } from "../dialect"; +import { + getNestedValue, + getRelationField, + recordsTableName, + spacesRecordsTableName, + nsidForShortName, +} from "../types"; +import { batchedInQuery, formatRecord } from "./helpers"; + +/** Group rows by their origin: public (undefined key) or a specific spaceUri. */ +function groupBySource<T extends { space?: string }>(rows: T[]): Map<string | undefined, T[]> { + const groups = new Map<string | undefined, T[]>(); + for (const r of rows) { + const key = r.space; + const g = groups.get(key); + if (g) g.push(r); + else groups.set(key, [r]); + } + return groups; +} + +// --- Hydration: embed related records --- + +export function parseHydrateParams( + params: URLSearchParams, + relations: Record<string, RelationConfig>, + references: Record<string, ReferenceConfig> +): { relations: Record<string, number>; references: Set<string> } { + const relHydrates: Record<string, number> = {}; + const refHydrates = new Set<string>(); + const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); + + for (const relName of Object.keys(relations)) { + const val = params.get(`hydrate${capitalize(relName)}`); + if (val) { + const limit = parseInt(val, 10); + if (!isNaN(limit) && limit > 0) { + relHydrates[relName] = Math.min(limit, 50); + } + } + } + + for (const refName of Object.keys(references)) { + const val = params.get(`hydrate${capitalize(refName)}`); + if (val === "true" || val === "1") { + refHydrates.add(refName); + } + } + + return { relations: relHydrates, references: refHydrates }; +} + +// Per-relation hydrate result: array for ungrouped, Record<group, array> for grouped +export type HydrateResult = Record<string, Record<string, any[] | Record<string, any[]>>>; + +export async function resolveHydrates( + db: Database, + relations: Record<string, RelationConfig>, + requested: Record<string, number>, + records: RecordRow[], + config?: ContrailConfig +): Promise<HydrateResult> { + if (Object.keys(requested).length === 0 || records.length === 0) return {}; + + const grouped: Record<string, Record<string, Record<string, any[]>>> = {}; + + const sourceGroups = groupBySource(records); + + for (const [relName, hydrateLimit] of Object.entries(requested)) { + const rel = relations[relName]; + const field = getRelationField(rel); + const matchMode = rel.match ?? "uri"; + + for (const [sourceSpace, sourceRecords] of sourceGroups) { + const matchValues = matchMode === "did" + ? [...new Set(sourceRecords.map((r) => r.did))] + : sourceRecords.map((r) => r.uri); + + if (matchValues.length === 0) continue; + + const groupCount = rel.groupBy ? 10 : 1; + const maxRows = matchValues.length * hydrateLimit * groupCount; + + const table = sourceSpace + ? spacesRecordsTableName(rel.collection) + : recordsTableName(rel.collection); + const where = sourceSpace + ? `space_uri = ? AND ${getDialect(db).jsonExtract('record', field)} IN (__IN__)` + : `${getDialect(db).jsonExtract('record', field)} IN (__IN__)`; + const prefix = sourceSpace ? [sourceSpace] : []; + + const relatedRows = await batchedInQuery<Omit<RecordRow, "collection">>( + db, + `SELECT uri, did, rkey, record, time_us FROM ${table} + WHERE ${where} + ORDER BY time_us DESC + LIMIT ${maxRows}`, + prefix, + matchValues + ); + + for (const row of relatedRows) { + const record = row.record ? JSON.parse(row.record) : null; + const matchedValue = getNestedValue(record, field); + if (!matchedValue) continue; + + const parentUris = matchMode === "did" + ? sourceRecords.filter((r) => r.did === matchedValue).map((r) => r.uri) + : [matchedValue]; + + const groupValue = rel.groupBy + ? String(getNestedValue(record, rel.groupBy) ?? "other") + : "_flat"; + + for (const parentUri of parentUris) { + const targetUri = matchMode === "did" ? parentUri : matchedValue; + + if (!grouped[targetUri]) grouped[targetUri] = {}; + if (!grouped[targetUri][relName]) grouped[targetUri][relName] = {}; + if (!grouped[targetUri][relName][groupValue]) grouped[targetUri][relName][groupValue] = []; + + const group = grouped[targetUri][relName][groupValue]; + if (group.length < hydrateLimit) { + const childNsid = config + ? nsidForShortName(config, rel.collection) ?? rel.collection + : rel.collection; + group.push( + formatRecord({ + ...(row as any), + collection: childNsid, + ...(sourceSpace ? { space: sourceSpace } : {}), + } as RecordRow) + ); + } + } + } + } + } + + const result: HydrateResult = {}; + for (const [uri, rels] of Object.entries(grouped)) { + result[uri] = {}; + for (const [relName, groups] of Object.entries(rels)) { + if (relations[relName].groupBy) { + result[uri][relName] = groups; + } else { + result[uri][relName] = groups["_flat"] ?? []; + } + } + } + + return result; +} + +// --- References: embed records that our records point at --- + +export type ReferenceResult = Record<string, Record<string, any>>; + +export async function resolveReferences( + db: Database, + references: Record<string, ReferenceConfig>, + requested: Set<string>, + records: RecordRow[], + config?: ContrailConfig +): Promise<ReferenceResult> { + if (requested.size === 0 || records.length === 0) return {}; + + const result: ReferenceResult = {}; + + const sourceGroups = groupBySource(records); + + for (const refName of requested) { + const ref = references[refName]; + if (!ref) continue; + + const refNsid = config + ? nsidForShortName(config, ref.collection) ?? ref.collection + : ref.collection; + + for (const [sourceSpace, sourceRecords] of sourceGroups) { + const targetMap = new Map<string, string[]>(); + for (const r of sourceRecords) { + const parsed = r.record ? JSON.parse(r.record) : null; + const targetValue = parsed ? getNestedValue(parsed, ref.field) : null; + if (!targetValue) continue; + if (!targetMap.has(targetValue)) targetMap.set(targetValue, []); + targetMap.get(targetValue)!.push(r.uri); + } + + const targetUris = [...targetMap.keys()]; + if (targetUris.length === 0) continue; + + const table = sourceSpace + ? spacesRecordsTableName(ref.collection) + : recordsTableName(ref.collection); + const where = sourceSpace ? `space_uri = ? AND uri IN (__IN__)` : `uri IN (__IN__)`; + const prefix = sourceSpace ? [sourceSpace] : []; + + const rows = await batchedInQuery<Omit<RecordRow, "collection">>( + db, + `SELECT uri, did, rkey, record, time_us FROM ${table} WHERE ${where}`, + prefix, + targetUris + ); + + for (const row of rows) { + const parentUris = targetMap.get(row.uri) ?? []; + for (const parentUri of parentUris) { + if (!result[parentUri]) result[parentUri] = {}; + result[parentUri][refName] = formatRecord({ + ...(row as any), + collection: refNsid, + ...(sourceSpace ? { space: sourceSpace } : {}), + } as RecordRow); + } + } + } + } + + return result; +} diff --git a/packages/contrail-appview/src/core/router/index.ts b/packages/contrail-appview/src/core/router/index.ts new file mode 100644 index 0000000..0c5e058 --- /dev/null +++ b/packages/contrail-appview/src/core/router/index.ts @@ -0,0 +1,247 @@ +import { Hono } from "hono"; +import { cors } from "hono/cors"; +import type { Database, ContrailConfig } from "../types"; +import { normalizeProfileConfig } from "../types"; +import { registerAdminRoutes } from "./admin"; +import { registerCollectionRoutes } from "./collection"; +import { registerFeedRoutes } from "./feed"; +import { registerNotifyRoute } from "./notify"; +import { registerSpacesRoutes } from "../spaces/router"; +import type { SpacesRoutesOptions } from "../spaces/router"; +import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; +import { HostedAdapter } from "../spaces/adapter"; +import type { StorageAdapter } from "../spaces/types"; +import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import type { CommunityIntegration } from "../community-integration"; +import { registerRealtimeRoutes } from "../realtime/router"; +import type { RealtimeRoutesOptions } from "../realtime/router"; +import { registerInviteRoutes } from "../invite/router"; +import { InMemoryPubSub } from "../realtime/in-memory"; +import { wrapWithPublishing } from "../realtime/publishing-adapter"; +import type { PubSub } from "../realtime/types"; +import { resolveActor } from "../identity"; +import { resolveProfiles } from "./profiles"; +import { backfillUser } from "../backfill"; +import { selectAcceptedLabelers } from "../labels/select"; +import { hydrateLabels } from "../labels/hydrate"; +import type { MiddlewareHandler } from "hono"; + +export interface SpacesContext { + adapter: StorageAdapter; + verifier: ServiceJwtVerifier; +} + +export interface CreateAppOptions { + spaces?: SpacesRoutesOptions; + /** Pre-built community integration. Construct via the community package's + * `createCommunityIntegration({ ... })`. When set, contrail wires + * community whoami extension, invite handler, route registration, etc. + * When omitted, deployment runs without community features. */ + community?: CommunityIntegration | null; + /** Auth middleware override for community routes (rare — mostly for tests). */ + communityAuthMiddleware?: MiddlewareHandler; + realtime?: Partial<RealtimeRoutesOptions>; + /** Separate DB for the spaces tables. Defaults to `db`. */ + spacesDb?: Database; + /** Full spaces context override (escape hatch for tests). */ + spacesCtx?: SpacesContext | null; + /** Lexicon JSONs to serve at `/lexicons` so consumer apps can fetch + + * typegen against this deployment. Emit with `contrail-lex generate` — + * its `lexicons/generated/index.ts` exports the right shape. If omitted, + * the endpoint returns `404`. */ + lexicons?: object[]; +} + +export function createApp( + db: Database, + config: ContrailConfig, + options: CreateAppOptions = {} +): Hono { + const app = new Hono(); + app.use("*", cors()); + + app.get("/", (c) => c.json({ status: "ok" })); + app.get("/health", (c) => c.json({ status: "ok" })); + app.get("/xrpc/_health", (c) => c.json({ status: "ok" })); + + const ns = config.namespace; + + // Lexicon manifest — lets consumer apps fetch every lexicon this + // deployment speaks (generated + pulled + custom) over HTTP and + // typegen clients, without needing a PDS or DNS resolution. Only + // registered when the caller passed bundled lexicons at build time + // via `contrail-lex generate`. + if (options.lexicons && options.lexicons.length > 0) { + const lexicons = options.lexicons; + app.get(`/xrpc/${ns}.lexicons`, (c) => c.json({ lexicons })); + } + + app.get(`/xrpc/${ns}.getProfile`, async (c) => { + const actor = c.req.query("actor"); + if (!actor) return c.json({ error: "actor parameter required" }, 400); + + const did = await resolveActor(db, actor); + if (!did) return c.json({ error: "Could not resolve actor" }, 400); + + // Ensure profile records are backfilled + const profileConfigs = (config.profiles ?? []).map(normalizeProfileConfig); + for (const pc of profileConfigs) { + await backfillUser(db, did, pc.collection, Date.now() + 3_000, config, { + maxRetries: 0, + requestTimeout: 3_000, + }); + } + + const profileMap = await resolveProfiles(db, config, [did]); + const profiles = profileMap[did]; + if (!profiles || profiles.length === 0) return c.json({ error: "Profile not found" }, 404); + + if (config.labels) { + const params = new URL(c.req.url).searchParams; + const sel = selectAcceptedLabelers( + c.req.raw.headers.get("atproto-accept-labelers"), + params.get("labelers"), + config.labels, + ); + if (sel.accepted.length > 0) { + const labelsByUri = await hydrateLabels(db, [did], sel.accepted); + const ls = labelsByUri[did]; + if (ls && ls.length > 0) { + for (const entry of profiles) { + entry.labels = ls; + } + } + c.header("atproto-content-labelers", sel.accepted.join(",")); + } + } + + return c.json({ profiles }); + }); + + // Shared spaces context — verifier + adapter — reused by both the per-collection + // routes (for `?spaceUri=...` dispatch) and the `<ns>.space.*` routes. + // Built when an authority is configured (spaces are gated on the authority, + // not the record host — a record-host-only deployment still needs an + // authority somewhere, just possibly external). + const spacesDb = options.spacesDb ?? db; + let spacesCtx: SpacesContext | null = + options.spacesCtx !== undefined + ? options.spacesCtx + : config.spaces?.authority + ? { + adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), + verifier: buildVerifier(config.spaces.authority), + } + : null; + + // Community is provided as a pre-built integration — contrail core never + // imports from the community package. The integration object is opaque; + // we just pass through its probe / whoamiExtension / inviteHandler / + // registerRoutes hooks at the right wiring points. + const community = options.community ?? null; + + // Realtime pubsub is built whenever realtime is configured — independent of + // spaces. With spaces, the spaces adapter is wrapped so private record/member + // events publish to space:/community: topics. Without spaces, only public + // topics (collection:/actor:) see traffic — those are published from + // applyEvents (jetstream ingestion), not from here. + let realtimePubsub: PubSub | null = null; + if (config.realtime) { + realtimePubsub = + options.realtime?.pubsub ?? config.realtime.pubsub ?? new InMemoryPubSub({ + queueBound: config.realtime.queueBound, + }); + if (spacesCtx) { + const isCommunityDid = community + ? cachedIsCommunityDid(community.probe) + : undefined; + spacesCtx = { + ...spacesCtx, + adapter: wrapWithPublishing(spacesCtx.adapter, realtimePubsub, { isCommunityDid }), + }; + } + } + + registerAdminRoutes(app, db, config); + + registerCollectionRoutes(app, db, config, spacesCtx, { + pubsub: realtimePubsub, + community: community?.probe ?? null, + }); + registerFeedRoutes(app, db, config); + registerNotifyRoute(app, db, config); + + // Spaces routes — get a whoami extension from the community integration + // when one's wired so community-owned spaces get an `accessLevel` field. + const spacesOptions = { + ...options.spaces, + whoamiExtension: + options.spaces?.whoamiExtension ?? community?.whoamiExtension, + }; + registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); + + if (community && spacesCtx) { + // Community routes reuse the spaces service-auth middleware (same JWT verifier). + const authMiddleware = + options.communityAuthMiddleware ?? + options.spaces?.authMiddleware ?? + createServiceAuthMiddleware(spacesCtx.verifier); + community.registerRoutes(app, { authMiddleware }); + } + + if (config.spaces?.authority && spacesCtx) { + // Unified invite surface: one `<ns>.invite.*` family that dispatches on + // space ownership (user-owned → addMember; community-owned → grant via + // the integration's invite handler). + const authMiddleware = + options.spaces?.authMiddleware ?? + createServiceAuthMiddleware(spacesCtx.verifier); + registerInviteRoutes( + app, + config, + spacesCtx.adapter, + community?.inviteHandler ?? null, + { authMiddleware } + ); + } + + if (config.realtime && realtimePubsub) { + // The ticket endpoint still needs a JWT verifier — but that verifier only + // exists when spaces is configured. Without spaces, private-topic ticket + // minting simply isn't offered; public subscriptions (collection:/actor:) + // require no auth and still work. + const authMiddleware = spacesCtx + ? options.realtime?.authMiddleware ?? + options.spaces?.authMiddleware ?? + createServiceAuthMiddleware(spacesCtx.verifier) + : null; + registerRealtimeRoutes( + app, + config, + spacesCtx?.adapter ?? null, + community?.probe ?? null, + { + authMiddleware, + pubsub: realtimePubsub, + } + ); + } + + return app; +} + +function cachedIsCommunityDid( + probe: import("../community-integration").CommunityProbe +): (did: string) => Promise<boolean> { + const TTL = 60_000; + const cache = new Map<string, { value: boolean; expires: number }>(); + return async (did: string) => { + const now = Date.now(); + const hit = cache.get(did); + if (hit && hit.expires > now) return hit.value; + const row = await probe.getCommunity(did); + const value = row != null; + cache.set(did, { value, expires: now + TTL }); + return value; + }; +} diff --git a/packages/contrail-appview/src/core/router/notify.ts b/packages/contrail-appview/src/core/router/notify.ts new file mode 100644 index 0000000..5ff630b --- /dev/null +++ b/packages/contrail-appview/src/core/router/notify.ts @@ -0,0 +1,179 @@ +import type { Hono } from "hono"; +import type { Database, ContrailConfig, IngestEvent } from "../types"; +import { shortNameForNsid } from "../types"; +import { applyEvents, lookupExistingRecords } from "../db/records"; +import { getPDS } from "../client"; +import type { Did } from "@atcute/lexicons"; + +/** Parse an AT URI into its components. */ +export function parseAtUri(uri: string): { did: string; collection: string; rkey: string } | null { + const match = uri.match(/^at:\/\/(did:[^/]+)\/([^/]+)\/([^/]+)$/); + if (!match) return null; + return { did: match[1], collection: match[2], rkey: match[3] }; +} + +/** + * Fetch a single record from the user's PDS. + * Returns the record + cid on success, null if not found. + */ +async function fetchRecordFromPDS( + pds: string, + did: string, + collection: string, + rkey: string +): Promise<{ value: unknown; cid: string } | null> { + const url = new URL(`/xrpc/com.atproto.repo.getRecord`, pds); + url.searchParams.set("repo", did); + url.searchParams.set("collection", collection); + url.searchParams.set("rkey", rkey); + + const res = await fetch(url.toString()); + if (!res.ok) return null; + + const data = (await res.json()) as { value?: unknown; cid?: string }; + if (!data.value || !data.cid) return null; + return { value: data.value, cid: data.cid }; +} + +export interface NotifyResult { + indexed: number; + deleted: number; + errors?: string[]; +} + +/** + * Process notify URIs: fetch from PDS, detect changes, apply events. + * Shared by both the Hono route and the Contrail.notify() method. + */ +export async function processNotifyUris( + db: Database, + config: ContrailConfig, + uris: string[] +): Promise<NotifyResult> { + const events: IngestEvent[] = []; + const errors: string[] = []; + + // Validate and parse all URIs first + const validUris: { uri: string; parsed: { did: string; collection: string; rkey: string } }[] = []; + for (const uri of uris) { + const parsed = parseAtUri(uri); + if (!parsed) { + errors.push(`invalid AT URI: ${uri}`); + continue; + } + // `parsed.collection` is an NSID; look up the matching short name. + if (!shortNameForNsid(config, parsed.collection)) { + errors.push(`collection not tracked: ${parsed.collection}`); + continue; + } + validUris.push({ uri, parsed }); + } + + // Single batch lookup for all existing records (cid + record in one query) + const existing = await lookupExistingRecords( + db, + validUris.map(({ uri, parsed }) => ({ uri, collection: parsed.collection })), + true, + config + ); + + for (const { uri, parsed } of validUris) { + const pds = await getPDS(parsed.did as Did, db); + if (!pds) { + errors.push(`could not resolve PDS for ${parsed.did}`); + continue; + } + + const result = await fetchRecordFromPDS( + pds, + parsed.did, + parsed.collection, + parsed.rkey + ); + + const now = Date.now() * 1000; // microseconds + const existingInfo = existing.get(uri); + + if (result) { + if (existingInfo?.cid === result.cid) { + // Same CID — nothing changed + continue; + } + + events.push({ + uri, + did: parsed.did, + collection: parsed.collection, + rkey: parsed.rkey, + operation: existingInfo ? "update" : "create", + cid: result.cid, + record: JSON.stringify(result.value), + time_us: now, + indexed_at: now, + }); + } else if (existingInfo) { + // Record gone from PDS but exists locally — delete it. + events.push({ + uri, + did: parsed.did, + collection: parsed.collection, + rkey: parsed.rkey, + operation: "delete", + cid: null, + record: existingInfo.record, + time_us: now, + indexed_at: now, + }); + } + } + + if (events.length > 0) { + // Pass pre-fetched existing records so applyEvents skips re-querying + await applyEvents(db, events, config, { existing }); + } + + return { + indexed: events.filter((e) => e.operation === "create" || e.operation === "update").length, + deleted: events.filter((e) => e.operation === "delete").length, + errors: errors.length > 0 ? errors : undefined, + }; +} + +export function registerNotifyRoute( + app: Hono, + db: Database, + config: ContrailConfig +) { + // Endpoint is off by default. Set config.notify to true or a secret string to enable. + if (!config.notify) return; + + const ns = config.namespace; + const secret = typeof config.notify === "string" ? config.notify : null; + + app.post(`/xrpc/${ns}.notifyOfUpdate`, async (c) => { + if (secret) { + const auth = c.req.header("Authorization"); + if (auth !== `Bearer ${secret}`) { + return c.json({ error: "unauthorized" }, 401); + } + } + + const body = await c.req.json<{ uri?: string; uris?: string[] }>().catch(() => null); + const uris: string[] = []; + + if (body?.uris && Array.isArray(body.uris)) { + uris.push(...body.uris); + } else if (body?.uri) { + uris.push(body.uri); + } else { + return c.json({ error: "uri or uris required" }, 400); + } + + if (uris.length > 25) { + return c.json({ error: "max 25 URIs per request" }, 400); + } + + const result = await processNotifyUris(db, config, uris); + return c.json(result); + }); +} diff --git a/packages/contrail-appview/src/core/router/profiles.ts b/packages/contrail-appview/src/core/router/profiles.ts new file mode 100644 index 0000000..5552357 --- /dev/null +++ b/packages/contrail-appview/src/core/router/profiles.ts @@ -0,0 +1,181 @@ +import type { Database, ContrailConfig, RecordRow, ProfileConfig } from "../types"; +import { recordsTableName, normalizeProfileConfig } from "../types"; +import { resolveIdentities } from "../identity"; +import { getPDS } from "../client"; +import type { Did } from "@atcute/lexicons"; +import { batchedInQuery } from "./helpers"; + +export interface ProfileEntry { + did: string; + handle: string | null; + uri?: string; + cid?: string | null; + value?: unknown; + collection?: string; + rkey?: string; + /** Hydrated by the labels module when the caller has accepted-labelers + * active and there are matching labels on this DID. */ + labels?: unknown; +} + +export function collectDids( + records: RecordRow[], + hydrates: Record<string, Record<string, any[] | Record<string, any[]>>> +): string[] { + const dids = new Set(records.map((r) => r.did)); + for (const rels of Object.values(hydrates)) { + for (const value of Object.values(rels)) { + const items = Array.isArray(value) + ? value + : Object.values(value).flat(); + for (const item of items) { + if (item.did) dids.add(item.did); + } + } + } + return [...dids]; +} + +export async function resolveProfiles( + db: Database, + config: ContrailConfig, + dids: string[] +): Promise<Record<string, ProfileEntry[]>> { + if (dids.length === 0 || !config.profiles || config.profiles.length === 0) { + return {}; + } + + const profileConfigs = config.profiles.map(normalizeProfileConfig); + const result: Record<string, ProfileEntry[]> = {}; + + // Batch-lookup profile records for each configured profile collection + for (const pc of profileConfigs) { + const { collection, rkey: configRkey, shortName } = pc; + const rkey = configRkey ?? "self"; + const table = recordsTableName(shortName ?? collection); + const uris = dids.map((did) => `at://${did}/${collection}/${rkey}`); + + const rows = await batchedInQuery<Omit<RecordRow, "collection">>( + db, + `SELECT uri, did, rkey, cid, record FROM ${table} WHERE uri IN (__IN__)`, + [], + uris + ); + + for (const row of rows) { + let value: unknown = null; + if (row.record) { + try { + value = JSON.parse(row.record); + } catch { + value = row.record; + } + } + if (!result[row.did]) result[row.did] = []; + result[row.did].push({ + did: row.did, + handle: null, // filled below + uri: row.uri, + collection, + rkey: row.rkey, + cid: row.cid, + value, + }); + } + } + + // Resolve identities for all DIDs + const identities = await resolveIdentities(db, dids); + + // Fetch missing profile records from PDS on demand + const missingDids = dids.filter((d) => !result[d]); + if (missingDids.length > 0 && profileConfigs.length > 0) { + const fetched = await fetchMissingProfiles(db, config, missingDids); + for (const [did, entries] of Object.entries(fetched)) { + if (!result[did]) result[did] = []; + result[did].push(...entries); + } + } + + // Fill in handles and create entries for DIDs without profile records + for (const did of dids) { + const identity = identities.get(did); + const handle = identity?.handle ?? null; + + if (result[did]) { + for (const entry of result[did]) { + entry.handle = handle; + } + } else { + result[did] = [{ did, handle }]; + } + } + + return result; +} + +/** + * Fetch profile records from PDS for DIDs not yet in the index. + * Fetches in parallel across all configured profile collections, + * indexes the results into D1 for future requests. + */ +async function fetchMissingProfiles( + db: Database, + config: ContrailConfig, + dids: string[] +): Promise<Record<string, ProfileEntry[]>> { + const result: Record<string, ProfileEntry[]> = {}; + const profileConfigs = config.profiles!.map(normalizeProfileConfig); + + await Promise.all( + dids.flatMap((did) => + profileConfigs.map(async (pc) => { + const { collection, rkey: configRkey, shortName } = pc; + const rkey = configRkey ?? "self"; + const table = recordsTableName(shortName ?? collection); + try { + const pds = await getPDS(did as Did, db); + if (!pds) return; + + const url = new URL("/xrpc/com.atproto.repo.getRecord", pds); + url.searchParams.set("repo", did); + url.searchParams.set("collection", collection); + url.searchParams.set("rkey", rkey); + + const res = await fetch(url.toString()); + if (!res.ok) return; + + const data = (await res.json()) as { uri?: string; value?: unknown; cid?: string }; + if (!data.value || !data.cid) return; + + const uri = data.uri ?? `at://${did}/${collection}/${rkey}`; + const record = data.value; + const cid = data.cid; + + // Index into D1 for future requests + await db + .prepare( + `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, indexed_at = excluded.indexed_at` + ) + .bind(uri, did, rkey, cid, JSON.stringify(record), Date.now() * 1000, Date.now()) + .run(); + + if (!result[did]) result[did] = []; + result[did].push({ + did, + handle: null, + uri, + collection, + rkey, + cid, + value: record, + }); + } catch { + // Skip failures silently + } + }) + ) + ); + + return result; +} diff --git a/packages/contrail-appview/src/core/search.ts b/packages/contrail-appview/src/core/search.ts new file mode 100644 index 0000000..a33dc5e --- /dev/null +++ b/packages/contrail-appview/src/core/search.ts @@ -0,0 +1,31 @@ +import type { CollectionConfig } from "./types"; +import { getNestedValue } from "./types"; + +/** + * Resolve which fields are searchable for a collection. + * Returns null if search is disabled or no fields found. + */ +export function getSearchableFields( + collection: string, + colConfig: CollectionConfig +): string[] | null { + if (!Array.isArray(colConfig.searchable)) return null; + return colConfig.searchable.length > 0 ? colConfig.searchable : null; +} + +/** Sanitized FTS table name for a collection. */ +export function ftsTableName(collection: string): string { + return `fts_${collection.replace(/[^a-zA-Z0-9]/g, "_")}`; +} + +/** Extract searchable field values from a record and join them into a single string. */ +export function buildFtsContent(record: unknown, fields: string[]): string | null { + const parts: string[] = []; + for (const field of fields) { + const value = getNestedValue(record, field); + if (typeof value === "string" && value.length > 0) { + parts.push(value); + } + } + return parts.length > 0 ? parts.join(" ") : null; +} diff --git a/packages/contrail-appview/src/core/spaces/acl.ts b/packages/contrail-appview/src/core/spaces/acl.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/acl.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/adapter.ts b/packages/contrail-appview/src/core/spaces/adapter.ts new file mode 100644 index 0000000..50fb33d --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/adapter.ts @@ -0,0 +1,505 @@ +/** Contrail's all-in-one default adapter — extends the authority package's + * {@link HostedAuthorityAdapter} (which owns space lifecycle, member list, + * invites) and adds the record-host methods (records, blobs, enrollment). + * + * Phase 7a step 3 will lift the record-host methods into a separate + * HostedRecordHostAdapter, at which point this class becomes a thin + * composition / re-export. For now we keep both roles in one class so + * consumers can wire a single object that satisfies the full StorageAdapter. */ + +import type { ContrailConfig, RelationConfig, ResolvedContrailConfig } from "../types"; +import { + shortNameForNsid, + spacesRecordsTableName, + countColumnName, + groupedCountColumnName, + getRelationField, + getNestedValue, +} from "../types"; +import { getDialect } from "../dialect"; +import type { + BlobMetaRow, + CollectionCount, + EnrollmentRow, + ListBlobsOptions, + ListBlobsResult, + ListOptions, + ListResult, + StorageAdapter, + StoredRecord, +} from "./types"; +import type { Database } from "../types"; +import { buildRecordUri } from "./uri"; +import { HostedAuthorityAdapter, parseJson, toNum } from "@atmo-dev/contrail-authority"; + +function mapBlobMetaRow(row: any): BlobMetaRow { + return { + spaceUri: row.space_uri, + cid: row.cid, + mimeType: row.mime_type, + size: Number(row.size), + authorDid: row.author_did, + createdAt: toNum(row.created_at), + }; +} + +function mapEnrollmentRow(row: any): EnrollmentRow { + return { + spaceUri: row.space_uri, + authorityDid: row.authority_did, + enrolledAt: toNum(row.enrolled_at), + enrolledBy: row.enrolled_by, + }; +} + +/** Row mapper for per-collection spaces_records_<short> tables. + * `collection` is injected by the caller (known from the table name). */ +function mapRecordRow(row: any, collection: string): StoredRecord { + return { + spaceUri: row.space_uri, + collection, + authorDid: row.did, + rkey: row.rkey, + cid: row.cid ?? null, + record: parseJson<Record<string, unknown>>(row.record) ?? {}, + createdAt: toNum(row.time_us), + }; +} + +export class HostedAdapter extends HostedAuthorityAdapter implements StorageAdapter { + /** Resolve the per-collection spaces table name, or throw if the collection + * isn't configured (and therefore has no table). */ + private tableFor(collection: string): string { + if (!this.config) { + throw new Error( + `HostedAdapter: config not provided; cannot resolve table for collection ${collection}` + ); + } + const short = shortNameForNsid(this.config, collection); + if (!short) { + throw new Error( + `HostedAdapter: collection ${collection} is not configured in this deployment` + ); + } + return spacesRecordsTableName(short); + } + + // ---- Enrollment ---- + + async enroll(input: EnrollmentRow): Promise<void> { + await this.db + .prepare( + `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri) DO UPDATE SET + authority_did = excluded.authority_did, + enrolled_at = excluded.enrolled_at, + enrolled_by = excluded.enrolled_by` + ) + .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) + .run(); + } + + async getEnrollment(spaceUri: string): Promise<EnrollmentRow | null> { + const row = await this.db + .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .first<any>(); + return row ? mapEnrollmentRow(row) : null; + } + + async listEnrollments( + options: { authorityDid?: string; limit?: number } = {} + ): Promise<EnrollmentRow[]> { + const limit = Math.min(options.limit ?? 200, 1000); + if (options.authorityDid) { + const { results } = await this.db + .prepare( + `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` + ) + .bind(options.authorityDid, limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + const { results } = await this.db + .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) + .bind(limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + + async removeEnrollment(spaceUri: string): Promise<void> { + await this.db + .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .run(); + } + + // ---- Records ---- + + async putRecord(record: StoredRecord): Promise<void> { + const table = this.tableFor(record.collection); + const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); + + const childShort = this.config ? shortNameForNsid(this.config, record.collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(record.spaceUri, record.authorDid, record.rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `INSERT INTO ${table} (space_uri, uri, did, rkey, cid, record, time_us, indexed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, did, rkey) DO UPDATE SET + uri = excluded.uri, + cid = excluded.cid, + record = excluded.record, + time_us = excluded.time_us, + indexed_at = excluded.indexed_at` + ) + .bind( + record.spaceUri, + uri, + record.authorDid, + record.rkey, + record.cid, + JSON.stringify(record.record), + record.createdAt, + Date.now() + ) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace( + record.spaceUri, + childShort, + beforeRecord, + record.record, + record.authorDid + ); + } + } + + async getRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<StoredRecord | null> { + const table = this.tableFor(collection); + const row = await this.db + .prepare( + `SELECT * FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .first<any>(); + return row ? mapRecordRow(row, collection) : null; + } + + async listRecords( + spaceUri: string, + collection: string, + options: ListOptions = {} + ): Promise<ListResult> { + const table = this.tableFor(collection); + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("time_us < ?"); + params.push(Number(options.cursor)); + } + + const sql = `SELECT * FROM ${table} + WHERE ${clauses.join(" AND ")} + ORDER BY time_us DESC + LIMIT ?`; + params.push(limit + 1); + + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const records = results.map((r) => mapRecordRow(r, collection)); + let cursor: string | undefined; + if (records.length > limit) { + const next = records.pop()!; + cursor = String(next.createdAt); + } + return { records, cursor }; + } + + async deleteRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<void> { + const table = this.tableFor(collection); + + const childShort = this.config ? shortNameForNsid(this.config, collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(spaceUri, authorDid, rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `DELETE FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace(spaceUri, childShort, beforeRecord, null, authorDid); + } + } + + /** Recompute count columns on parent records in the same space, scoped to the + * targets derived from before/after versions of the written/deleted child record. */ + private async recountParentsForSpace( + spaceUri: string, + childShort: string, + before: Record<string, unknown> | null, + after: Record<string, unknown> | null, + childDid: string + ): Promise<void> { + if (!this.config) return; + const config = this.config; + const resolved = (config as ResolvedContrailConfig)._resolved; + const childTable = spacesRecordsTableName(childShort); + + type Inbound = { parentShort: string; relationName: string; rel: RelationConfig }; + const inbound: Inbound[] = []; + for (const [parentShort, parentCfg] of Object.entries(config.collections)) { + if (parentCfg.allowInSpaces === false) continue; + for (const [relName, rel] of Object.entries(parentCfg.relations ?? {})) { + if (rel.count === false) continue; + if (rel.collection !== childShort) continue; + inbound.push({ parentShort, relationName: relName, rel }); + } + } + if (inbound.length === 0) return; + + // Deduplicate (parent, relation, target) across before/after. + const keyed = new Map<string, { parentShort: string; relationName: string; rel: RelationConfig; target: string }>(); + for (const { parentShort, relationName, rel } of inbound) { + const field = getRelationField(rel); + const collectTarget = (rec: Record<string, unknown> | null) => { + if (!rec) return; + if (rel.match === "did") { + keyed.set(`${parentShort}:${relationName}:${childDid}`, { + parentShort, relationName, rel, target: childDid, + }); + return; + } + const v = getNestedValue(rec, field); + if (typeof v === "string" && v.length > 0) { + keyed.set(`${parentShort}:${relationName}:${v}`, { + parentShort, relationName, rel, target: v, + }); + } + }; + collectTarget(before); + collectTarget(after); + } + if (keyed.size === 0) return; + + const dialect = getDialect(this.db); + const stmts: ReturnType<Database["prepare"]>[] = []; + + for (const { parentShort, relationName, rel, target } of keyed.values()) { + const parentTable = spacesRecordsTableName(parentShort); + const matchColumn = rel.match === "did" ? "did" : "uri"; + const field = getRelationField(rel); + const countExpr = rel.countDistinct + ? `COUNT(DISTINCT ${rel.countDistinct})` + : "COUNT(*)"; + + const setClauses: string[] = []; + const binds: (string | number)[] = []; + + const totalCol = countColumnName(rel.collection); + setClauses.push( + `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ?)` + ); + binds.push(spaceUri, target); + + if (rel.groupBy) { + const mapping = resolved?.relations[parentShort]?.[relationName]; + if (mapping?.groups) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + setClauses.push( + `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ? AND ${dialect.jsonExtract("record", rel.groupBy)} = ?)` + ); + binds.push(spaceUri, target, fullToken); + } + } + } + + binds.push(spaceUri, target); + stmts.push( + this.db + .prepare( + `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE space_uri = ? AND ${matchColumn} = ?` + ) + .bind(...binds) + ); + } + + if (stmts.length > 0) await this.db.batch(stmts); + } + + async listCollections( + spaceUri: string, + options: { byUser?: string } = {} + ): Promise<CollectionCount[]> { + if (!this.config) return []; + const results: CollectionCount[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + const table = spacesRecordsTableName(short); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + try { + const row = await this.db + .prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${clauses.join(" AND ")}`) + .bind(...params) + .first<{ count: number }>(); + const count = Number(row?.count ?? 0); + if (count > 0) results.push({ collection: colConfig.collection, count }); + } catch { + // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip + } + } + return results; + } + + // ---- Blobs ---- + + async putBlobMeta(row: BlobMetaRow): Promise<void> { + const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, cid) DO NOTHING`; + await this.db + .prepare(sql) + .bind(row.spaceUri, row.cid, row.mimeType, row.size, row.authorDid, row.createdAt) + .run(); + } + + async getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null> { + const r = await this.db + .prepare(`SELECT * FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .first<any>(); + return r ? mapBlobMetaRow(r) : null; + } + + async listBlobMeta( + spaceUri: string, + options: ListBlobsOptions = {} + ): Promise<ListBlobsResult> { + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("author_did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("created_at < ?"); + params.push(Number(options.cursor)); + } + const sql = `SELECT * FROM spaces_blobs + WHERE ${clauses.join(" AND ")} + ORDER BY created_at DESC + LIMIT ?`; + params.push(limit + 1); + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const blobs = results.map(mapBlobMetaRow); + let cursor: string | undefined; + if (blobs.length > limit) { + const next = blobs.pop()!; + cursor = String(next.createdAt); + } + return { blobs, cursor }; + } + + async deleteBlobMeta(spaceUri: string, cid: string): Promise<void> { + await this.db + .prepare(`DELETE FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .run(); + } + + async findOrphanBlobs( + spaceUri: string, + cutoff: number, + limit: number + ): Promise<BlobMetaRow[]> { + if (!this.config) return []; + // Gather candidate blobs older than cutoff, then filter out any whose CID + // appears in any record JSON in this space. We use a cheap substring probe + // (LIKE) per collection — false positives are OK because an orphan that + // survives GC just gets collected next cycle; false negatives (deleting + // a referenced blob) would be a bug, and substring search over the full + // CID is safe enough for that. + const { results } = await this.db + .prepare( + `SELECT * FROM spaces_blobs + WHERE space_uri = ? AND created_at < ? + ORDER BY created_at ASC + LIMIT ?` + ) + .bind(spaceUri, cutoff, limit) + .all<any>(); + const candidates = results.map(mapBlobMetaRow); + if (candidates.length === 0) return []; + + const tables: string[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + tables.push(spacesRecordsTableName(short)); + } + + const orphans: BlobMetaRow[] = []; + for (const blob of candidates) { + let referenced = false; + const pattern = `%${blob.cid}%`; + for (const table of tables) { + try { + const row = await this.db + .prepare( + `SELECT 1 FROM ${table} WHERE space_uri = ? AND record LIKE ? LIMIT 1` + ) + .bind(spaceUri, pattern) + .first<any>(); + if (row) { + referenced = true; + break; + } + } catch { + // table missing — ignore + } + } + if (!referenced) orphans.push(blob); + } + return orphans; + } +} diff --git a/packages/contrail-appview/src/core/spaces/auth.ts b/packages/contrail-appview/src/core/spaces/auth.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/auth.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/binding.ts b/packages/contrail-appview/src/core/spaces/binding.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/binding.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/blob-adapter.ts b/packages/contrail-appview/src/core/spaces/blob-adapter.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/blob-adapter.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/blob-gc.ts b/packages/contrail-appview/src/core/spaces/blob-gc.ts new file mode 100644 index 0000000..d523fcd --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/blob-gc.ts @@ -0,0 +1,38 @@ +import type { BlobAdapter } from "./blob-adapter"; +import { blobKey } from "./blob-adapter"; +import type { StorageAdapter } from "./types"; + +export interface BlobGcOptions { + /** Orphan rows created before this timestamp are eligible for deletion. */ + olderThan: number; + /** Maximum number of blobs to delete in this pass. Defaults to 500. */ + batchSize?: number; +} + +export interface BlobGcResult { + deleted: number; + cids: string[]; +} + +/** Delete blob bytes + metadata for any blob older than `olderThan` that + * is not referenced by any record in the space. Safe to run periodically. */ +export async function gcOrphanBlobs( + storage: StorageAdapter, + blobs: BlobAdapter, + spaceUri: string, + options: BlobGcOptions +): Promise<BlobGcResult> { + const batchSize = options.batchSize ?? 500; + const orphans = await storage.findOrphanBlobs(spaceUri, options.olderThan, batchSize); + if (orphans.length === 0) return { deleted: 0, cids: [] }; + + const keys: string[] = []; + for (const row of orphans) { + keys.push(await blobKey(row.spaceUri, row.cid)); + } + await blobs.delete(keys); + for (const row of orphans) { + await storage.deleteBlobMeta(row.spaceUri, row.cid); + } + return { deleted: orphans.length, cids: orphans.map((o) => o.cid) }; +} diff --git a/packages/contrail-appview/src/core/spaces/blob-refs.ts b/packages/contrail-appview/src/core/spaces/blob-refs.ts new file mode 100644 index 0000000..62cf693 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/blob-refs.ts @@ -0,0 +1,27 @@ +/** + * Walk a record JSON and collect every atproto blob ref. + * + * Blob refs look like: + * { "$type": "blob", "ref": { "$link": "<cid>" }, "mimeType": "...", "size": N } + * + * We return the CID strings. + */ +export function collectBlobCids(value: unknown, out: Set<string> = new Set()): Set<string> { + if (value == null) return out; + if (Array.isArray(value)) { + for (const v of value) collectBlobCids(v, out); + return out; + } + if (typeof value !== "object") return out; + + const obj = value as Record<string, unknown>; + if (obj["$type"] === "blob") { + const ref = obj["ref"] as { $link?: unknown } | undefined; + if (ref && typeof ref["$link"] === "string") out.add(ref["$link"]); + // Don't descend — a blob ref's own shape has no nested blobs. + return out; + } + + for (const v of Object.values(obj)) collectBlobCids(v, out); + return out; +} diff --git a/packages/contrail-appview/src/core/spaces/credentials.ts b/packages/contrail-appview/src/core/spaces/credentials.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/credentials.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/in-process.ts b/packages/contrail-appview/src/core/spaces/in-process.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/in-process.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/router.ts b/packages/contrail-appview/src/core/spaces/router.ts new file mode 100644 index 0000000..943f6a9 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/router.ts @@ -0,0 +1,94 @@ +/** Umbrella that wires the authority + record-host route registrations from + * their respective packages onto a single Hono app. The actual route + * handlers live in `@atmo-dev/contrail-authority` and + * `@atmo-dev/contrail-record-host`. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { ContrailConfig, Database } from "../types"; +import { HostedAdapter } from "./adapter"; +import { + buildVerifier, + createBindingCredentialVerifier, + createCompositeBindingResolver, + createEnrollmentBindingResolver, + createLocalBindingResolver, + createLocalKeyResolver, + createServiceAuthMiddleware, +} from "@atmo-dev/contrail-base"; +import type { + CredentialVerifier, + StorageAdapter, + WhoamiExtension, +} from "@atmo-dev/contrail-base"; +import { registerAuthorityRoutes } from "@atmo-dev/contrail-authority"; +import { registerRecordHostRoutes } from "@atmo-dev/contrail-record-host"; + +// Re-export the route-registration functions and WhoamiExtension type so +// existing consumers of `@atmo-dev/contrail` keep their imports working +// without switching to the new packages. +export { registerAuthorityRoutes, registerRecordHostRoutes }; +export type { WhoamiExtension }; + +export interface SpacesRoutesOptions { + /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ + authMiddleware?: MiddlewareHandler; + /** Storage adapter override. Defaults to HostedAdapter(db). */ + adapter?: StorageAdapter; + /** Optional whoami extension; see {@link WhoamiExtension}. */ + whoamiExtension?: WhoamiExtension; + /** Optional credential verifier for the record host. */ + credentialVerifier?: CredentialVerifier; +} + +/** Umbrella registration: wires both the authority and the record-host + * routes against the same adapter. Today's deployments enable both via + * `config.spaces.authority` and `config.spaces.recordHost`. */ +export function registerSpacesRoutes( + app: Hono, + db: Database, + config: ContrailConfig, + options: SpacesRoutesOptions = {}, + ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null +): void { + const spacesConfig = config.spaces; + if (!spacesConfig) return; + const authorityConfig = spacesConfig.authority; + if (!authorityConfig) return; + + const adapter = options.adapter ?? ctx?.adapter ?? new HostedAdapter(db, config); + const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); + const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); + + const localRecordHost = spacesConfig.recordHost ? adapter : null; + registerAuthorityRoutes( + app, + adapter, + authorityConfig, + config, + auth, + options.whoamiExtension, + localRecordHost + ); + + if (spacesConfig.recordHost) { + // Default in-process verifier: enrollment is the canonical binding + // source; Local-binding is a fallback for spaces created but not yet + // enrolled. Caller overrides via `options.credentialVerifier` to + // accept external authorities. + const credentialVerifier = + options.credentialVerifier ?? + (authorityConfig.signing + ? createBindingCredentialVerifier({ + bindings: createCompositeBindingResolver([ + createEnrollmentBindingResolver({ recordHost: adapter }), + createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), + ]), + keys: createLocalKeyResolver({ + authorityDid: authorityConfig.serviceDid, + publicKey: authorityConfig.signing.publicKey, + }), + }) + : undefined); + registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, credentialVerifier); + } +} diff --git a/packages/contrail-appview/src/core/spaces/schema.ts b/packages/contrail-appview/src/core/spaces/schema.ts new file mode 100644 index 0000000..ae33a7c --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/schema.ts @@ -0,0 +1,103 @@ +import type { ContrailConfig, Database } from "../types"; +import type { SqlDialect } from "../dialect"; +import { getDialect } from "../dialect"; +import { + buildCollectionTables, + buildDynamicIndexes, + buildFtsTables, + buildCountColumns, +} from "../db/schema"; + +/** Spaces metadata tables — spaces, members, invites. No per-collection tables. */ +export function buildSpacesBaseSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS spaces ( + uri TEXT PRIMARY KEY, + owner_did TEXT NOT NULL, + type TEXT NOT NULL, + key TEXT NOT NULL, + service_did TEXT NOT NULL, + app_policy_ref TEXT, + app_policy ${dialect.recordColumnType}, + created_at ${dialect.bigintType} NOT NULL, + deleted_at ${dialect.bigintType} + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_owner ON spaces(owner_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_type ON spaces(type)`, + + `CREATE TABLE IF NOT EXISTS spaces_members ( + space_uri TEXT NOT NULL, + did TEXT NOT NULL, + added_at ${dialect.bigintType} NOT NULL, + added_by TEXT, + PRIMARY KEY (space_uri, did) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_members_did ON spaces_members(did)`, + + `CREATE TABLE IF NOT EXISTS spaces_blobs ( + space_uri TEXT NOT NULL, + cid TEXT NOT NULL, + mime_type TEXT NOT NULL, + size INTEGER NOT NULL, + author_did TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + PRIMARY KEY (space_uri, cid) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_author ON spaces_blobs(space_uri, author_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_created ON spaces_blobs(space_uri, created_at)`, + + `CREATE TABLE IF NOT EXISTS spaces_invites ( + token_hash TEXT PRIMARY KEY, + space_uri TEXT NOT NULL, + kind TEXT NOT NULL DEFAULT 'join', + expires_at ${dialect.bigintType}, + max_uses INTEGER, + used_count INTEGER NOT NULL DEFAULT 0, + created_by TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + revoked_at ${dialect.bigintType}, + note TEXT + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, + + // Record-host enrollment table — the host's local cache of "spaces I + // accept records for, and which authority signs credentials for each." + // Filled by the recordHost.enroll endpoint, or auto-populated by the + // authority's createSpace when both roles run in the same process. + `CREATE TABLE IF NOT EXISTS record_host_enrollments ( + space_uri TEXT PRIMARY KEY, + authority_did TEXT NOT NULL, + enrolled_at ${dialect.bigintType} NOT NULL, + enrolled_by TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, + ]; +} + +/** Full spaces schema (base + per-collection tables + indexes). For callers + * that need a single array of statements. Note: this does NOT include FTS + * virtual tables or ALTER TABLE count columns — those must be applied with + * try/catch fallbacks and are handled by `initSchema`. */ +export function buildSpacesSchema(db: Database, config?: ContrailConfig): string[] { + const dialect = getDialect(db); + const base = buildSpacesBaseSchema(dialect); + if (!config) return base; + return [ + ...base, + ...buildCollectionTables(config, dialect, { forSpaces: true }), + ...buildDynamicIndexes(config, dialect, { forSpaces: true }), + ]; +} + +export async function initSpacesSchema(db: Database, config?: ContrailConfig): Promise<void> { + const dialect = getDialect(db); + const stmts = buildSpacesSchema(db, config); + await db.batch(stmts.map((s) => db.prepare(s))); + if (!config) return; + for (const stmt of buildFtsTables(config, dialect, { forSpaces: true })) { + try { await db.prepare(stmt).run(); } catch { /* ignore */ } + } + for (const stmt of buildCountColumns(config, { forSpaces: true })) { + try { await db.prepare(stmt).run(); } catch { /* ignore */ } + } +} diff --git a/packages/contrail-appview/src/core/spaces/tid.ts b/packages/contrail-appview/src/core/spaces/tid.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/tid.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/types.ts b/packages/contrail-appview/src/core/spaces/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/spaces/uri.ts b/packages/contrail-appview/src/core/spaces/uri.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/spaces/uri.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/core/types.ts b/packages/contrail-appview/src/core/types.ts new file mode 100644 index 0000000..1129419 --- /dev/null +++ b/packages/contrail-appview/src/core/types.ts @@ -0,0 +1 @@ +export * from "@atmo-dev/contrail-base"; diff --git a/packages/contrail-appview/src/index.ts b/packages/contrail-appview/src/index.ts new file mode 100644 index 0000000..1aad674 --- /dev/null +++ b/packages/contrail-appview/src/index.ts @@ -0,0 +1,65 @@ +/** @atmo-dev/contrail-appview — public-records appview for contrail. + * + * Owns: jetstream ingestion, backfill, refresh, query layer, per-collection + * XRPC routes, feeds, profiles, labels, the umbrella `createApp` that wires + * authority + record-host integrations, plus the `HostedAdapter` + * composition for in-process deployments. + * + * Re-exported wholesale from each module so consumers don't need to know + * internal path layout. */ + +// Forward base + authority + record-host so a consumer that imports +// `@atmo-dev/contrail-appview` (or a shim that re-exports it) sees the full +// shared surface in one place. Contrail bundle re-exports from here to keep +// its public API surface unchanged. +export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-authority"; +export * from "@atmo-dev/contrail-record-host"; + +// Indexing pipeline (jetstream, persistent, backfill, refresh, ingest helpers) +export * from "./core/jetstream"; +export * from "./core/persistent"; +export * from "./core/backfill"; +export * from "./core/refresh"; +export * from "./core/search"; + +// DB +export * from "./core/db/schema"; +export * from "./core/db/records"; +// note: ./core/db/index is implicitly covered by the wildcard if we export it +// — but we don't, since both schema and records may export overlapping names. +// Tests can import the specifics they need. + +// Router (createApp, registerCollectionRoutes, admin, feed, notify, profiles, hydrate) +export * from "./core/router"; +export * from "./core/router/notify"; +export * from "./core/router/profiles"; +export * from "./core/router/feed"; +export * from "./core/router/admin"; +export * from "./core/router/collection"; +export * from "./core/router/hydrate"; +export * from "./core/router/helpers"; + +// Spaces — re-exports + the bundle's HostedAdapter composition +export { HostedAdapter } from "./core/spaces/adapter"; +export { + registerSpacesRoutes, + registerAuthorityRoutes, + registerRecordHostRoutes, +} from "./core/spaces/router"; +export type { + SpacesRoutesOptions, + WhoamiExtension, +} from "./core/spaces/router"; + +// Realtime +export * from "./core/realtime"; + +// Labels +export * from "./core/labels/types"; +export * from "./core/labels/hydrate"; +export * from "./core/labels/select"; +export * from "./core/labels/apply"; +export * from "./core/labels/subscribe"; +export * from "./core/labels/resolve"; +export * from "./core/labels/schema"; diff --git a/packages/contrail-appview/tsconfig.build.json b/packages/contrail-appview/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-appview/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-appview/tsconfig.json b/packages/contrail-appview/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-appview/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-appview/tsup.config.ts b/packages/contrail-appview/tsup.config.ts new file mode 100644 index 0000000..eb81354 --- /dev/null +++ b/packages/contrail-appview/tsup.config.ts @@ -0,0 +1,15 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: [ + "@atmo-dev/contrail-base", + "@atmo-dev/contrail-authority", + "@atmo-dev/contrail-record-host", + ], +}); diff --git a/packages/contrail-authority/src/index.ts b/packages/contrail-authority/src/index.ts index 638b762..8bd56ef 100644 --- a/packages/contrail-authority/src/index.ts +++ b/packages/contrail-authority/src/index.ts @@ -1,8 +1,7 @@ /** @atmo-dev/contrail-authority — default space-authority implementation. * * Owns the authority-side adapter (member list, invites, app policy, space - * lifecycle) and DDL. Route registration currently lives in - * @atmo-dev/contrail and will move here in a subsequent extraction pass. */ + * lifecycle, credential issuance), DDL, and route registration. */ export { HostedAuthorityAdapter, @@ -17,3 +16,8 @@ export { buildAuthoritySchema, applyAuthoritySchema, } from "./schema"; + +export { registerAuthorityRoutes } from "./routes"; + +export { registerInviteRoutes } from "./invite-routes"; +export type { InviteRoutesOptions } from "./invite-routes"; diff --git a/packages/contrail-authority/src/invite-routes.ts b/packages/contrail-authority/src/invite-routes.ts new file mode 100644 index 0000000..dd0fc04 --- /dev/null +++ b/packages/contrail-authority/src/invite-routes.ts @@ -0,0 +1,240 @@ +/** Unified invite surface: a single `<ns>.invite.*` family serving both + * user-owned spaces (handled here directly via the authority adapter) and + * community-owned spaces (delegated to a {@link CommunityInviteHandler}). + * + * Storage stays separate (`spaces_invites` vs `community_invites` tables) — + * schemas differ enough that unifying them would be net-negative. The token + * primitive and HTTP dance are shared. */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { + CommunityInviteHandler, + ContrailConfig, + HandlerResponse, + InviteKind, + InviteRow, + ServiceAuth, + SpaceAuthority, +} from "@atmo-dev/contrail-base"; +import { hashInviteToken, mintInviteToken } from "@atmo-dev/contrail-base"; + +export interface InviteRoutesOptions { + authMiddleware: MiddlewareHandler; +} + +interface PublicInviteView { + tokenHash: string; + spaceUri: string; + kind?: InviteKind; + createdBy: string; + createdAt: number; + expiresAt: number | null; + maxUses: number | null; + usedCount: number; + revokedAt: number | null; + note: string | null; +} + +function toSpacesView(row: InviteRow): PublicInviteView { + return { + tokenHash: row.tokenHash, + spaceUri: row.spaceUri, + kind: row.kind, + createdBy: row.createdBy, + createdAt: row.createdAt, + expiresAt: row.expiresAt, + maxUses: row.maxUses, + usedCount: row.usedCount, + revokedAt: row.revokedAt, + note: row.note, + }; +} + +export function registerInviteRoutes( + app: Hono, + config: ContrailConfig, + authority: SpaceAuthority, + community: CommunityInviteHandler | null, + options: InviteRoutesOptions +): void { + if (!config.spaces?.authority) return; + + const NS = `${config.namespace}.invite`; + const auth = options.authMiddleware; + + const classifySpace = async (spaceUri: string) => { + const space = await authority.getSpace(spaceUri); + if (!space) return null; + const isCommunity = community ? await community.isCommunityOwned(spaceUri) : false; + return { space, isCommunity }; + }; + + app.post(`/xrpc/${NS}.create`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { + spaceUri?: string; + kind?: string; + accessLevel?: string; + expiresAt?: number; + maxUses?: number; + note?: string; + } + | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + if (body.kind && body.accessLevel) { + return c.json( + { error: "InvalidRequest", reason: "kind-or-accessLevel", message: "pass kind OR accessLevel, not both" }, + 400 + ); + } + + const classified = await classifySpace(body.spaceUri); + if (!classified) return c.json({ error: "NotFound" }, 404); + const { space, isCommunity } = classified; + + if (isCommunity) { + if (!community) return c.json({ error: "InvalidState" }, 500); + return relay(c, await community.create({ + spaceUri: body.spaceUri, + callerDid: sa.issuer, + accessLevel: body.accessLevel, + kind: body.kind, + expiresAt: body.expiresAt ?? null, + maxUses: body.maxUses ?? null, + note: body.note ?? null, + })); + } + + if (body.accessLevel) { + return c.json( + { error: "InvalidRequest", reason: "accessLevel-on-user-space", message: "user-owned spaces take kind, not accessLevel" }, + 400 + ); + } + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const kind = (body.kind ?? "join") as InviteKind; + if (kind !== "join" && kind !== "read" && kind !== "read-join") { + return c.json({ error: "InvalidRequest", message: "kind must be 'join', 'read', or 'read-join'" }, 400); + } + const { token, tokenHash } = await mintInviteToken(); + const invite = await authority.createInvite({ + spaceUri: body.spaceUri, + tokenHash, + kind, + expiresAt: body.expiresAt ?? null, + maxUses: body.maxUses ?? null, + createdBy: sa.issuer, + note: body.note ?? null, + }); + return c.json({ token, invite: toSpacesView(invite) }); + }); + + app.get(`/xrpc/${NS}.list`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const includeRevoked = c.req.query("includeRevoked") === "true"; + + const classified = await classifySpace(spaceUri); + if (!classified) return c.json({ error: "NotFound" }, 404); + const { space, isCommunity } = classified; + + if (isCommunity) { + return relay(c, await community!.list({ + spaceUri, + callerDid: sa.issuer, + includeRevoked, + })); + } + + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const rows = await authority.listInvites(spaceUri, { includeRevoked }); + return c.json({ invites: rows.map(toSpacesView) }); + }); + + app.post(`/xrpc/${NS}.revoke`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; tokenHash?: string } + | null; + if (!body?.tokenHash) { + return c.json({ error: "InvalidRequest", message: "tokenHash required" }, 400); + } + + if (body.spaceUri) { + const classified = await classifySpace(body.spaceUri); + if (!classified) return c.json({ error: "NotFound" }, 404); + if (classified.isCommunity) { + return relay(c, await community!.revoke({ + spaceUri: body.spaceUri, + tokenHash: body.tokenHash, + callerDid: sa.issuer, + })); + } + if (classified.space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const ok = await authority.revokeInvite(body.tokenHash); + return c.json({ ok }); + } + + if (community) { + const r = await community.tryRevokeByToken({ + tokenHash: body.tokenHash, + callerDid: sa.issuer, + }); + if (r) return relay(c, r); + } + const srow = await authority.getInvite(body.tokenHash); + if (!srow) return c.json({ error: "NotFound" }, 404); + const space = await authority.getSpace(srow.spaceUri); + if (space && space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + const ok = await authority.revokeInvite(body.tokenHash); + return c.json({ ok }); + }); + + app.post(`/xrpc/${NS}.redeem`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { token?: string } | null; + if (!body?.token) { + return c.json({ error: "InvalidRequest", message: "token required" }, 400); + } + const tokenHash = await hashInviteToken(body.token); + const now = Date.now(); + + if (community) { + const r = await community.tryRedeem({ + tokenHash, + callerDid: sa.issuer, + now, + }); + if (r) return relay(c, r); + } + + const sinvite = await authority.redeemInvite(tokenHash, now); + if (!sinvite) { + return c.json({ error: "InvalidInvite", reason: "expired-revoked-or-exhausted" }, 400); + } + await authority.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); + return c.json({ spaceUri: sinvite.spaceUri, kind: sinvite.kind }); + }); +} + +function relay(c: Context, r: HandlerResponse) { + return c.json(r.body, r.status as Parameters<typeof c.json>[1]); +} + +function getAuth(c: Context): ServiceAuth { + const a = c.get("serviceAuth") as ServiceAuth | undefined; + if (!a) throw new Error("service auth not set"); + return a; +} diff --git a/packages/contrail-authority/src/routes.ts b/packages/contrail-authority/src/routes.ts new file mode 100644 index 0000000..fe80e37 --- /dev/null +++ b/packages/contrail-authority/src/routes.ts @@ -0,0 +1,412 @@ +/** Authority XRPC routes — space lifecycle, members, app policy, whoami, + * credential issuance. Does NOT touch records or blobs. + * + * Deployments wire this via `registerAuthorityRoutes(app, authority, ...)`. + * The umbrella `registerSpacesRoutes` in @atmo-dev/contrail composes this + * with the record-host routes; split deployments call this directly. */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { + AuthorityConfig, + ContrailConfig, + CredentialClaims, + RecordHost, + ServiceAuth, + SpaceAuthority, + SpaceRow, + WhoamiExtension, +} from "@atmo-dev/contrail-base"; +import { + buildSpaceUri, + checkInviteReadGrant, + decodeUnverifiedClaims, + DEFAULT_CREDENTIAL_TTL_MS, + extractInviteToken, + hashInviteToken, + issueCredential, + nextTid, + verifyCredential, +} from "@atmo-dev/contrail-base"; + +/** When `localRecordHost` is non-null, the authority's `createSpace` handler + * also enrolls the new space on that host — convenient for in-process + * deployments where the same operator runs both roles. Split deployments + * pass `null` and arrange enrollment explicitly via `recordHost.enroll`. */ +export function registerAuthorityRoutes( + app: Hono, + authority: SpaceAuthority, + authorityConfig: AuthorityConfig, + config: ContrailConfig, + auth: MiddlewareHandler, + whoamiExtension?: WhoamiExtension, + localRecordHost?: RecordHost | null +): void { + /** Space endpoints are emitted per-deployment under the configured namespace; + * the deployment owns and publishes its own lexicons. */ + const SPACE = `${config.namespace}.space`; + const SPACE_EXT = `${config.namespace}.spaceExt`; + + // ---- Read endpoints ---- + + app.get(`/xrpc/${SPACE}.listSpaces`, auth, async (c) => { + const sa = getAuth(c); + const scope = c.req.query("scope") ?? "member"; + const type = c.req.query("type") ?? undefined; + const owner = c.req.query("owner") ?? undefined; + const cursor = c.req.query("cursor") ?? undefined; + const limit = c.req.query("limit") ? Number(c.req.query("limit")) : undefined; + + const opts: Parameters<typeof authority.listSpaces>[0] = { type, cursor, limit }; + if (scope === "owner") opts.ownerDid = sa.issuer; + else { + opts.memberDid = sa.issuer; + if (owner) opts.ownerDid = owner; + } + + const result = await authority.listSpaces(opts); + return c.json({ + spaces: result.spaces.map((s) => publicSpaceView(s, s.ownerDid === sa.issuer)), + cursor: result.cursor, + }); + }); + + app.get(`/xrpc/${SPACE}.listMembers`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + const member = isOwner ? null : await authority.getMember(spaceUri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + const members = await authority.listMembers(spaceUri); + return c.json({ members }); + }); + + /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is + * present so anonymous bearer reads don't 401 before the route handler can + * validate the token. */ + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return auth(c, next); + }; + + app.get(`/xrpc/${SPACE}.getSpace`, readAuth, async (c) => { + const uri = c.req.query("uri"); + if (!uri) return c.json({ error: "InvalidRequest", message: "uri required" }, 400); + const space = await authority.getSpace(uri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const authz = await authorizeRead(c, authority, uri); + if (authz instanceof Response) return authz; + + if (authz.via === "token") { + return c.json({ space: publicSpaceView(space, false) }); + } + + if (authz.via === "credential") { + const isOwner = authz.claims.sub === space.ownerDid; + return c.json({ space: publicSpaceView(space, isOwner) }); + } + + const sa = authz.sa; + const isOwner = sa.issuer === space.ownerDid; + const member = isOwner ? null : await authority.getMember(uri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + return c.json({ space: publicSpaceView(space, isOwner) }); + }); + + // ---- Space management (owner-gated) ---- + + app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => ({}))) as { + type?: string; + key?: string; + appPolicy?: SpaceRow["appPolicy"]; + appPolicyRef?: string; + }; + + const type = body.type ?? authorityConfig.type; + const key = body.key ?? nextTid(); + const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); + + const existing = await authority.getSpace(uri); + if (existing) return c.json({ error: "AlreadyExists", uri }, 409); + + const space = await authority.createSpace({ + uri, + ownerDid: sa.issuer, + type, + key, + serviceDid: authorityConfig.serviceDid, + appPolicyRef: body.appPolicyRef ?? null, + appPolicy: body.appPolicy ?? authorityConfig.defaultAppPolicy ?? null, + }); + await authority.addMember(uri, sa.issuer, sa.issuer); + + if (localRecordHost) { + await localRecordHost.enroll({ + spaceUri: uri, + authorityDid: authorityConfig.serviceDid, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + } + + return c.json({ space: publicSpaceView(space, true) }); + }); + + app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + await authority.addMember(body.spaceUri, body.did, sa.issuer); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; did?: string } + | null; + if (!body?.spaceUri || !body.did) { + return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid !== sa.issuer) { + return c.json({ error: "Forbidden", reason: "not-owner" }, 403); + } + if (body.did === space.ownerDid) { + return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); + } + await authority.removeMember(body.spaceUri, body.did); + return c.json({ ok: true }); + }); + + app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + if (space.ownerDid === sa.issuer) { + return c.json( + { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, + 400 + ); + } + await authority.removeMember(body.spaceUri, sa.issuer); + return c.json({ ok: true }); + }); + + // Unified whoami — extension can override with richer data (e.g. community + // accessLevel); without one, returns binary owner/member. + app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { + const sa = getAuth(c); + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + + if (whoamiExtension) { + const ext = await whoamiExtension({ + spaceUri, + callerDid: sa.issuer, + isOwner, + ownerDid: space.ownerDid, + }); + if (ext) return c.json(ext); + } + + if (isOwner) return c.json({ isOwner: true, isMember: true }); + const member = await authority.getMember(spaceUri, sa.issuer); + return c.json({ isOwner: false, isMember: !!member }); + }); + + // ---- Credential endpoints ---- + + app.post(`/xrpc/${SPACE}.getCredential`, auth, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; + if (!body?.spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + + const isOwner = space.ownerDid === sa.issuer; + const member = isOwner ? null : await authority.getMember(body.spaceUri, sa.issuer); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + if (space.appPolicy) { + const allowed = checkClientId(space.appPolicy, sa.clientId); + if (!allowed) return c.json({ error: "Forbidden", reason: "app-not-allowed" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: sa.issuer, + space: body.spaceUri, + scope: "rw", + ttlMs: ttl, + }, + authorityConfig.signing + ); + return c.json({ credential, expiresAt }); + }); + + app.post(`/xrpc/${SPACE}.refreshCredential`, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign credentials" }, + 501 + ); + } + const body = (await c.req.json().catch(() => null)) as { credential?: string } | null; + if (!body?.credential) { + return c.json({ error: "InvalidRequest", message: "credential required" }, 400); + } + const signing = authorityConfig.signing; + const claims = await verifyAndAuthorizeRefresh(body.credential, authorityConfig); + if ("error" in claims) return c.json(claims, claims.status); + + const space = await authority.getSpace(claims.space); + if (!space) return c.json({ error: "NotFound" }, 404); + const isOwner = space.ownerDid === claims.sub; + const member = isOwner ? null : await authority.getMember(claims.space, claims.sub); + if (!isOwner && !member) { + return c.json({ error: "Forbidden", reason: "not-member" }, 403); + } + + const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; + const { credential, expiresAt } = await issueCredential( + { + iss: authorityConfig.serviceDid, + sub: claims.sub, + space: claims.space, + scope: claims.scope, + ttlMs: ttl, + }, + signing + ); + return c.json({ credential, expiresAt }); + }); +} + +/** Verify a credential presented at refreshCredential. */ +async function verifyAndAuthorizeRefresh( + credential: string, + authorityConfig: AuthorityConfig +): Promise<CredentialClaims | { error: string; reason?: string; message?: string; status: 400 | 401 }> { + const peek = decodeUnverifiedClaims(credential); + if (!peek) return { error: "InvalidRequest", reason: "malformed", status: 400 }; + if (peek.iss !== authorityConfig.serviceDid) { + return { error: "Forbidden", reason: "wrong-issuer", status: 401 }; + } + if (!authorityConfig.signing) { + return { error: "InvalidState", status: 401 }; + } + const signing = authorityConfig.signing; + const result = await verifyCredential(credential, { + expectedSpace: peek.space, + resolveKey: async (iss) => (iss === authorityConfig.serviceDid ? signing.publicKey : null), + }); + if (!result.ok) { + return { error: "InvalidCredential", reason: result.reason, status: 401 }; + } + return result.claims; +} + +function checkClientId( + appPolicy: NonNullable<SpaceRow["appPolicy"]>, + clientId: string | undefined +): boolean { + const listed = clientId ? appPolicy.apps.includes(clientId) : false; + if (appPolicy.mode === "allow") return !listed; + return listed; +} + +/** Authorize a read request on the authority side — three valid paths: + * credential (set by upstream middleware), invite token, or service-auth JWT. */ +async function authorizeRead( + c: Context, + authority: SpaceAuthority, + spaceUri: string +): Promise< + | { via: "credential"; claims: CredentialClaims } + | { via: "token" } + | { via: "jwt"; sa: ServiceAuth } + | Response +> { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== spaceUri) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + return { via: "credential", claims: cred }; + } + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); + if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); + return { via: "token" }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (sa) return { via: "jwt", sa }; + return c.json( + { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, + 401 + ); +} + +function getAuth(c: Context): ServiceAuth { + const auth = c.get("serviceAuth") as ServiceAuth | undefined; + if (!auth) throw new Error("service auth not set"); + return auth; +} + +function publicSpaceView(space: SpaceRow, forOwner: boolean) { + return { + uri: space.uri, + ownerDid: space.ownerDid, + type: space.type, + key: space.key, + serviceDid: space.serviceDid, + appPolicyRef: space.appPolicyRef, + createdAt: space.createdAt, + ...(forOwner ? { appPolicy: space.appPolicy } : {}), + }; +} diff --git a/packages/contrail-record-host/package.json b/packages/contrail-record-host/package.json new file mode 100644 index 0000000..2182f37 --- /dev/null +++ b/packages/contrail-record-host/package.json @@ -0,0 +1,43 @@ +{ + "name": "@atmo-dev/contrail-record-host", + "version": "0.6.0", + "description": "Default record-host implementation for contrail — stores records and blobs for permissioned spaces, enforces local enrollment as the host's consent layer.", + "type": "module", + "sideEffects": false, + "files": [ + "dist" + ], + "publishConfig": { + "access": "public" + }, + "exports": { + ".": { + "types": "./dist/index.d.ts", + "import": "./dist/index.js" + } + }, + "repository": { + "type": "git", + "url": "https://github.com/flo-bit/contrail.git", + "directory": "packages/contrail-record-host" + }, + "keywords": [ + "atproto", + "contrail" + ], + "scripts": { + "build": "tsup", + "clean": "rm -rf dist", + "typecheck": "tsc --noEmit" + }, + "dependencies": { + "@atcute/cid": "^2.4.1", + "@atmo-dev/contrail-base": "workspace:*", + "hono": "^4.12.8" + }, + "devDependencies": { + "tsup": "^8.5.0", + "typescript": "^5.7.3" + }, + "license": "MIT" +} diff --git a/packages/contrail-record-host/src/adapter.ts b/packages/contrail-record-host/src/adapter.ts new file mode 100644 index 0000000..4bfb321 --- /dev/null +++ b/packages/contrail-record-host/src/adapter.ts @@ -0,0 +1,530 @@ +/** Default {@link RecordHost} implementation backed by a Database. + * + * Owns the host-side tables — `spaces_records_<short>` (per-collection), + * `spaces_blobs`, `record_host_enrollments` — and exposes record CRUD, + * blob metadata management, enrollment, and `findOrphanBlobs` for GC. + * + * Independent of the authority adapter: takes a Database directly, doesn't + * inherit from anything. Bundles that want a single adapter satisfying both + * roles instantiate this alongside HostedAuthorityAdapter against the same + * DB. The duplication of the underlying tables is fine — `IF NOT EXISTS` + * guards make schema application idempotent. */ + +import type { + ContrailConfig, + Database, + RelationConfig, + ResolvedContrailConfig, + RecordHost, + BlobMetaRow, + CollectionCount, + EnrollmentRow, + ListBlobsOptions, + ListBlobsResult, + ListOptions, + ListResult, + StoredRecord, +} from "@atmo-dev/contrail-base"; +import { + shortNameForNsid, + spacesRecordsTableName, + countColumnName, + groupedCountColumnName, + getRelationField, + getNestedValue, + getDialect, + buildRecordUri, +} from "@atmo-dev/contrail-base"; + +function parseJson<T>(value: unknown): T | null { + if (value == null) return null; + if (typeof value === "string") { + try { + return JSON.parse(value) as T; + } catch { + return null; + } + } + return value as T; +} + +function toNum(v: unknown): number { + return typeof v === "string" ? Number(v) : (v as number); +} + +export function mapBlobMetaRow(row: any): BlobMetaRow { + return { + spaceUri: row.space_uri, + cid: row.cid, + mimeType: row.mime_type, + size: Number(row.size), + authorDid: row.author_did, + createdAt: toNum(row.created_at), + }; +} + +export function mapEnrollmentRow(row: any): EnrollmentRow { + return { + spaceUri: row.space_uri, + authorityDid: row.authority_did, + enrolledAt: toNum(row.enrolled_at), + enrolledBy: row.enrolled_by, + }; +} + +/** Row mapper for per-collection spaces_records_<short> tables. + * `collection` is injected by the caller (known from the table name). */ +export function mapRecordRow(row: any, collection: string): StoredRecord { + return { + spaceUri: row.space_uri, + collection, + authorDid: row.did, + rkey: row.rkey, + cid: row.cid ?? null, + record: parseJson<Record<string, unknown>>(row.record) ?? {}, + createdAt: toNum(row.time_us), + }; +} + +export class HostedRecordHostAdapter implements RecordHost { + constructor( + protected readonly db: Database, + protected readonly config?: ContrailConfig + ) {} + + /** Resolve the per-collection spaces table name, or throw if the collection + * isn't configured (and therefore has no table). */ + protected tableFor(collection: string): string { + if (!this.config) { + throw new Error( + `HostedRecordHostAdapter: config not provided; cannot resolve table for collection ${collection}` + ); + } + const short = shortNameForNsid(this.config, collection); + if (!short) { + throw new Error( + `HostedRecordHostAdapter: collection ${collection} is not configured in this deployment` + ); + } + return spacesRecordsTableName(short); + } + + // ---- Enrollment ---- + + async enroll(input: EnrollmentRow): Promise<void> { + await this.db + .prepare( + `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) + VALUES (?, ?, ?, ?) + ON CONFLICT (space_uri) DO UPDATE SET + authority_did = excluded.authority_did, + enrolled_at = excluded.enrolled_at, + enrolled_by = excluded.enrolled_by` + ) + .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) + .run(); + } + + async getEnrollment(spaceUri: string): Promise<EnrollmentRow | null> { + const row = await this.db + .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .first<any>(); + return row ? mapEnrollmentRow(row) : null; + } + + async listEnrollments( + options: { authorityDid?: string; limit?: number } = {} + ): Promise<EnrollmentRow[]> { + const limit = Math.min(options.limit ?? 200, 1000); + if (options.authorityDid) { + const { results } = await this.db + .prepare( + `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` + ) + .bind(options.authorityDid, limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + const { results } = await this.db + .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) + .bind(limit) + .all<any>(); + return results.map(mapEnrollmentRow); + } + + async removeEnrollment(spaceUri: string): Promise<void> { + await this.db + .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) + .bind(spaceUri) + .run(); + } + + // ---- Records ---- + + async putRecord(record: StoredRecord): Promise<void> { + const table = this.tableFor(record.collection); + const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); + + const childShort = this.config ? shortNameForNsid(this.config, record.collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(record.spaceUri, record.authorDid, record.rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `INSERT INTO ${table} (space_uri, uri, did, rkey, cid, record, time_us, indexed_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, did, rkey) DO UPDATE SET + uri = excluded.uri, + cid = excluded.cid, + record = excluded.record, + time_us = excluded.time_us, + indexed_at = excluded.indexed_at` + ) + .bind( + record.spaceUri, + uri, + record.authorDid, + record.rkey, + record.cid, + JSON.stringify(record.record), + record.createdAt, + Date.now() + ) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace( + record.spaceUri, + childShort, + beforeRecord, + record.record, + record.authorDid + ); + } + } + + async getRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<StoredRecord | null> { + const table = this.tableFor(collection); + const row = await this.db + .prepare( + `SELECT * FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .first<any>(); + return row ? mapRecordRow(row, collection) : null; + } + + async listRecords( + spaceUri: string, + collection: string, + options: ListOptions = {} + ): Promise<ListResult> { + const table = this.tableFor(collection); + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("time_us < ?"); + params.push(Number(options.cursor)); + } + + const sql = `SELECT * FROM ${table} + WHERE ${clauses.join(" AND ")} + ORDER BY time_us DESC + LIMIT ?`; + params.push(limit + 1); + + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const records = results.map((r) => mapRecordRow(r, collection)); + let cursor: string | undefined; + if (records.length > limit) { + const next = records.pop()!; + cursor = String(next.createdAt); + } + return { records, cursor }; + } + + async deleteRecord( + spaceUri: string, + collection: string, + authorDid: string, + rkey: string + ): Promise<void> { + const table = this.tableFor(collection); + + const childShort = this.config ? shortNameForNsid(this.config, collection) : null; + const prev = childShort + ? await this.db + .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) + .bind(spaceUri, authorDid, rkey) + .first<{ record: unknown } | null>() + : null; + const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); + + await this.db + .prepare( + `DELETE FROM ${table} + WHERE space_uri = ? AND did = ? AND rkey = ?` + ) + .bind(spaceUri, authorDid, rkey) + .run(); + + if (childShort && this.config) { + await this.recountParentsForSpace(spaceUri, childShort, beforeRecord, null, authorDid); + } + } + + /** Recompute count columns on parent records in the same space, scoped to the + * targets derived from before/after versions of the written/deleted child record. */ + protected async recountParentsForSpace( + spaceUri: string, + childShort: string, + before: Record<string, unknown> | null, + after: Record<string, unknown> | null, + childDid: string + ): Promise<void> { + if (!this.config) return; + const config = this.config; + const resolved = (config as ResolvedContrailConfig)._resolved; + const childTable = spacesRecordsTableName(childShort); + + type Inbound = { parentShort: string; relationName: string; rel: RelationConfig }; + const inbound: Inbound[] = []; + for (const [parentShort, parentCfg] of Object.entries(config.collections)) { + if (parentCfg.allowInSpaces === false) continue; + for (const [relName, rel] of Object.entries(parentCfg.relations ?? {})) { + if (rel.count === false) continue; + if (rel.collection !== childShort) continue; + inbound.push({ parentShort, relationName: relName, rel }); + } + } + if (inbound.length === 0) return; + + // Deduplicate (parent, relation, target) across before/after. + const keyed = new Map<string, { parentShort: string; relationName: string; rel: RelationConfig; target: string }>(); + for (const { parentShort, relationName, rel } of inbound) { + const field = getRelationField(rel); + const collectTarget = (rec: Record<string, unknown> | null) => { + if (!rec) return; + if (rel.match === "did") { + keyed.set(`${parentShort}:${relationName}:${childDid}`, { + parentShort, relationName, rel, target: childDid, + }); + return; + } + const v = getNestedValue(rec, field); + if (typeof v === "string" && v.length > 0) { + keyed.set(`${parentShort}:${relationName}:${v}`, { + parentShort, relationName, rel, target: v, + }); + } + }; + collectTarget(before); + collectTarget(after); + } + if (keyed.size === 0) return; + + const dialect = getDialect(this.db); + const stmts: ReturnType<Database["prepare"]>[] = []; + + for (const { parentShort, relationName, rel, target } of keyed.values()) { + const parentTable = spacesRecordsTableName(parentShort); + const matchColumn = rel.match === "did" ? "did" : "uri"; + const field = getRelationField(rel); + const countExpr = rel.countDistinct + ? `COUNT(DISTINCT ${rel.countDistinct})` + : "COUNT(*)"; + + const setClauses: string[] = []; + const binds: (string | number)[] = []; + + const totalCol = countColumnName(rel.collection); + setClauses.push( + `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ?)` + ); + binds.push(spaceUri, target); + + if (rel.groupBy) { + const mapping = resolved?.relations[parentShort]?.[relationName]; + if (mapping?.groups) { + for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { + const groupCol = groupedCountColumnName(rel.collection, groupKey); + setClauses.push( + `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ? AND ${dialect.jsonExtract("record", rel.groupBy)} = ?)` + ); + binds.push(spaceUri, target, fullToken); + } + } + } + + binds.push(spaceUri, target); + stmts.push( + this.db + .prepare( + `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE space_uri = ? AND ${matchColumn} = ?` + ) + .bind(...binds) + ); + } + + if (stmts.length > 0) await this.db.batch(stmts); + } + + async listCollections( + spaceUri: string, + options: { byUser?: string } = {} + ): Promise<CollectionCount[]> { + if (!this.config) return []; + const results: CollectionCount[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + const table = spacesRecordsTableName(short); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("did = ?"); + params.push(options.byUser); + } + try { + const row = await this.db + .prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${clauses.join(" AND ")}`) + .bind(...params) + .first<{ count: number }>(); + const count = Number(row?.count ?? 0); + if (count > 0) results.push({ collection: colConfig.collection, count }); + } catch { + // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip + } + } + return results; + } + + // ---- Blobs ---- + + async putBlobMeta(row: BlobMetaRow): Promise<void> { + const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) + VALUES (?, ?, ?, ?, ?, ?) + ON CONFLICT (space_uri, cid) DO NOTHING`; + await this.db + .prepare(sql) + .bind(row.spaceUri, row.cid, row.mimeType, row.size, row.authorDid, row.createdAt) + .run(); + } + + async getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null> { + const r = await this.db + .prepare(`SELECT * FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .first<any>(); + return r ? mapBlobMetaRow(r) : null; + } + + async listBlobMeta( + spaceUri: string, + options: ListBlobsOptions = {} + ): Promise<ListBlobsResult> { + const limit = Math.min(options.limit ?? 50, 200); + const clauses: string[] = ["space_uri = ?"]; + const params: any[] = [spaceUri]; + if (options.byUser) { + clauses.push("author_did = ?"); + params.push(options.byUser); + } + if (options.cursor) { + clauses.push("created_at < ?"); + params.push(Number(options.cursor)); + } + const sql = `SELECT * FROM spaces_blobs + WHERE ${clauses.join(" AND ")} + ORDER BY created_at DESC + LIMIT ?`; + params.push(limit + 1); + const { results } = await this.db.prepare(sql).bind(...params).all<any>(); + const blobs = results.map(mapBlobMetaRow); + let cursor: string | undefined; + if (blobs.length > limit) { + const next = blobs.pop()!; + cursor = String(next.createdAt); + } + return { blobs, cursor }; + } + + async deleteBlobMeta(spaceUri: string, cid: string): Promise<void> { + await this.db + .prepare(`DELETE FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) + .bind(spaceUri, cid) + .run(); + } + + async findOrphanBlobs( + spaceUri: string, + cutoff: number, + limit: number + ): Promise<BlobMetaRow[]> { + if (!this.config) return []; + // Gather candidate blobs older than cutoff, then filter out any whose CID + // appears in any record JSON in this space. We use a cheap substring probe + // (LIKE) per collection — false positives are OK because an orphan that + // survives GC just gets collected next cycle; false negatives (deleting + // a referenced blob) would be a bug, and substring search over the full + // CID is safe enough for that. + const { results } = await this.db + .prepare( + `SELECT * FROM spaces_blobs + WHERE space_uri = ? AND created_at < ? + ORDER BY created_at ASC + LIMIT ?` + ) + .bind(spaceUri, cutoff, limit) + .all<any>(); + const candidates = results.map(mapBlobMetaRow); + if (candidates.length === 0) return []; + + const tables: string[] = []; + for (const [short, colConfig] of Object.entries(this.config.collections)) { + if (colConfig.allowInSpaces === false) continue; + tables.push(spacesRecordsTableName(short)); + } + + const orphans: BlobMetaRow[] = []; + for (const blob of candidates) { + let referenced = false; + const pattern = `%${blob.cid}%`; + for (const table of tables) { + try { + const row = await this.db + .prepare( + `SELECT 1 FROM ${table} WHERE space_uri = ? AND record LIKE ? LIMIT 1` + ) + .bind(spaceUri, pattern) + .first<any>(); + if (row) { + referenced = true; + break; + } + } catch { + // table missing — ignore + } + } + if (!referenced) orphans.push(blob); + } + return orphans; + } +} diff --git a/packages/contrail-record-host/src/blob-gc.ts b/packages/contrail-record-host/src/blob-gc.ts new file mode 100644 index 0000000..19ca37f --- /dev/null +++ b/packages/contrail-record-host/src/blob-gc.ts @@ -0,0 +1,37 @@ +import type { BlobAdapter, RecordHost } from "@atmo-dev/contrail-base"; +import { blobKey } from "@atmo-dev/contrail-base"; + +export interface BlobGcOptions { + /** Orphan rows created before this timestamp are eligible for deletion. */ + olderThan: number; + /** Maximum number of blobs to delete in this pass. Defaults to 500. */ + batchSize?: number; +} + +export interface BlobGcResult { + deleted: number; + cids: string[]; +} + +/** Delete blob bytes + metadata for any blob older than `olderThan` that + * is not referenced by any record in the space. Safe to run periodically. */ +export async function gcOrphanBlobs( + storage: RecordHost, + blobs: BlobAdapter, + spaceUri: string, + options: BlobGcOptions +): Promise<BlobGcResult> { + const batchSize = options.batchSize ?? 500; + const orphans = await storage.findOrphanBlobs(spaceUri, options.olderThan, batchSize); + if (orphans.length === 0) return { deleted: 0, cids: [] }; + + const keys: string[] = []; + for (const row of orphans) { + keys.push(await blobKey(row.spaceUri, row.cid)); + } + await blobs.delete(keys); + for (const row of orphans) { + await storage.deleteBlobMeta(row.spaceUri, row.cid); + } + return { deleted: orphans.length, cids: orphans.map((o) => o.cid) }; +} diff --git a/packages/contrail-record-host/src/blob-refs.ts b/packages/contrail-record-host/src/blob-refs.ts new file mode 100644 index 0000000..62cf693 --- /dev/null +++ b/packages/contrail-record-host/src/blob-refs.ts @@ -0,0 +1,27 @@ +/** + * Walk a record JSON and collect every atproto blob ref. + * + * Blob refs look like: + * { "$type": "blob", "ref": { "$link": "<cid>" }, "mimeType": "...", "size": N } + * + * We return the CID strings. + */ +export function collectBlobCids(value: unknown, out: Set<string> = new Set()): Set<string> { + if (value == null) return out; + if (Array.isArray(value)) { + for (const v of value) collectBlobCids(v, out); + return out; + } + if (typeof value !== "object") return out; + + const obj = value as Record<string, unknown>; + if (obj["$type"] === "blob") { + const ref = obj["ref"] as { $link?: unknown } | undefined; + if (ref && typeof ref["$link"] === "string") out.add(ref["$link"]); + // Don't descend — a blob ref's own shape has no nested blobs. + return out; + } + + for (const v of Object.values(obj)) collectBlobCids(v, out); + return out; +} diff --git a/packages/contrail-record-host/src/index.ts b/packages/contrail-record-host/src/index.ts new file mode 100644 index 0000000..ff4eacd --- /dev/null +++ b/packages/contrail-record-host/src/index.ts @@ -0,0 +1,27 @@ +/** @atmo-dev/contrail-record-host — default record-host implementation. + * + * Owns the host-side adapter (records, blobs, enrollment) + DDL + blob-GC. + * Independent of the authority — takes a Database directly, doesn't inherit + * from anything. Bundles can instantiate this alongside HostedAuthorityAdapter + * against the same DB to get full StorageAdapter behavior. */ + +export { + HostedRecordHostAdapter, + mapBlobMetaRow, + mapEnrollmentRow, + mapRecordRow, +} from "./adapter"; + +export { + buildRecordHostBaseSchema, + applyRecordHostSchema, +} from "./schema"; + +export { + gcOrphanBlobs, +} from "./blob-gc"; +export type { BlobGcOptions, BlobGcResult } from "./blob-gc"; + +export { collectBlobCids } from "./blob-refs"; + +export { registerRecordHostRoutes } from "./routes"; diff --git a/packages/contrail-record-host/src/routes.ts b/packages/contrail-record-host/src/routes.ts new file mode 100644 index 0000000..63fc951 --- /dev/null +++ b/packages/contrail-record-host/src/routes.ts @@ -0,0 +1,510 @@ +/** Record-host XRPC routes — record + blob CRUD plus enrollment. + * + * Auth precedence on every route: + * 1. `X-Space-Credential` header (if a verifier is wired and the credential + * is valid) — caller DID = credential `sub`, no clientId. + * 2. Read-route invite token (`?inviteToken=` or `Bearer atmo-invite:...`). + * 3. Service-auth JWT — caller DID = JWT issuer. + * + * When a credential is presented, the record host trusts it: no member + * check, no app-policy check (those happen at issuance time on the + * authority side). Service-auth requests still consult the authority. */ + +import type { Context, Hono, MiddlewareHandler } from "hono"; +import type { + ContrailConfig, + CredentialClaims, + CredentialScope, + CredentialVerifier, + RecordHost, + RecordHostConfig, + ServiceAuth, + SpaceAuthority, +} from "@atmo-dev/contrail-base"; +import { + blobKey, + checkAccess, + checkInviteReadGrant, + DEFAULT_BLOB_MAX_SIZE, + extractInviteToken, + extractSpaceCredential, + hashInviteToken, + nextTid, + parseSpaceUri, +} from "@atmo-dev/contrail-base"; +import { collectBlobCids } from "./blob-refs"; +import { create as createCid, toString as cidToString } from "@atcute/cid"; + +export function registerRecordHostRoutes( + app: Hono, + recordHost: RecordHost, + authority: SpaceAuthority, + recordHostConfig: RecordHostConfig, + config: ContrailConfig, + auth: MiddlewareHandler, + /** Optional credential verifier. When present, the record host accepts + * `X-Space-Credential` as an alternative to a service-auth JWT. */ + credentialVerifier?: CredentialVerifier +): void { + const SPACE = `${config.namespace}.space`; + + /** Auth wrapper: tries credential first, then delegates to JWT auth. */ + const authWithCredential: MiddlewareHandler = async (c, next) => { + const credToken = extractSpaceCredential(c.req.raw); + if (credToken) { + if (!credentialVerifier) { + return c.json( + { error: "AuthRequired", reason: "credential-verifier-not-configured" }, + 401 + ); + } + const result = await credentialVerifier.verify(credToken); + if (!result.ok) { + return c.json({ error: "AuthRequired", reason: result.reason }, 401); + } + c.set("spaceCredential", result.claims); + await next(); + return; + } + return auth(c, next); + }; + + const readAuth: MiddlewareHandler = async (c, next) => { + if (extractInviteToken(c.req.raw)) { + await next(); + return; + } + return authWithCredential(c, next); + }; + + /** Hard gate on every record-host operation: the space must be enrolled. */ + async function requireEnrollment( + c: Context, + spaceUri: string + ): Promise<{ authorityDid: string } | Response> { + const enrollment = await recordHost.getEnrollment(spaceUri); + if (!enrollment) { + return c.json( + { error: "NotFound", reason: "not-enrolled", message: "space is not enrolled on this record host" }, + 404 + ); + } + return enrollment; + } + + // ---- Enrollment endpoint ---- + const RECORD_HOST = `${config.namespace}.recordHost`; + + app.post(`/xrpc/${RECORD_HOST}.enroll`, auth, async (c) => { + const sa = getAuth(c); + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; authority?: string } + | null; + if (!body?.spaceUri || !body.authority) { + return c.json({ error: "InvalidRequest", message: "spaceUri and authority required" }, 400); + } + const parts = parseSpaceUri(body.spaceUri); + if (!parts) { + return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); + } + const callerIsOwner = sa.issuer === parts.ownerDid; + const callerIsAuthority = sa.issuer === body.authority; + if (!callerIsOwner && !callerIsAuthority) { + return c.json( + { error: "Forbidden", reason: "not-owner-or-authority" }, + 403 + ); + } + await recordHost.enroll({ + spaceUri: body.spaceUri, + authorityDid: body.authority, + enrolledAt: Date.now(), + enrolledBy: sa.issuer, + }); + return c.json({ ok: true }); + }); + + app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { + const spaceUri = c.req.query("spaceUri"); + const collection = c.req.query("collection"); + if (!spaceUri || !collection) { + return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const authz = await authorizeRead(c, authority, spaceUri); + if (authz instanceof Response) return authz; + + if (authz.via === "jwt") { + const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, sa.issuer); + const result = checkAccess({ + op: "read", + space, + callerDid: sa.issuer, + member, + clientId: sa.clientId, + }); + if (!result.allow) { + return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + } + + const list = await recordHost.listRecords(spaceUri, collection, { + byUser: c.req.query("byUser") ?? undefined, + cursor: c.req.query("cursor") ?? undefined, + limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, + }); + return c.json(list); + }); + + app.get(`/xrpc/${SPACE}.getRecord`, readAuth, async (c) => { + const spaceUri = c.req.query("spaceUri"); + const collection = c.req.query("collection"); + const author = c.req.query("author"); + const rkey = c.req.query("rkey"); + if (!spaceUri || !collection || !author || !rkey) { + return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const authz = await authorizeRead(c, authority, spaceUri); + if (authz instanceof Response) return authz; + + if (authz.via === "jwt") { + const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, sa.issuer); + const result = checkAccess({ + op: "read", + space, + callerDid: sa.issuer, + member, + clientId: sa.clientId, + targetAuthorDid: author, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + + const record = await recordHost.getRecord(spaceUri, collection, author, rkey); + if (!record) return c.json({ error: "NotFound" }, 404); + return c.json({ record }); + }); + + app.post(`/xrpc/${SPACE}.putRecord`, authWithCredential, async (c) => { + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; collection?: string; rkey?: string; record?: Record<string, unknown> } + | null; + if (!body?.spaceUri || !body.collection || !body.record) { + return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); + } + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + + if (recordHostConfig.blobs) { + const cids = collectBlobCids(body.record); + for (const cid of cids) { + const meta = await recordHost.getBlobMeta(body.spaceUri, cid); + if (!meta) { + return c.json( + { + error: "InvalidRequest", + reason: "unknown-blob-ref", + message: `Record references blob ${cid} that has not been uploaded to this space.`, + }, + 400 + ); + } + } + } + + const rkey = body.rkey ?? nextTid(); + const now = Date.now(); + await recordHost.putRecord({ + spaceUri: body.spaceUri, + collection: body.collection, + authorDid: caller.callerDid, + rkey, + cid: null, + record: body.record, + createdAt: now, + }); + return c.json({ rkey, authorDid: caller.callerDid, createdAt: now }); + }); + + app.post(`/xrpc/${SPACE}.deleteRecord`, authWithCredential, async (c) => { + const body = (await c.req.json().catch(() => null)) as + | { spaceUri?: string; collection?: string; rkey?: string } + | null; + if (!body?.spaceUri || !body.collection || !body.rkey) { + return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); + } + const enrollment = await requireEnrollment(c, body.spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, body.spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(body.spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(body.spaceUri, caller.callerDid); + const result = checkAccess({ + op: "delete", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + targetAuthorDid: caller.callerDid, + }); + if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); + } + + await recordHost.deleteRecord(body.spaceUri, body.collection, caller.callerDid, body.rkey); + return c.json({ ok: true }); + }); + + // Blobs (only registered when a blob adapter is configured) + if (recordHostConfig.blobs) { + const blobsCfg = recordHostConfig.blobs; + const blobAdapter = blobsCfg.adapter; + const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; + const accept = blobsCfg.accept; + + app.post(`/xrpc/${SPACE}.uploadBlob`, authWithCredential, async (c) => { + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, spaceUri, "rw"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "write", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } + } + + const mimeType = c.req.header("content-type") ?? "application/octet-stream"; + if (accept && !accept.includes(mimeType)) { + return c.json( + { error: "InvalidMimeType", message: `MIME type ${mimeType} is not accepted.` }, + 400 + ); + } + + const declaredLen = c.req.header("content-length"); + if (declaredLen && Number(declaredLen) > maxSize) { + return c.json( + { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, + 413 + ); + } + + const buf = await c.req.arrayBuffer(); + const bytes = new Uint8Array(buf); + if (bytes.byteLength > maxSize) { + return c.json( + { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, + 413 + ); + } + + const cid = await createCid(0x55, bytes); + const cidString = cidToString(cid); + const key = await blobKey(spaceUri, cidString); + + await blobAdapter.put(key, bytes, { mimeType, size: bytes.byteLength }); + await recordHost.putBlobMeta({ + spaceUri, + cid: cidString, + mimeType, + size: bytes.byteLength, + authorDid: caller.callerDid, + createdAt: Date.now(), + }); + + return c.json({ + blob: { + $type: "blob", + ref: { $link: cidString }, + mimeType, + size: bytes.byteLength, + }, + }); + }); + + app.get(`/xrpc/${SPACE}.getBlob`, readAuth, async (c) => { + const spaceUri = c.req.query("spaceUri"); + const cid = c.req.query("cid"); + if (!spaceUri || !cid) { + return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const authz = await authorizeRead(c, authority, spaceUri); + if (authz instanceof Response) return authz; + + if (authz.via === "jwt") { + const sa = authz.sa; + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, sa.issuer); + const aclResult = checkAccess({ + op: "read", + space, + callerDid: sa.issuer, + member, + clientId: sa.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } + } + + const meta = await recordHost.getBlobMeta(spaceUri, cid); + if (!meta) return c.json({ error: "NotFound" }, 404); + const key = await blobKey(spaceUri, cid); + const bytes = await blobAdapter.get(key); + if (!bytes) return c.json({ error: "NotFound" }, 404); + + return new Response(bytes as BodyInit, { + headers: { + "content-type": meta.mimeType, + "content-length": String(meta.size), + }, + }); + }); + + app.get(`/xrpc/${SPACE}.listBlobs`, authWithCredential, async (c) => { + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) { + return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); + } + const enrollment = await requireEnrollment(c, spaceUri); + if (enrollment instanceof Response) return enrollment; + + const caller = resolveCaller(c, spaceUri, "read"); + if (caller instanceof Response) return caller; + + if (!caller.viaCredential) { + const space = await authority.getSpace(spaceUri); + if (!space) return c.json({ error: "NotFound" }, 404); + const member = await authority.getMember(spaceUri, caller.callerDid); + const aclResult = checkAccess({ + op: "read", + space, + callerDid: caller.callerDid, + member, + clientId: caller.clientId, + }); + if (!aclResult.allow) { + return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); + } + } + + const result = await recordHost.listBlobMeta(spaceUri, { + byUser: c.req.query("byUser") ?? undefined, + cursor: c.req.query("cursor") ?? undefined, + limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, + }); + return c.json(result); + }); + } +} + +/** Authorize a read request — three valid paths: a verified space credential + * (set by the credential middleware), a read-grant invite token, or a + * service-auth JWT. */ +async function authorizeRead( + c: Context, + authority: SpaceAuthority, + spaceUri: string +): Promise< + | { via: "credential"; claims: CredentialClaims } + | { via: "token" } + | { via: "jwt"; sa: ServiceAuth } + | Response +> { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== spaceUri) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + return { via: "credential", claims: cred }; + } + const rawToken = extractInviteToken(c.req.raw); + if (rawToken) { + const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); + if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); + return { via: "token" }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (sa) return { via: "jwt", sa }; + return c.json( + { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, + 401 + ); +} + +function resolveCaller( + c: Context, + requestSpace: string, + requiredScope: CredentialScope +): { callerDid: string; clientId: string | undefined; viaCredential: boolean } | Response { + const cred = c.get("spaceCredential") as CredentialClaims | undefined; + if (cred) { + if (cred.space !== requestSpace) { + return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); + } + if (requiredScope === "rw" && cred.scope !== "rw") { + return c.json({ error: "Forbidden", reason: "credential-wrong-scope" }, 403); + } + return { callerDid: cred.sub, clientId: undefined, viaCredential: true }; + } + const sa = c.get("serviceAuth") as ServiceAuth | undefined; + if (!sa) return c.json({ error: "AuthRequired", reason: "no-auth" }, 401); + return { callerDid: sa.issuer, clientId: sa.clientId, viaCredential: false }; +} + +function getAuth(c: Context): ServiceAuth { + const auth = c.get("serviceAuth") as ServiceAuth | undefined; + if (!auth) throw new Error("service auth not set"); + return auth; +} diff --git a/packages/contrail-record-host/src/schema.ts b/packages/contrail-record-host/src/schema.ts new file mode 100644 index 0000000..881d16b --- /dev/null +++ b/packages/contrail-record-host/src/schema.ts @@ -0,0 +1,51 @@ +/** Record-host DDL: `spaces_blobs`, `record_host_enrollments`, plus the + * per-collection `spaces_records_<short>` tables. The latter are config- + * driven so they're built via the shared collection-table helper from + * contrail-base, not declared here directly. */ + +import type { Database, ContrailConfig, SqlDialect } from "@atmo-dev/contrail-base"; +import { getDialect } from "@atmo-dev/contrail-base"; + +/** The fixed host tables — independent of the user's collection config. */ +export function buildRecordHostBaseSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS spaces_blobs ( + space_uri TEXT NOT NULL, + cid TEXT NOT NULL, + mime_type TEXT NOT NULL, + size INTEGER NOT NULL, + author_did TEXT NOT NULL, + created_at ${dialect.bigintType} NOT NULL, + PRIMARY KEY (space_uri, cid) + )`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_author ON spaces_blobs(space_uri, author_did)`, + `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_created ON spaces_blobs(space_uri, created_at)`, + + // Local cache: spaces this host has agreed to store records for, and + // which authority signs credentials for each. Filled by the + // recordHost.enroll endpoint, or auto-populated by the colocated + // authority's createSpace. + `CREATE TABLE IF NOT EXISTS record_host_enrollments ( + space_uri TEXT PRIMARY KEY, + authority_did TEXT NOT NULL, + enrolled_at ${dialect.bigintType} NOT NULL, + enrolled_by TEXT NOT NULL + )`, + `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, + ]; +} + +/** SchemaModule-shaped function suitable for `initSchema({ extraSchemas: [...] })`. + * Applies the host tables that don't depend on the collection config. The + * per-collection `spaces_records_<short>` tables are still applied through + * the appview's initSchema path (which knows about collections). */ +export async function applyRecordHostSchema(db: Database): Promise<void> { + const dialect = getDialect(db); + const stmts = buildRecordHostBaseSchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); +} + +// Re-export for callers that want a config-driven schema (per-collection +// tables plus blobs/enrollments). Kept as a convenience for split +// deployments where the host doesn't share a DB with the appview. +export type { ContrailConfig }; diff --git a/packages/contrail-record-host/tsconfig.build.json b/packages/contrail-record-host/tsconfig.build.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-record-host/tsconfig.build.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-record-host/tsconfig.json b/packages/contrail-record-host/tsconfig.json new file mode 100644 index 0000000..6092abf --- /dev/null +++ b/packages/contrail-record-host/tsconfig.json @@ -0,0 +1,7 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "lib": ["ES2022", "DOM"] + }, + "include": ["src"] +} diff --git a/packages/contrail-record-host/tsup.config.ts b/packages/contrail-record-host/tsup.config.ts new file mode 100644 index 0000000..7d6b7eb --- /dev/null +++ b/packages/contrail-record-host/tsup.config.ts @@ -0,0 +1,11 @@ +import { defineConfig } from "tsup"; + +export default defineConfig({ + entry: ["src/index.ts"], + format: ["esm"], + dts: true, + sourcemap: true, + clean: true, + tsconfig: "tsconfig.build.json", + external: ["@atmo-dev/contrail-base"], +}); diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 02a833f..4a3e79b 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -73,8 +73,10 @@ "@atcute/jetstream": "^1.0.2", "@atcute/lexicons": "^1.2.9", "@atcute/xrpc-server": "^0.1.12", + "@atmo-dev/contrail-appview": "workspace:*", "@atmo-dev/contrail-authority": "workspace:*", "@atmo-dev/contrail-base": "workspace:*", + "@atmo-dev/contrail-record-host": "workspace:*", "cac": "^7.0.0", "hono": "^4.12.8", "jiti": "^2.4.0" diff --git a/packages/contrail/src/core/backfill.ts b/packages/contrail/src/core/backfill.ts index 713e23e..551c495 100644 --- a/packages/contrail/src/core/backfill.ts +++ b/packages/contrail/src/core/backfill.ts @@ -1,523 +1 @@ -import type {} from "@atcute/atproto"; -import { type Did } from "@atcute/lexicons"; -import { isDid, isNsid } from "@atcute/lexicons/syntax"; - -import type { Client } from "@atcute/client"; -import type { ContrailConfig, Database, IngestEvent } from "./types"; -import { getDiscoverableNsids, getDependentNsids, DEFAULT_RELAYS } from "./types"; -import { applyEvents, getLastCursor, saveCursor } from "./db"; -import { getClient, getPDS } from "./client"; - -const PAGE_SIZE = 100; -const BATCH_SIZE = 100; -const MAX_RETRIES = 5; - -const REQUEST_TIMEOUT_MS = 10_000; - -async function withRetry<T>( - fn: () => Promise<T>, - label: string, - maxRetries = 3, - timeoutMs = REQUEST_TIMEOUT_MS -): Promise<T> { - let lastError: unknown; - for (let attempt = 0; attempt <= maxRetries; attempt++) { - try { - return await Promise.race([ - fn(), - new Promise<never>((_, reject) => - setTimeout(() => reject(new Error(`Timeout: ${label}`)), timeoutMs) - ), - ]); - } catch (err) { - lastError = err; - if (attempt < maxRetries) { - const delay = Math.min(1000 * 2 ** attempt, 10000); - await new Promise((r) => setTimeout(r, delay)); - } - } - } - throw lastError; -} - -async function markFailed( - db: Database, - did: string, - collection: string, - error: string -): Promise<void> { - await db - .prepare( - "UPDATE backfills SET retries = retries + 1, last_error = ? WHERE did = ? AND collection = ?" - ) - .bind(error, did, collection) - .run(); -} - -export interface BackfillOptions { - /** Pre-resolved client — avoids redundant PDS lookups when batching by DID */ - client?: Client; - /** Skip replay detection in applyEvents (safe during initial backfill) */ - skipReplayDetection?: boolean; - /** Max retries per request (default: 3). Set to 0 for single-attempt mode. */ - maxRetries?: number; - /** Per-request timeout in ms (default: 10000). */ - requestTimeout?: number; -} - -export async function backfillUser( - db: Database, - did: string, - collection: string, - deadline: number, - config?: ContrailConfig, - options?: BackfillOptions -): Promise<number> { - if (Date.now() >= deadline) return 0; - - const status = await db - .prepare( - "SELECT completed, pds_cursor, retries FROM backfills WHERE did = ? AND collection = ?" - ) - .bind(did, collection) - .first<{ completed: number; pds_cursor: string | null; retries: number }>(); - - if (status?.completed) return 0; - - if (!status) { - await db - .prepare( - "INSERT INTO backfills (did, collection, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" - ) - .bind(did, collection) - .run(); - } - - let currentCursor: string | undefined = status?.pds_cursor ?? undefined; - const retries = options?.maxRetries ?? 3; - const timeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; - - if (!isDid(did)) { - await markFailed(db, did, collection, `Invalid DID: ${did}`); - return 0; - } - - if (!isNsid(collection)) { - await markFailed(db, did, collection, `Invalid NSID: ${collection}`); - return 0; - } - - let client = options?.client; - if (!client) { - try { - client = await withRetry( - () => getClient(did as Did, db), - `getClient(${did})`, - Math.min(retries, 1), - timeout - ); - } catch (err) { - await markFailed(db, did, collection, String(err)); - return 0; - } - } - - let totalInserted = 0; - let done = false; - - try { - while (Date.now() < deadline) { - const response = await withRetry( - () => - client!.get("com.atproto.repo.listRecords", { - params: { - repo: did as Did, - collection, - limit: PAGE_SIZE, - cursor: currentCursor, - }, - }), - `listRecords(${did}/${collection})`, - retries, - timeout - ); - if (!response.ok) { - await markFailed( - db, - did, - collection, - `listRecords status ${response.status}` - ); - return totalInserted; - } - - if (response.data.records.length === 0) { - done = true; - break; - } - - const now = Date.now(); - const events: IngestEvent[] = response.data.records.map((r) => ({ - uri: r.uri, - did, - collection, - rkey: r.uri.split("/").pop()!, - operation: "create" as const, - cid: r.cid, - record: JSON.stringify(r.value), - time_us: now * 1000, - indexed_at: now * 1000, - })); - - await applyEvents(db, events, config, { - skipReplayDetection: options?.skipReplayDetection, - skipFeedFanout: true, - }); - totalInserted += events.length; - - currentCursor = response.data.cursor ?? undefined; - - await db - .prepare( - "UPDATE backfills SET pds_cursor = ? WHERE did = ? AND collection = ?" - ) - .bind(currentCursor ?? null, did, collection) - .run(); - - if (!currentCursor) { - done = true; - break; - } - } - } catch (err) { - await markFailed(db, did, collection, String(err)); - return totalInserted; - } - - if (done) { - await db - .prepare( - "UPDATE backfills SET completed = 1 WHERE did = ? AND collection = ?" - ) - .bind(did, collection) - .run(); - } - - return totalInserted; -} - -// --- Bulk backfill (groups by DID, resolves client once) --- - -export interface BackfillProgress { - records: number; - usersComplete: number; - usersTotal: number; - usersFailed: number; -} - -export interface BackfillAllOptions { - concurrency?: number; - onProgress?: (progress: BackfillProgress) => void; -} - -export async function backfillPending( - db: Database, - config: ContrailConfig, - options?: BackfillAllOptions -): Promise<number> { - const concurrency = options?.concurrency ?? 100; - let totalBackfilled = 0; - - // Anchor the jetstream cursor to now if it hasn't been set yet, so records - // emitted during backfill are replayed once jetstream starts. - if ((await getLastCursor(db)) === null) { - await saveCursor(db, Date.now() * 1000); - } - - // Reset retries so users that hit the cap in a prior run get another chance. - await db - .prepare("UPDATE backfills SET retries = 0 WHERE completed = 0") - .run(); - - while (true) { - const pending = await db - .prepare( - "SELECT did, collection FROM backfills WHERE completed = 0 AND retries < ? ORDER BY did" - ) - .bind(MAX_RETRIES) - .all<{ did: string; collection: string }>(); - - const rows = pending.results ?? []; - if (rows.length === 0) break; - - // Group by DID so we resolve PDS once per user - const byDid = new Map<string, string[]>(); - for (const row of rows) { - const cols = byDid.get(row.did) ?? []; - cols.push(row.collection); - byDid.set(row.did, cols); - } - - const dids = [...byDid.keys()]; - - // Resolve PDS endpoints in background (populates in-memory cache) - const resolvePromise = (async () => { - for (let i = 0; i < dids.length; i += 200) { - await Promise.allSettled( - dids.slice(i, i + 200).map((did) => - getPDS(did as Did, db).catch(() => {}) - ) - ); - } - })(); - - let roundBackfilled = 0; - let usersComplete = 0; - let usersFailed = 0; - const failedDids: string[] = []; - - const FAST_TIMEOUT = 3_000; - - const emitProgress = () => - options?.onProgress?.({ - records: totalBackfilled + roundBackfilled, - usersComplete, - usersTotal: dids.length, - usersFailed, - }); - - // Fast pass: single attempt per user with short timeout - for (let i = 0; i < dids.length; i += concurrency) { - const batch = dids.slice(i, i + concurrency); - - const results = await Promise.allSettled( - batch.map(async (did) => { - let client: Client | undefined; - try { - client = await withRetry( - () => getClient(did as Did, db), - `getClient(${did})`, - 0, - FAST_TIMEOUT - ); - } catch { - failedDids.push(did); - return 0; - } - - const cols = byDid.get(did)!; - const counts = await Promise.all( - cols.map((col) => - backfillUser(db, did, col, Infinity, config, { - client, - skipReplayDetection: true, - maxRetries: 0, - requestTimeout: FAST_TIMEOUT, - }).catch(() => { - failedDids.push(did); - return 0; - }) - ) - ); - - usersComplete++; - return counts.reduce((a, b) => a + b, 0); - }) - ); - - for (const r of results) { - if (r.status === "fulfilled") roundBackfilled += r.value; - } - - emitProgress(); - } - - // Retry pass: failed DIDs get retries with backoff, still in concurrent batches - if (failedDids.length > 0) { - const uniqueFailed = [...new Set(failedDids)]; - usersComplete -= uniqueFailed.length; // don't count them yet - - for (let i = 0; i < uniqueFailed.length; i += concurrency) { - const batch = uniqueFailed.slice(i, i + concurrency); - - const results = await Promise.allSettled( - batch.map(async (did) => { - let client: Client | undefined; - try { - client = await withRetry( - () => getClient(did as Did, db), - `getClient(${did})`, - 2 - ); - } catch (err) { - for (const col of byDid.get(did)!) { - await markFailed(db, did, col, String(err)); - } - usersFailed++; - usersComplete++; - return 0; - } - - const cols = byDid.get(did)!; - const counts = await Promise.all( - cols.map((col) => - backfillUser(db, did, col, Infinity, config, { - client, - skipReplayDetection: true, - maxRetries: 2, - }) - ) - ); - usersComplete++; - return counts.reduce((a, b) => a + b, 0); - }) - ); - - for (const r of results) { - if (r.status === "fulfilled") roundBackfilled += r.value; - } - - emitProgress(); - } - } - - await resolvePromise; - totalBackfilled += roundBackfilled; - - // If nothing was backfilled this round, we're stuck - if (roundBackfilled === 0) break; - } - - return totalBackfilled; -} - -// --- Discovery --- - -interface DiscoveryPage { - repos: { did: string }[]; - cursor?: string; -} - -async function fetchPage( - relay: string, - collection: string, - cursor?: string -): Promise<DiscoveryPage | null> { - const url = new URL( - `/xrpc/com.atproto.sync.listReposByCollection`, - relay - ); - url.searchParams.set("collection", collection); - url.searchParams.set("limit", "1000"); - if (cursor) { - url.searchParams.set("cursor", cursor); - } - - try { - return await withRetry( - async () => { - const response = await fetch(url.toString()); - if (!response.ok) { - throw new Error(`HTTP ${response.status}`); - } - return (await response.json()) as DiscoveryPage; - }, - `fetchPage(${relay}, ${collection})` - ); - } catch (err) { - // Discovery page fetch failed after retries — skip this relay - return null; - } -} - -async function insertDiscoveredDIDs( - db: Database, - dids: string[], - collection: string -): Promise<void> { - if (dids.length === 0) return; - - // Use multi-row INSERT to reduce the number of statements - const CHUNK_SIZE = 50; - for (let i = 0; i < dids.length; i += CHUNK_SIZE) { - const chunk = dids.slice(i, i + CHUNK_SIZE); - const placeholders = chunk.map(() => "(?, ?, 0)").join(", "); - const bindings: string[] = []; - for (const did of chunk) { - bindings.push(did, collection); - } - await db - .prepare( - `INSERT INTO backfills (did, collection, completed) VALUES ${placeholders} ON CONFLICT DO NOTHING` - ) - .bind(...bindings) - .run(); - } -} - -async function saveDiscoveryState( - db: Database, - collection: string, - relay: string, - cursor: string | null, - completed: boolean -): Promise<void> { - await db - .prepare( - "INSERT INTO discovery (collection, relay, cursor, completed) VALUES (?, ?, ?, ?) ON CONFLICT(collection, relay) DO UPDATE SET cursor = excluded.cursor, completed = excluded.completed" - ) - .bind(collection, relay, cursor, completed ? 1 : 0) - .run(); -} - -export async function discoverDIDs( - db: Database, - config: ContrailConfig, - deadline: number -): Promise<string[]> { - const collections = getDiscoverableNsids(config); - const relays = config.relays ?? DEFAULT_RELAYS; - if (relays.length === 0 || collections.length === 0) return []; - - const discovered: string[] = []; - - for (const collection of collections) { - if (Date.now() >= deadline) break; - - let data: DiscoveryPage | null = null; - let relay: string | null = null; - - for (const r of relays) { - const row = await db - .prepare( - "SELECT cursor, completed FROM discovery WHERE collection = ? AND relay = ?" - ) - .bind(collection, r) - .first<{ cursor: string | null; completed: number }>(); - - if (row?.completed) continue; - - data = await fetchPage(r, collection, row?.cursor ?? undefined); - if (data) { - relay = r; - break; - } else { - await saveDiscoveryState(db, collection, r, null, true); - } - } - if (!data || !relay) continue; - - const dids = data.repos?.map((r) => r.did) ?? []; - await insertDiscoveredDIDs(db, dids, collection); - discovered.push(...dids); - - for (const depCollection of getDependentNsids(config)) { - await insertDiscoveredDIDs(db, dids, depCollection); - } - - const completed = !data.cursor; - await saveDiscoveryState(db, collection, relay, data.cursor ?? null, completed); - } - - return discovered; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/client.ts b/packages/contrail/src/core/client.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/client.ts +++ b/packages/contrail/src/core/client.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/community-integration.ts b/packages/contrail/src/core/community-integration.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/community-integration.ts +++ b/packages/contrail/src/core/community-integration.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/db/index.ts b/packages/contrail/src/core/db/index.ts index 2469f31..551c495 100644 --- a/packages/contrail/src/core/db/index.ts +++ b/packages/contrail/src/core/db/index.ts @@ -1,4 +1 @@ -export { initSchema } from "./schema"; -export { getLastCursor, saveCursor, applyEvents, lookupExistingRecords, queryRecords, queryAcrossSources, pruneFeedItems } from "./records"; -export type { QueryOptions, SortOption, ExistingRecordInfo } from "./records"; -export type { RecordSource } from "../types"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/db/records.ts b/packages/contrail/src/core/db/records.ts index a45ab9b..551c495 100644 --- a/packages/contrail/src/core/db/records.ts +++ b/packages/contrail/src/core/db/records.ts @@ -1,896 +1 @@ -import type { - ContrailConfig, - ResolvedContrailConfig, - RelationConfig, - Database, - Statement, - IngestEvent, - RecordRow, - RecordSource, -} from "../types"; -import { - getNestedValue, - getRelationField, - countColumnName, - groupedCountColumnName, - getFeedFollowShortNames, - recordsTableName, - spacesRecordsTableName, - shortNameForNsid, - nsidForShortName, -} from "../types"; -import { getSearchableFields, ftsTableName, buildFtsContent } from "../search"; -import { ftsQueryClause, getDialect } from "../dialect"; - -// --- Counts --- - -interface InboundRelation { - /** Short name of the parent collection. */ - parentCollection: string; - relationName: string; - rel: RelationConfig; -} - -/** Find relations that target the given short-named child collection. */ -function getInboundRelations( - config: ContrailConfig, - childShortName: string -): InboundRelation[] { - const results: InboundRelation[] = []; - for (const [colName, colConfig] of Object.entries(config.collections)) { - for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { - if (rel.collection === childShortName) { - results.push({ parentCollection: colName, relationName: relName, rel }); - } - } - } - return results; -} - -/** - * Collect recount targets from a single event into a shared map. - * The map is keyed by `parentCollection:relationName:targetValue` to deduplicate - * across the entire batch — so 50 RSVPs to the same event produce one recount, not 50. - */ -function collectCountTargets( - event: IngestEvent, - config: ContrailConfig, - existingRecordJson: string | null, - targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> -): void { - const childShort = shortNameForNsid(config, event.collection); - if (!childShort) return; - const inbound = getInboundRelations(config, childShort); - if (inbound.length === 0) return; - - const record = event.record ? JSON.parse(event.record) : null; - const existingRecord = existingRecordJson ? JSON.parse(existingRecordJson) : null; - - for (const { parentCollection, relationName, rel } of inbound) { - if (rel.count === false) continue; - - const field = getRelationField(rel); - - const values: string[] = []; - if (record) { - const t = getNestedValue(record, field); - if (t) values.push(t); - } - if (existingRecord) { - const t = getNestedValue(existingRecord, field); - if (t && !values.includes(t)) values.push(t); - } - - for (const targetValue of values) { - const key = `${parentCollection}:${relationName}:${targetValue}`; - if (!targets.has(key)) { - targets.set(key, { parentCollection, relationName, rel, targetValue }); - } - } - } -} - -/** - * Build deduplicated count UPDATE statements from collected targets. - * One UPDATE per unique parent+relation+target, regardless of how many - * events in the batch affected that target. - */ -function buildBatchCountStatements( - db: Database, - config: ContrailConfig, - targets: Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }> -): Statement[] { - const statements: Statement[] = []; - - for (const { parentCollection, relationName, rel, targetValue } of targets.values()) { - const field = getRelationField(rel); - const matchColumn = rel.match === "did" ? "did" : "uri"; - const childTable = recordsTableName(rel.collection); - const parentTable = recordsTableName(parentCollection); - - const setClauses: string[] = []; - const setBindings: (string | number)[] = []; - - const countExpr = rel.countDistinct - ? `COUNT(DISTINCT ${rel.countDistinct})` - : "COUNT(*)"; - - // Total count - const totalCol = countColumnName(rel.collection); - setClauses.push( - `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ?)` - ); - setBindings.push(targetValue); - - // Grouped counts — column names are `count_<child-short>_<group-key>`; match - // against the group's full token value in the record. - if (rel.groupBy) { - const mapping = (config as ResolvedContrailConfig)._resolved?.relations[parentCollection]?.[relationName]; - if (mapping?.groups) { - for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - setClauses.push( - `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE ${getDialect(db).jsonExtract('record', field)} = ? AND ${getDialect(db).jsonExtract('record', rel.groupBy)} = ?)` - ); - setBindings.push(targetValue, fullToken); - } - } - } - - if (setClauses.length > 0) { - statements.push( - db - .prepare( - `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE ${matchColumn} = ?` - ) - .bind(...setBindings, targetValue) - ); - } - } - - return statements; -} - -// --- FTS --- - -function buildFtsStatements( - db: Database, - event: IngestEvent, - config: ContrailConfig, - existingMap: Map<string, ExistingRecordInfo> -): Statement[] { - // PostgreSQL: tsvector generated column is auto-maintained, no manual FTS sync - if (getDialect(db).ftsStrategy === "generated-column") return []; - - const short = shortNameForNsid(config, event.collection); - if (!short) return []; - const colConfig = config.collections[short]; - if (!colConfig) return []; - - const fields = getSearchableFields(short, colConfig); - if (!fields || fields.length === 0) return []; - - const table = ftsTableName(short); - const stmts: Statement[] = []; - - if (event.operation === "delete") { - stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); - } else { - const record = event.record ? JSON.parse(event.record) : null; - if (!record) return []; - - const content = buildFtsContent(record, fields); - if (!content) return []; - - // Only delete existing FTS row if this is an update (record already existed) - if (existingMap.has(event.uri)) { - stmts.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(event.uri)); - } - stmts.push( - db.prepare(`INSERT INTO ${table} (uri, content) VALUES (?, ?)`).bind(event.uri, content) - ); - } - - return stmts; -} - -// --- Feeds --- - -function buildFeedStatements( - db: Database, - event: IngestEvent, - config: ContrailConfig, - existingRecords: Map<string, string | null> -): Statement[] { - if (!config.feeds) return []; - - const stmts: Statement[] = []; - - const eventShort = shortNameForNsid(config, event.collection); - if (!eventShort) return []; - - for (const [, feedConfig] of Object.entries(config.feeds)) { - const followTable = recordsTableName(feedConfig.follow); - - // Target collection: fan out to followers - if (feedConfig.targets.includes(eventShort)) { - if (event.operation === "create" || event.operation === "update") { - stmts.push( - db - .prepare( - getDialect(db).insertOrIgnore( - `INSERT INTO feed_items (actor, uri, collection, time_us) - SELECT r.did, ?, ?, ? - FROM ${followTable} r - WHERE ${getDialect(db).jsonExtract('r.record', 'subject')} = ?` - ) - ) - .bind(event.uri, event.collection, event.time_us, event.did) - ); - } else if (event.operation === "delete") { - stmts.push( - db.prepare("DELETE FROM feed_items WHERE uri = ?").bind(event.uri) - ); - } - } - - // Follow collection: handle follow/unfollow - if (eventShort === feedConfig.follow) { - if (event.operation === "create") { - const record = event.record ? JSON.parse(event.record) : null; - const subject = record?.subject; - if (subject) { - for (const targetShort of feedConfig.targets) { - const targetTable = recordsTableName(targetShort); - const targetNsid = nsidForShortName(config, targetShort) ?? targetShort; - stmts.push( - db - .prepare( - getDialect(db).insertOrIgnore( - `INSERT INTO feed_items (actor, uri, collection, time_us) - SELECT ?, r.uri, ?, r.time_us - FROM ${targetTable} r - WHERE r.did = ? - ORDER BY r.time_us DESC - LIMIT 100` - ) - ) - .bind(event.did, targetNsid, subject) - ); - } - } - } else if (event.operation === "delete") { - const existingRecord = existingRecords.get(event.uri); - if (existingRecord) { - const parsed = JSON.parse(existingRecord); - const subject = parsed?.subject; - if (subject) { - for (const targetShort of feedConfig.targets) { - const targetTable = recordsTableName(targetShort); - stmts.push( - db - .prepare( - `DELETE FROM feed_items WHERE actor = ? AND uri IN ( - SELECT uri FROM ${targetTable} WHERE did = ? - )` - ) - .bind(event.did, subject) - ); - } - } - } - } - } - } - - return stmts; -} - -// --- Feed pruning --- - -export async function pruneFeedItems( - db: Database, - maxItems: number -): Promise<number> { - const result = await db - .prepare( - `DELETE FROM feed_items WHERE (actor, uri) NOT IN ( - SELECT actor, uri FROM ( - SELECT actor, uri, ROW_NUMBER() OVER (PARTITION BY actor ORDER BY time_us DESC) as rn - FROM feed_items - ) sub WHERE rn <= ? - )` - ) - .bind(maxItems) - .run(); - return (result as any)?.changes ?? 0; -} - -// --- Cursor --- - -export async function getLastCursor(db: Database): Promise<number | null> { - const row = await db - .prepare("SELECT time_us FROM cursor WHERE id = 1") - .first<{ time_us: number }>(); - return row ? row.time_us : null; -} - -export async function saveCursor( - db: Database, - timeUs: number -): Promise<void> { - await db - .prepare( - "INSERT INTO cursor (id, time_us) VALUES (1, ?) ON CONFLICT(id) DO UPDATE SET time_us = excluded.time_us" - ) - .bind(timeUs) - .run(); -} - -// --- Existing record lookup --- - -export interface ExistingRecordInfo { - cid: string | null; - record: string | null; - /** When the row was last written to our DB (microseconds). Populated - * whenever `lookupExistingRecords` runs, regardless of `includeRecord`. */ - indexed_at: number | null; -} - -/** - * Look up existing records for a set of events, grouped by collection. - * Returns a map of uri → { cid, record }. - * When includeRecord is false, record will always be null (saves reading large blobs). - */ -export async function lookupExistingRecords( - db: Database, - events: { uri: string; collection: string }[], - includeRecord: boolean = true, - config?: ContrailConfig -): Promise<Map<string, ExistingRecordInfo>> { - const result = new Map<string, ExistingRecordInfo>(); - if (events.length === 0) return result; - - // Group by short name (config lookup); skip events for collections not in our config. - const byShort = new Map<string, string[]>(); - for (const e of events) { - const short = config ? shortNameForNsid(config, e.collection) : e.collection; - if (!short) continue; - const uris = byShort.get(short) ?? []; - uris.push(e.uri); - byShort.set(short, uris); - } - - const selectCols = includeRecord ? "uri, cid, record, indexed_at" : "uri, cid, indexed_at"; - for (const [short, uris] of byShort) { - const table = recordsTableName(short); - for (let i = 0; i < uris.length; i += 50) { - const chunk = uris.slice(i, i + 50); - const placeholders = chunk.map(() => "?").join(","); - const rows = await db - .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri IN (${placeholders})`) - .bind(...chunk) - .all<{ - uri: string; - cid: string | null; - record?: string | null; - indexed_at: number | null; - }>(); - for (const row of rows.results ?? []) { - result.set(row.uri, { - cid: row.cid, - record: includeRecord ? (row.record ?? null) : null, - indexed_at: row.indexed_at ?? null, - }); - } - } - } - - return result; -} - -// --- Events --- - -export async function applyEvents( - db: Database, - events: IngestEvent[], - config?: ContrailConfig, - options?: { - skipReplayDetection?: boolean; - skipFeedFanout?: boolean; - /** Pre-fetched existing records — skips the internal lookup when provided */ - existing?: Map<string, ExistingRecordInfo>; - /** When provided, publish `collection:<nsid>` and `actor:<did>` realtime - * events for each applied event. Space-scoped publishing happens elsewhere - * (see `realtime/publishing-adapter.ts`); public topics carry public - * records only, which is exactly the scope of this function. */ - pubsub?: import("../realtime/types").PubSub; - } -): Promise<void> { - if (events.length === 0) return; - - const followCollections = config ? getFeedFollowShortNames(config) : []; - const hasCountingRelations = config ? Object.values(config.collections).some(c => - Object.values(c.relations ?? {}).some(r => r.count !== false) - ) : false; - const needRecordContent = followCollections.length > 0 || hasCountingRelations; - - // Use pre-fetched data or look up existing records - let existingMap: Map<string, ExistingRecordInfo>; - if (options?.existing) { - existingMap = options.existing; - } else if (config && !options?.skipReplayDetection) { - existingMap = await lookupExistingRecords(db, events, needRecordContent, config); - } else { - existingMap = new Map(); - } - - const batch: Statement[] = []; - - // Build a record-content map for feed statements (needs string values) - const existingRecordStrings = new Map<string, string | null>(); - for (const [uri, info] of existingMap) { - existingRecordStrings.set(uri, info.record); - } - - // Collect all count recount targets across the batch, deduplicated - const countTargets = new Map<string, { parentCollection: string; relationName: string; rel: RelationConfig; targetValue: string }>(); - - for (const e of events) { - // Event's collection is an NSID. Look up the short name from config. - // If no config or not found, treat collection string as-is (for tests that pre-populate tables). - const short = config - ? shortNameForNsid(config, e.collection) ?? (config.collections[e.collection] ? e.collection : null) - : e.collection; - if (!short) { - (config?.logger ?? console).warn( - `[ingest] drop (unknown collection in applyEvents): ${e.operation} ${e.uri} collection=${e.collection}` - ); - continue; - } - const table = recordsTableName(short); - - if (e.operation === "delete") { - batch.push(db.prepare(`DELETE FROM ${table} WHERE uri = ?`).bind(e.uri)); - } else { - batch.push( - db.prepare( - `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, time_us = excluded.time_us, indexed_at = excluded.indexed_at` - ).bind( - e.uri, - e.did, - e.rkey, - e.cid, - e.record, - e.time_us, - e.indexed_at - ) - ); - } - - if (config) { - // Collect count targets (deduplicated across the whole batch) - const existingRecordJson = existingMap.get(e.uri)?.record ?? null; - collectCountTargets(e, config, existingRecordJson, countTargets); - - // Feed fanout still needs replay detection - const existingInfo = existingMap.get(e.uri); - const isReplay = - e.operation === "delete" - ? existingInfo === undefined - : existingInfo?.cid === e.cid; - - if (!isReplay && !options?.skipFeedFanout) { - batch.push(...buildFeedStatements(db, e, config, existingRecordStrings)); - } - batch.push(...buildFtsStatements(db, e, config, existingMap)); - } - } - - // Build deduplicated count statements — one UPDATE per unique target - if (config) { - batch.push(...buildBatchCountStatements(db, config, countTargets)); - } - - await db.batch(batch); - - // Publish realtime events for public records (collection: and actor:). - // Space records publish via the wrapping adapter; this path is public-only. - if (options?.pubsub) { - const pubsub = options.pubsub; - const ts = Date.now(); - for (const e of events) { - if (e.operation === "delete") { - const payload = { - uri: e.uri, - did: e.did, - collection: e.collection, - rkey: e.rkey, - }; - await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.deleted", payload, ts }); - await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.deleted", payload, ts }); - } else { - const record = e.record ? safeParseJson(e.record) : {}; - const payload = { - uri: e.uri, - did: e.did, - collection: e.collection, - rkey: e.rkey, - cid: e.cid, - record, - time_us: e.time_us, - }; - await pubsub.publish({ topic: `collection:${e.collection}`, kind: "record.created", payload, ts }); - await pubsub.publish({ topic: `actor:${e.did}`, kind: "record.created", payload, ts }); - } - } - } -} - -function safeParseJson(s: string): Record<string, unknown> { - try { - const v = JSON.parse(s); - return v && typeof v === "object" && !Array.isArray(v) ? (v as Record<string, unknown>) : {}; - } catch { - return {}; - } -} - -// --- Count columns --- - -/** Count column descriptor. `type` is the identifier returned in API responses and - * accepted in countFilters — we keep the full record token for grouped counts so - * callers pass e.g. "community.lexicon.calendar.rsvp#going" and filter/hydrate by it. */ -function getCountColumns( - config: ContrailConfig, - shortName: string -): { type: string; column: string }[] { - const colConfig = config.collections[shortName]; - if (!colConfig?.relations) return []; - const columns: { type: string; column: string }[] = []; - const relMap = (config as ResolvedContrailConfig)._resolved?.relations[shortName] ?? {}; - - for (const [relName, rel] of Object.entries(colConfig.relations)) { - if (rel.count === false) continue; - // Total: identifier is the child's short name; column is `count_<child-short>`. - columns.push({ type: rel.collection, column: countColumnName(rel.collection) }); - const mapping = relMap[relName]; - if (mapping) { - for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { - // Grouped: identifier is the full record token (stable across deployments); - // column is `count_<child-short>_<group-key>`. - columns.push({ - type: fullToken, - column: groupedCountColumnName(rel.collection, groupKey), - }); - } - } - } - return columns; -} - -/** For a given "count type" (short name or full group token), return the DB column. */ -function countColumnForType( - config: ContrailConfig, - shortName: string, - type: string -): string | null { - for (const col of getCountColumns(config, shortName)) { - if (col.type === type) return col.column; - } - return null; -} - -// --- Query --- - -export interface SortOption { - recordField?: string; - countType?: string; - direction: "asc" | "desc"; -} - -/** Opaque keyset cursor. `t` is the tiebreaker (time_us of the last row), - * `v` is the sort-key value (string for record fields, number for counts), - * `k` identifies the sort so we can reject mismatched cursors. */ -interface CursorPayload { - t: number; - v?: string | number; - k: "time" | string; // "time" | `field:<name>` | `count:<type>` -} - -function sortKind(sort?: SortOption): "time" | string { - if (sort?.recordField) return `field:${sort.recordField}`; - if (sort?.countType) return `count:${sort.countType}`; - return "time"; -} - -function encodeCursor(payload: CursorPayload): string { - return Buffer.from(JSON.stringify(payload), "utf8").toString("base64url"); -} - -function decodeCursor(cursor: string): CursorPayload | null { - try { - const json = Buffer.from(cursor, "base64url").toString("utf8"); - const p = JSON.parse(json); - if (typeof p?.t !== "number" || typeof p?.k !== "string") return null; - return p as CursorPayload; - } catch { - return null; - } -} - -export interface QueryOptions { - collection: string; - did?: string; - limit?: number; - cursor?: string; - filters?: Record<string, string>; - rangeFilters?: Record<string, { min?: string; max?: string }>; - countFilters?: Record<string, number>; - sort?: SortOption; - search?: string; - source?: RecordSource; - /** When set, query the per-space table (`spaces_records_<short>`) instead of the - * public `records_<short>` table, scoped to rows where `space_uri = ?`. */ - spaceUri?: string; -} - -export async function queryRecords( - db: Database, - config: ContrailConfig, - options: QueryOptions -): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { - const { - collection: collectionInput, - did, - limit: rawLimit, - cursor, - filters = {}, - rangeFilters = {}, - countFilters = {}, - sort, - search, - source, - spaceUri, - } = options; - - // Accept either the short name (canonical) or the full NSID for convenience. - const collection = - config.collections[collectionInput] - ? collectionInput - : shortNameForNsid(config, collectionInput) ?? collectionInput; - - const table = spaceUri ? spacesRecordsTableName(collection) : recordsTableName(collection); - const limit = Math.min(Math.max(1, rawLimit ?? 50), 200); - const conditions: string[] = []; - const bindings: (string | number)[] = []; - - if (spaceUri) { - conditions.push("r.space_uri = ?"); - bindings.push(spaceUri); - } - - if (source?.conditions) conditions.push(...source.conditions); - if (source?.params) bindings.push(...source.params); - - const countCols = getCountColumns(config, collection); - - if (did) { - conditions.push("r.did = ?"); - bindings.push(did); - } - - // Opaque keyset cursor encoding { t, v?, k }. Silently ignored if it doesn't - // match the current sort — callers shouldn't mix sort params with stale cursors. - const expectedKind = sortKind(sort); - if (cursor) { - const payload = decodeCursor(cursor); - if (payload && payload.k === expectedKind) { - if (sort?.recordField) { - const sortExpr = getDialect(db).jsonExtract('r.record', sort.recordField); - const cmp = sort.direction === "desc" ? "<" : ">"; - conditions.push(`(${sortExpr} ${cmp} ? OR (${sortExpr} = ? AND r.time_us < ?))`); - const v = payload.v ?? ""; - bindings.push(v as string | number, v as string | number, payload.t); - } else if (sort?.countType) { - const sortCol = countColumnForType(config, collection, sort.countType); - if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); - const cmp = sort.direction === "desc" ? "<" : ">"; - conditions.push(`(r.${sortCol} ${cmp} ? OR (r.${sortCol} = ? AND r.time_us < ?))`); - const v = Number(payload.v ?? 0); - bindings.push(v, v, payload.t); - } else { - conditions.push("r.time_us < ?"); - bindings.push(payload.t); - } - } - } - - for (const [field, value] of Object.entries(filters)) { - conditions.push(`${getDialect(db).jsonExtract('r.record', field)} = ?`); - bindings.push(value); - } - - for (const [field, range] of Object.entries(rangeFilters)) { - if (range.min != null) { - conditions.push(`${getDialect(db).jsonExtract('r.record', field)} >= ?`); - bindings.push(range.min); - } - if (range.max != null) { - conditions.push(`${getDialect(db).jsonExtract('r.record', field)} <= ?`); - bindings.push(range.max); - } - } - - for (const [type, minCount] of Object.entries(countFilters)) { - const col = countColumnForType(config, collection, type); - if (!col) continue; // unknown count type — skip filter - conditions.push(`r.${col} >= ?`); - bindings.push(minCount); - } - - // FTS search. Not supported in space mode yet (would need composite keying - // because the same at-URI can appear in multiple spaces). - let ftsJoin = ""; - let ftsClause: ReturnType<typeof ftsQueryClause> | null = null; - if (search && !spaceUri) { - const colConfig2 = config.collections[collection]; - const fields = colConfig2 ? getSearchableFields(collection, colConfig2) : null; - if (fields && fields.length > 0) { - ftsClause = ftsQueryClause(getDialect(db), recordsTableName(collection)); - ftsJoin = ftsClause.join; - conditions.push(ftsClause.condition); - // SECURITY: `search` is user input bound as a parameter, not interpolated. - bindings.push(search); - } - } - - const where = conditions.length > 0 ? `WHERE ${conditions.join(" AND ")}` : ""; - - const countSelect = countCols.length > 0 - ? ", " + countCols.map(({ column }) => `r.${column}`).join(", ") - : ""; - const select = `r.uri, r.did, r.rkey, r.cid, r.record, r.time_us, r.indexed_at${countSelect}`; - - const join = [source?.joins, ftsJoin].filter(Boolean).join(" "); - - let orderBy: string; - if (sort?.recordField) { - const dir = sort.direction === "desc" ? "DESC" : "ASC"; - orderBy = `${getDialect(db).jsonExtract('r.record', sort.recordField)} ${dir}, r.time_us DESC`; - } else if (sort?.countType) { - const dir = sort.direction === "desc" ? "DESC" : "ASC"; - const sortCol = countColumnForType(config, collection, sort.countType); - if (!sortCol) throw new Error(`Unknown countType: ${sort.countType}`); - orderBy = `r.${sortCol} ${dir}, r.time_us DESC`; - } else if (ftsClause) { - orderBy = `${ftsClause.orderExpr}, r.time_us DESC`; - // PG ts_rank needs the search term bound again for ORDER BY - if (getDialect(db).ftsStrategy === "generated-column" && search) { - bindings.push(search); - } - } else { - orderBy = "r.time_us DESC"; - } - - bindings.push(limit); - - const query = `SELECT ${select} FROM ${table} r ${join} ${where} ORDER BY ${orderBy} LIMIT ?`; - - const result = await db - .prepare(query) - .bind(...bindings) - .all<any>(); - - const nsid = nsidForShortName(config, collection) ?? collection; - const records = (result.results ?? []).map((row: any) => { - const rec: RecordRow & { counts?: Record<string, number> } = { - uri: row.uri, - did: row.did, - collection: nsid, - rkey: row.rkey, - cid: row.cid, - record: row.record, - time_us: row.time_us, - indexed_at: row.indexed_at, - ...(spaceUri ? { space: spaceUri } : {}), - }; - if (countCols.length > 0) { - const counts: Record<string, number> = {}; - for (const { type, column } of countCols) { - const val = row[column]; - if (val != null && val !== 0) counts[type] = val; - } - if (Object.keys(counts).length > 0) rec.counts = counts; - } - return rec; - }); - - const nextCursor = - records.length === limit - ? buildCursor(records[records.length - 1], sort, expectedKind) - : undefined; - - return { records, cursor: nextCursor }; -} - -/** Build an opaque keyset cursor from the last row of a page. */ -function buildCursor( - row: RecordRow & { counts?: Record<string, number> }, - sort: SortOption | undefined, - kind: string -): string { - const t = Number(row.time_us); - if (sort?.recordField) { - const parsed = row.record ? JSON.parse(row.record) : null; - const v = parsed ? getNestedValue(parsed, sort.recordField) : undefined; - return encodeCursor({ t, v: v == null ? "" : String(v), k: kind }); - } - if (sort?.countType) { - const v = row.counts?.[sort.countType] ?? 0; - return encodeCursor({ t, v, k: kind }); - } - return encodeCursor({ t, k: kind }); -} - -/** Compare two rows according to the active sort order. Returns negative if - * `a` should come before `b`, positive otherwise. Matches the SQL ORDER BY. */ -function compareRows( - a: RecordRow & { counts?: Record<string, number> }, - b: RecordRow & { counts?: Record<string, number> }, - sort: SortOption | undefined -): number { - const timeCmp = Number(b.time_us) - Number(a.time_us); // time_us DESC - if (sort?.recordField) { - const ar = a.record ? JSON.parse(a.record) : null; - const br = b.record ? JSON.parse(b.record) : null; - const av = ar ? getNestedValue(ar, sort.recordField) : undefined; - const bv = br ? getNestedValue(br, sort.recordField) : undefined; - const dir = sort.direction === "desc" ? -1 : 1; - const cmp = (av === bv ? 0 : (av! < bv! ? -1 : 1)) * dir; - return cmp !== 0 ? cmp : timeCmp; - } - if (sort?.countType) { - const av = a.counts?.[sort.countType] ?? 0; - const bv = b.counts?.[sort.countType] ?? 0; - const dir = sort.direction === "desc" ? -1 : 1; - const cmp = (av === bv ? 0 : (av < bv ? -1 : 1)) * dir; - return cmp !== 0 ? cmp : timeCmp; - } - return timeCmp; -} - -/** Run a listRecords query across the public table and a set of per-space tables - * in parallel, then merge according to the active sort order. The cursor is a - * shared keyset cursor — every sub-query applies the same `WHERE` keyset, so - * pagination is consistent across sources. */ -export async function queryAcrossSources( - db: Database, - config: ContrailConfig, - options: QueryOptions, - spaceUris: string[] -): Promise<{ records: (RecordRow & { counts?: Record<string, number> })[]; cursor?: string }> { - if (spaceUris.length === 0) { - return queryRecords(db, config, options); - } - const limit = Math.min(Math.max(1, options.limit ?? 50), 200); - const perSourceLimit = limit; // each source fetches up to `limit`; we trim after merge - - const tasks: Promise<{ records: (RecordRow & { counts?: Record<string, number> })[] }>[] = [ - queryRecords(db, config, { ...options, limit: perSourceLimit }), - ]; - for (const spaceUri of spaceUris) { - tasks.push(queryRecords(db, config, { ...options, spaceUri, limit: perSourceLimit })); - } - const results = await Promise.all(tasks); - const merged = results.flatMap((r) => r.records); - merged.sort((a, b) => compareRows(a, b, options.sort)); - const trimmed = merged.slice(0, limit); - const kind = sortKind(options.sort); - const cursor = - trimmed.length === limit ? buildCursor(trimmed[trimmed.length - 1], options.sort, kind) : undefined; - return { records: trimmed, cursor }; -} - -// --- Users --- - +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/db/schema.ts b/packages/contrail/src/core/db/schema.ts index 562a6ee..551c495 100644 --- a/packages/contrail/src/core/db/schema.ts +++ b/packages/contrail/src/core/db/schema.ts @@ -1,355 +1 @@ -import type { ContrailConfig, Database, ResolvedContrailConfig, ResolvedMaps } from "../types"; -import type { SqlDialect } from "../dialect"; -import { buildFtsSchema, getDialect } from "../dialect"; -import { - getRelationField, - countColumnName, - groupedCountColumnName, - recordsTableName, - spacesRecordsTableName, - resolveConfig, -} from "../types"; -import { getSearchableFields } from "../search"; -import { buildSpacesBaseSchema } from "../spaces/schema"; -import { buildLabelsSchema } from "../labels/schema"; - -function getResolved(config: ContrailConfig): ResolvedMaps { - return (config as ResolvedContrailConfig)._resolved ?? resolveConfig(config)._resolved; -} - -function buildBaseSchema(dialect: SqlDialect): string { - return ` -CREATE TABLE IF NOT EXISTS backfills ( - did TEXT NOT NULL, - collection TEXT NOT NULL, - completed INTEGER NOT NULL DEFAULT 0, - pds_cursor TEXT, - retries INTEGER NOT NULL DEFAULT 0, - last_error TEXT, - PRIMARY KEY (did, collection) -); -CREATE TABLE IF NOT EXISTS discovery ( - collection TEXT NOT NULL, - relay TEXT NOT NULL, - cursor TEXT, - completed INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (collection, relay) -); -CREATE TABLE IF NOT EXISTS cursor ( - id INTEGER PRIMARY KEY CHECK (id = 1), - time_us ${dialect.bigintType} NOT NULL -); -CREATE TABLE IF NOT EXISTS identities ( - did TEXT PRIMARY KEY, - handle TEXT, - pds TEXT, - resolved_at ${dialect.bigintType} NOT NULL -); -CREATE INDEX IF NOT EXISTS idx_identities_handle ON identities(handle); -`; -} - -function sanitizeName(name: string): string { - return name.replace(/[^a-zA-Z0-9]/g, "_"); -} - -interface BuilderOpts { - /** Emit tables for the spaces variant (spaces_records_<short> with space_uri column). */ - forSpaces?: boolean; -} - -function tableFor(shortName: string, opts: BuilderOpts): string { - return opts.forSpaces ? spacesRecordsTableName(shortName) : recordsTableName(shortName); -} - -function namePrefix(opts: BuilderOpts): string { - return opts.forSpaces ? "sp_" : ""; -} - -export function buildCollectionTables( - config: ContrailConfig, - dialect: SqlDialect, - opts: BuilderOpts = {} -): string[] { - const stmts: string[] = []; - for (const [shortName, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const table = tableFor(shortName, opts); - const np = namePrefix(opts); - if (opts.forSpaces) { - stmts.push( - `CREATE TABLE IF NOT EXISTS ${table} ( - space_uri TEXT NOT NULL, - uri TEXT NOT NULL, - did TEXT NOT NULL, - rkey TEXT NOT NULL, - cid TEXT, - record ${dialect.recordColumnType}, - time_us ${dialect.bigintType} NOT NULL, - indexed_at ${dialect.bigintType} NOT NULL, - PRIMARY KEY (space_uri, did, rkey) - )` - ); - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_time ON ${table}(space_uri, time_us DESC)` - ); - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(shortName)}_space_did ON ${table}(space_uri, did)` - ); - } else { - stmts.push( - `CREATE TABLE IF NOT EXISTS ${table} ( - uri TEXT PRIMARY KEY, - did TEXT NOT NULL, - rkey TEXT NOT NULL, - cid TEXT, - record ${dialect.recordColumnType}, - time_us ${dialect.bigintType} NOT NULL, - indexed_at ${dialect.bigintType} NOT NULL - )` - ); - stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_did ON ${table}(did)`); - stmts.push(`CREATE INDEX IF NOT EXISTS idx_${sanitizeName(shortName)}_time ON ${table}(time_us DESC)`); - } - } - return stmts; -} - -export function buildDynamicIndexes( - config: ContrailConfig, - dialect: SqlDialect, - opts: BuilderOpts = {} -): string[] { - const resolved = getResolved(config); - const indexes: string[] = []; - const np = namePrefix(opts); - for (const [collection, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const table = tableFor(collection, opts); - const queryable = resolved.queryable[collection] ?? colConfig.queryable ?? {}; - for (const field of Object.keys(queryable)) { - const idxName = `idx_${np}${sanitizeName(collection)}_${sanitizeName(field)}`; - indexes.push( - `CREATE INDEX IF NOT EXISTS ${idxName} ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', field))})` - ); - } - - for (const [, rel] of Object.entries(colConfig.relations ?? {})) { - const childShort = rel.collection; - const childConfig = config.collections[childShort]; - if (opts.forSpaces && childConfig?.allowInSpaces === false) continue; - const on = getRelationField(rel); - const childTable = tableFor(childShort, opts); - const idxName = `idx_${np}${sanitizeName(childShort)}_${sanitizeName(on)}`; - indexes.push( - `CREATE INDEX IF NOT EXISTS ${idxName} ON ${childTable}(${dialect.indexExpression(dialect.jsonExtract('record', on))})` - ); - } - } - return indexes; -} - -export function buildCountColumns(config: ContrailConfig, opts: BuilderOpts = {}): string[] { - const resolved = getResolved(config); - const stmts: string[] = []; - const addedColumns = new Map<string, Set<string>>(); - const np = namePrefix(opts); - - for (const [collection, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const table = tableFor(collection, opts); - const relMap = resolved.relations[collection] ?? {}; - - if (!addedColumns.has(table)) addedColumns.set(table, new Set()); - const tableColumns = addedColumns.get(table)!; - - for (const [relName, rel] of Object.entries(colConfig.relations ?? {})) { - if (rel.count === false) continue; - if (opts.forSpaces && config.collections[rel.collection]?.allowInSpaces === false) continue; - const totalCol = countColumnName(rel.collection); - if (!tableColumns.has(totalCol)) { - tableColumns.add(totalCol); - stmts.push( - `ALTER TABLE ${table} ADD COLUMN ${totalCol} INTEGER NOT NULL DEFAULT 0` - ); - } - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${totalCol} ON ${table}(${totalCol} DESC, time_us DESC)` - ); - - const mapping = relMap[relName]; - if (mapping) { - for (const groupKey of Object.keys(mapping.groups)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - if (!tableColumns.has(groupCol)) { - tableColumns.add(groupCol); - stmts.push( - `ALTER TABLE ${table} ADD COLUMN ${groupCol} INTEGER NOT NULL DEFAULT 0` - ); - } - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${np}${sanitizeName(collection)}_${groupCol} ON ${table}(${groupCol} DESC, time_us DESC)` - ); - } - } - } - } - return stmts; -} - -function buildFeedTables(config: ContrailConfig, dialect: SqlDialect): string[] { - if (!config.feeds || Object.keys(config.feeds).length === 0) return []; - const stmts = [ - `CREATE TABLE IF NOT EXISTS feed_items ( - actor TEXT NOT NULL, - uri TEXT NOT NULL, - collection TEXT NOT NULL, - time_us ${dialect.bigintType} NOT NULL, - PRIMARY KEY (actor, uri) - )`, - `CREATE INDEX IF NOT EXISTS idx_feed_actor_coll_time ON feed_items(actor, collection, time_us DESC)`, - `CREATE INDEX IF NOT EXISTS idx_feed_actor_time ON feed_items(actor, time_us DESC)`, - `CREATE TABLE IF NOT EXISTS feed_backfills ( - actor TEXT NOT NULL, - feed TEXT NOT NULL, - completed INTEGER NOT NULL DEFAULT 0, - PRIMARY KEY (actor, feed) - )`, - ]; - - const followCollections = new Set(Object.values(config.feeds).map((f) => f.follow)); - for (const col of followCollections) { - const table = recordsTableName(col); - const safe = sanitizeName(col); - stmts.push( - `CREATE INDEX IF NOT EXISTS idx_${safe}_subject ON ${table}(${dialect.indexExpression(dialect.jsonExtract('record', 'subject'))})` - ); - } - - return stmts; -} - -export function buildFtsTables( - config: ContrailConfig, - dialect: SqlDialect, - opts: BuilderOpts = {} -): string[] { - const stmts: string[] = []; - for (const [collection, colConfig] of Object.entries(config.collections)) { - if (opts.forSpaces && colConfig.allowInSpaces === false) continue; - const fields = getSearchableFields(collection, colConfig); - if (!fields || fields.length === 0) continue; - const table = tableFor(collection, opts); - stmts.push(...buildFtsSchema(dialect, table, fields)); - } - return stmts; -} - -const MIGRATIONS = [ - "ALTER TABLE backfills ADD COLUMN retries INTEGER NOT NULL DEFAULT 0", - "ALTER TABLE backfills ADD COLUMN last_error TEXT", - "ALTER TABLE spaces_invites ADD COLUMN kind TEXT NOT NULL DEFAULT 'join'", -]; - -async function runMigrations(db: Database): Promise<void> { - for (const sql of MIGRATIONS) { - try { - await db.prepare(sql).run(); - } catch { - // Column already exists — ignore - } - } -} - -/** Pluggable schema applier — passed in by extension packages (community, - * third-party plugins) to install their own tables alongside contrail's. */ -export type SchemaModule = (db: Database) => Promise<void>; - -export interface InitSchemaOptions { - /** Separate DB for the spaces tables. Defaults to the main `db`. */ - spacesDb?: Database; - /** Extra schema modules to apply after contrail's own DDL. Used by the - * community package to install its tables — contrail core no longer - * imports community schema directly. */ - extraSchemas?: SchemaModule[]; -} - -async function applySpacesSchema( - target: Database, - config: ContrailConfig, - dialect: SqlDialect -): Promise<void> { - const base = buildSpacesBaseSchema(dialect); - const perCollection = buildCollectionTables(config, dialect, { forSpaces: true }); - const indexes = buildDynamicIndexes(config, dialect, { forSpaces: true }); - await target.batch([...base, ...perCollection, ...indexes].map((s) => target.prepare(s))); - - const ftsStmts = buildFtsTables(config, dialect, { forSpaces: true }); - for (const stmt of ftsStmts) { - try { await target.prepare(stmt).run(); } catch { /* FTS5 unavailable */ } - } - for (const stmt of buildCountColumns(config, { forSpaces: true })) { - try { await target.prepare(stmt).run(); } catch { /* already exists */ } - } -} - -export async function initSchema( - db: Database, - config: ContrailConfig, - options: InitSchemaOptions = {} -): Promise<void> { - const dialect = getDialect(db); - const baseStatements = buildBaseSchema(dialect).split(";") - .map((s) => s.trim()) - .filter((s) => s.length > 0); - const collectionStatements = buildCollectionTables(config, dialect); - const indexStatements = buildDynamicIndexes(config, dialect); - const ftsStatements = buildFtsTables(config, dialect); - const feedStatements = buildFeedTables(config, dialect); - - const spacesDb = options.spacesDb; - const spacesSharesMainDb = !spacesDb || spacesDb === db; - - const all = [...baseStatements, ...collectionStatements, ...indexStatements, ...feedStatements]; - - await db.batch(all.map((s) => db.prepare(s))); - - if (config.spaces?.authority || config.spaces?.recordHost) { - await applySpacesSchema(spacesSharesMainDb ? db : spacesDb!, config, dialect); - } - - // Extension schemas (e.g. community) — applied to the spacesDb when one's - // configured separately, since they typically reference space_uri. The - // caller is responsible for routing the schema to the right db; we just - // hand it the spaces-or-main DB as a sensible default. - const extensionTarget = spacesSharesMainDb ? db : spacesDb!; - for (const apply of options.extraSchemas ?? []) { - await apply(extensionTarget); - } - - if (config.labels) { - // Labels tables live on the main DB — they're keyed by at-URI / DID and - // are read alongside public records during hydration. - const labelsStmts = buildLabelsSchema(dialect); - await db.batch(labelsStmts.map((s) => db.prepare(s))); - } - - // FTS5 may not be available (e.g. node:sqlite) — skip gracefully - for (const stmt of ftsStatements) { - try { - await db.prepare(stmt).run(); - } catch { - // FTS5 not supported in this environment - } - } - await runMigrations(db); - - // Add count columns (ALTER TABLE — may already exist) - for (const stmt of buildCountColumns(config)) { - try { - await db.prepare(stmt).run(); - } catch { - // Column/index already exists — ignore - } - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/dialect.ts b/packages/contrail/src/core/dialect.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/dialect.ts +++ b/packages/contrail/src/core/dialect.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/identity.ts b/packages/contrail/src/core/identity.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/identity.ts +++ b/packages/contrail/src/core/identity.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/community-handler.ts b/packages/contrail/src/core/invite/community-handler.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/invite/community-handler.ts +++ b/packages/contrail/src/core/invite/community-handler.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/index.ts b/packages/contrail/src/core/invite/index.ts index 8dae0df..551c495 100644 --- a/packages/contrail/src/core/invite/index.ts +++ b/packages/contrail/src/core/invite/index.ts @@ -1,3 +1 @@ -export { generateInviteToken, hashInviteToken, mintInviteToken } from "./token"; -export { registerInviteRoutes } from "./router"; -export type { InviteRoutesOptions } from "./router"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/router.ts b/packages/contrail/src/core/invite/router.ts index 941af9b..551c495 100644 --- a/packages/contrail/src/core/invite/router.ts +++ b/packages/contrail/src/core/invite/router.ts @@ -1,250 +1 @@ -/** Unified invite surface: a single `<ns>.invite.*` family serving both - * user-owned and community-owned spaces. Dispatches on space ownership. - * - * - User-owned space → `kind` in create, `addMember` on redeem, owner-only. - * - Community-owned → routed to a {@link CommunityInviteHandler} provided - * by the community module (or null when community is - * not configured). - * - * Storage stays separate (`spaces_invites` vs `community_invites` tables) — - * schemas differ enough that unifying them would be net-negative. The token - * primitive and HTTP dance are shared. invite/router has zero imports from - * community/ — coupling is via the {@link CommunityInviteHandler} interface. */ - -import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig } from "../types"; -import type { ServiceAuth } from "../spaces/auth"; -import type { SpaceAuthority } from "../spaces/types"; -import type { InviteKind, InviteRow } from "../spaces/types"; -import { hashInviteToken, mintInviteToken } from "./token"; -import type { CommunityInviteHandler, HandlerResponse } from "./community-handler"; - -export interface InviteRoutesOptions { - authMiddleware: MiddlewareHandler; -} - -interface PublicInviteView { - tokenHash: string; - spaceUri: string; - kind?: InviteKind; - createdBy: string; - createdAt: number; - expiresAt: number | null; - maxUses: number | null; - usedCount: number; - revokedAt: number | null; - note: string | null; -} - -function toSpacesView(row: InviteRow): PublicInviteView { - return { - tokenHash: row.tokenHash, - spaceUri: row.spaceUri, - kind: row.kind, - createdBy: row.createdBy, - createdAt: row.createdAt, - expiresAt: row.expiresAt, - maxUses: row.maxUses, - usedCount: row.usedCount, - revokedAt: row.revokedAt, - note: row.note, - }; -} - -export function registerInviteRoutes( - app: Hono, - config: ContrailConfig, - authority: SpaceAuthority, - community: CommunityInviteHandler | null, - options: InviteRoutesOptions -): void { - if (!config.spaces?.authority) return; - - const NS = `${config.namespace}.invite`; - const auth = options.authMiddleware; - - /** Resolve whether a space is community-owned. Returns null if the space - * doesn't exist. */ - const classifySpace = async (spaceUri: string) => { - const space = await authority.getSpace(spaceUri); - if (!space) return null; - const isCommunity = community ? await community.isCommunityOwned(spaceUri) : false; - return { space, isCommunity }; - }; - - app.post(`/xrpc/${NS}.create`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { - spaceUri?: string; - kind?: string; - accessLevel?: string; - expiresAt?: number; - maxUses?: number; - note?: string; - } - | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - if (body.kind && body.accessLevel) { - return c.json( - { error: "InvalidRequest", reason: "kind-or-accessLevel", message: "pass kind OR accessLevel, not both" }, - 400 - ); - } - - const classified = await classifySpace(body.spaceUri); - if (!classified) return c.json({ error: "NotFound" }, 404); - const { space, isCommunity } = classified; - - if (isCommunity) { - if (!community) return c.json({ error: "InvalidState" }, 500); - return relay(c, await community.create({ - spaceUri: body.spaceUri, - callerDid: sa.issuer, - accessLevel: body.accessLevel, - kind: body.kind, - expiresAt: body.expiresAt ?? null, - maxUses: body.maxUses ?? null, - note: body.note ?? null, - })); - } - - // User-owned space. - if (body.accessLevel) { - return c.json( - { error: "InvalidRequest", reason: "accessLevel-on-user-space", message: "user-owned spaces take kind, not accessLevel" }, - 400 - ); - } - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const kind = (body.kind ?? "join") as InviteKind; - if (kind !== "join" && kind !== "read" && kind !== "read-join") { - return c.json({ error: "InvalidRequest", message: "kind must be 'join', 'read', or 'read-join'" }, 400); - } - const { token, tokenHash } = await mintInviteToken(); - const invite = await authority.createInvite({ - spaceUri: body.spaceUri, - tokenHash, - kind, - expiresAt: body.expiresAt ?? null, - maxUses: body.maxUses ?? null, - createdBy: sa.issuer, - note: body.note ?? null, - }); - return c.json({ token, invite: toSpacesView(invite) }); - }); - - app.get(`/xrpc/${NS}.list`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const includeRevoked = c.req.query("includeRevoked") === "true"; - - const classified = await classifySpace(spaceUri); - if (!classified) return c.json({ error: "NotFound" }, 404); - const { space, isCommunity } = classified; - - if (isCommunity) { - return relay(c, await community!.list({ - spaceUri, - callerDid: sa.issuer, - includeRevoked, - })); - } - - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const rows = await authority.listInvites(spaceUri, { includeRevoked }); - return c.json({ invites: rows.map(toSpacesView) }); - }); - - app.post(`/xrpc/${NS}.revoke`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; tokenHash?: string } - | null; - if (!body?.tokenHash) { - return c.json({ error: "InvalidRequest", message: "tokenHash required" }, 400); - } - - if (body.spaceUri) { - const classified = await classifySpace(body.spaceUri); - if (!classified) return c.json({ error: "NotFound" }, 404); - if (classified.isCommunity) { - return relay(c, await community!.revoke({ - spaceUri: body.spaceUri, - tokenHash: body.tokenHash, - callerDid: sa.issuer, - })); - } - if (classified.space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const ok = await authority.revokeInvite(body.tokenHash); - return c.json({ ok }); - } - - // No spaceUri — try the community handler first (it returns null if the - // token isn't a community invite), then fall back to the user-owned path. - if (community) { - const r = await community.tryRevokeByToken({ - tokenHash: body.tokenHash, - callerDid: sa.issuer, - }); - if (r) return relay(c, r); - } - const srow = await authority.getInvite(body.tokenHash); - if (!srow) return c.json({ error: "NotFound" }, 404); - const space = await authority.getSpace(srow.spaceUri); - if (space && space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - const ok = await authority.revokeInvite(body.tokenHash); - return c.json({ ok }); - }); - - app.post(`/xrpc/${NS}.redeem`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { token?: string } | null; - if (!body?.token) { - return c.json({ error: "InvalidRequest", message: "token required" }, 400); - } - const tokenHash = await hashInviteToken(body.token); - const now = Date.now(); - - // Try community first (atomic — null if not a community invite). - if (community) { - const r = await community.tryRedeem({ - tokenHash, - callerDid: sa.issuer, - now, - }); - if (r) return relay(c, r); - } - - // Fall back to the user-owned spaces path. The redeem filter at the SQL - // level already restricts to `kind IN ('join','read-join')`. - const sinvite = await authority.redeemInvite(tokenHash, now); - if (!sinvite) { - return c.json({ error: "InvalidInvite", reason: "expired-revoked-or-exhausted" }, 400); - } - await authority.addMember(sinvite.spaceUri, sa.issuer, sinvite.createdBy); - return c.json({ spaceUri: sinvite.spaceUri, kind: sinvite.kind }); - }); -} - -/** Forward a community-handler response to the wire. */ -function relay(c: Context, r: HandlerResponse) { - return c.json(r.body, r.status as Parameters<typeof c.json>[1]); -} - -function getAuth(c: Context): ServiceAuth { - const a = c.get("serviceAuth") as ServiceAuth | undefined; - if (!a) throw new Error("service auth not set"); - return a; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/invite/token.ts b/packages/contrail/src/core/invite/token.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/invite/token.ts +++ b/packages/contrail/src/core/invite/token.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/jetstream.ts b/packages/contrail/src/core/jetstream.ts index d42a5e8..551c495 100644 --- a/packages/contrail/src/core/jetstream.ts +++ b/packages/contrail/src/core/jetstream.ts @@ -1,280 +1 @@ -import { JetstreamSubscription } from "@atcute/jetstream"; -import type { ContrailConfig, IngestEvent, Database, Logger } from "./types"; -import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS } from "./types"; -import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; -import { refreshStaleIdentities } from "./identity"; - -const BATCH_SIZE = 50; -const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; // 1 hour - -/** Mutable state that persists across ingest cycles within the same process. */ -export interface IngestState { - cachedKnownDids?: Set<string>; - schemaInitialized: boolean; - lastFeedPruneMs: number; -} - -export function createIngestState(): IngestState { - return { schemaInitialized: false, lastFeedPruneMs: 0 }; -} - -function getLogger(config: ContrailConfig): Logger { - return config.logger ?? console; -} - -export async function ingestEvents( - config: ContrailConfig, - cursor: number | null, - safetyTimeoutMs: number = 25_000, - knownDids?: Set<string> -): Promise<{ events: IngestEvent[]; lastCursor: number | null }> { - const log = getLogger(config); - const startTimeUs = Date.now() * 1000; - const deadline = Date.now() + safetyTimeoutMs; - const collected: IngestEvent[] = []; - - const collections = getCollectionNsids(config); - const dependentCollections = new Set(getDependentNsids(config)); - const urls = config.jetstreams ?? []; - - let totalCommits = 0; - let filteredUnknownDid = 0; - const filteredDidSamples = new Set<string>(); - let lastYieldedTimeUs: number | null = null; - let firstYieldedTimeUs: number | null = null; - let connectCount = 0; - const seenUris = new Map<string, number>(); // uri -> time_us of first occurrence - const duplicateUris: string[] = []; - - const subscription = new JetstreamSubscription({ - url: urls, - wantedCollections: collections, - ...(cursor !== null ? { cursor } : {}), - onConnectionOpen() { - connectCount++; - log.log( - `[ingest] connected to Jetstream #${connectCount} (url=${urls.join("|")}, cursor=${cursor ?? "none"}, wanted=${collections.join(",")})` - ); - }, - onConnectionClose(event) { - log.log( - `[ingest] disconnected from Jetstream: ${event.code} ${event.reason}` - ); - }, - onConnectionError(event) { - log.error("[ingest] Jetstream error:", event.error); - }, - }); - - for await (const event of subscription) { - if (firstYieldedTimeUs === null) firstYieldedTimeUs = event.time_us; - lastYieldedTimeUs = event.time_us; - if (event.kind === "commit") { - const { commit } = event; - totalCommits++; - - const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; - - if (dependentCollections.has(commit.collection) && knownDids) { - if (!knownDids.has(event.did)) { - filteredUnknownDid++; - if (filteredDidSamples.size < 10) filteredDidSamples.add(event.did); - continue; - } - } - - const prev = seenUris.get(uri); - if (prev !== undefined) { - duplicateUris.push(uri); - log.warn( - `[ingest] DUPLICATE in cycle: ${uri} first time_us=${prev}, again=${event.time_us}, delta=${event.time_us - prev}us` - ); - } else { - seenUris.set(uri, event.time_us); - } - - const now = Date.now(); - - collected.push({ - uri, - did: event.did, - time_us: event.time_us, - collection: commit.collection, - operation: commit.operation as "create" | "update" | "delete", - rkey: commit.rkey, - cid: commit.operation === "delete" ? null : commit.cid, - record: - commit.operation === "delete" - ? null - : JSON.stringify(commit.record), - indexed_at: now * 1000, - }); - - log.log( - `[ingest] keep: ${commit.operation} ${uri} time_us=${event.time_us}` - ); - - if (knownDids && !dependentCollections.has(commit.collection)) { - knownDids.add(event.did); - } - } - - if (event.time_us >= startTimeUs) { - log.log( - `[ingest] caught up to present, stopping (last time_us=${event.time_us}, startTimeUs=${startTimeUs})` - ); - break; - } - - if (Date.now() >= deadline) { - log.log( - `[ingest] safety timeout reached, stopping (deadline=${deadline}, collected=${collected.length})` - ); - break; - } - } - - if (filteredUnknownDid > 0) { - const sample = [...filteredDidSamples].join(", "); - log.log( - `[ingest] ${filteredUnknownDid} events filtered (unknown did). sample dids: ${sample}` - ); - } - const lastCursor = subscription.cursor || null; - - const cursorGap = - lastCursor !== null && lastYieldedTimeUs !== null - ? lastCursor - lastYieldedTimeUs - : null; - - // Detect the library's internal cursor rollback (picks a different URL → rolls - // back 10s → first event comes in BEFORE the cursor we asked it to start from). - const rolledBackUs = - cursor !== null && firstYieldedTimeUs !== null && firstYieldedTimeUs < cursor - ? cursor - firstYieldedTimeUs - : 0; - - log.log( - `[ingest] jetstream loop done. commits_seen=${totalCommits}, filtered=${filteredUnknownDid}, kept=${collected.length}, dupes=${duplicateUris.length}, connects=${connectCount}, first_yielded=${firstYieldedTimeUs ?? "none"}, last_yielded=${lastYieldedTimeUs ?? "none"}, subscription_cursor=${lastCursor ?? "none"}, cursor_gap=${cursorGap ?? "n/a"}us, rolled_back=${rolledBackUs}us` - ); - - if (cursorGap !== null && cursorGap > 1000) { - log.warn( - `[ingest] CURSOR GAP: subscription cursor is ${cursorGap}us (${Math.floor( - cursorGap / 1000 - )}ms) ahead of last yielded event — buffered events may be dropped` - ); - } - - if (connectCount > 1) { - log.warn( - `[ingest] RECONNECTED ${connectCount} times during cycle — each reconnect picks a URL at random and rolls cursor back 10s` - ); - } - - return { events: collected, lastCursor }; -} - -// Run a full ingest cycle: init schema, load cursor, ingest, apply, save cursor -export async function runIngestCycle( - db: Database, - config: ContrailConfig, - timeoutMs: number = 25_000, - state?: IngestState, - pubsub?: import("./realtime/types").PubSub -): Promise<void> { - const log = getLogger(config); - const s = state ?? createIngestState(); - - if (!s.schemaInitialized) { - await initSchema(db, config); - s.schemaInitialized = true; - } - - const cursor = await getLastCursor(db); - const collections = getCollectionNsids(config); - const nowUs = Date.now() * 1000; - const lagMs = cursor !== null ? Math.floor((nowUs - cursor) / 1000) : null; - - log.log( - `[ingest] starting cycle. cursor=${cursor ?? "none"}${ - lagMs !== null ? ` (lag=${lagMs}ms)` : "" - }, timeout=${timeoutMs}ms, collections=${collections.join(", ")}` - ); - - // Load known DIDs for filtering dependent collections - const dependentCollections = getDependentNsids(config); - let knownDids: Set<string> | undefined; - - if (dependentCollections.length > 0) { - if (s.cachedKnownDids) { - knownDids = s.cachedKnownDids; - log.log(`Using cached known DIDs (${knownDids.size} users)`); - } else { - const result = await db - .prepare("SELECT did FROM identities") - .all<{ did: string }>(); - knownDids = new Set((result.results ?? []).map((r) => r.did)); - s.cachedKnownDids = knownDids; - log.log(`Loaded ${knownDids.size} known DIDs from database`); - } - } - - const { events, lastCursor } = await ingestEvents( - config, - cursor, - timeoutMs, - knownDids - ); - - if (events.length > 0) { - const breakdown: Record<string, number> = {}; - for (const e of events) { - const key = `${e.collection}:${e.operation}`; - breakdown[key] = (breakdown[key] ?? 0) + 1; - } - log.log( - `[ingest] received ${events.length} events. breakdown=${JSON.stringify(breakdown)}` - ); - } else { - log.log(`[ingest] received 0 events from Jetstream`); - } - - for (let i = 0; i < events.length; i += BATCH_SIZE) { - const batch = events.slice(i, i + BATCH_SIZE); - await applyEvents(db, batch, config, { pubsub }); - } - - // Refresh stale/missing identities for DIDs in this batch - const uniqueDids = [...new Set(events.map((e) => e.did))]; - if (uniqueDids.length > 0) { - try { - await refreshStaleIdentities(db, uniqueDids); - } catch (err) { - log.warn(`Identity refresh failed: ${err}`); - } - } - - if (lastCursor !== null) { - await saveCursor(db, lastCursor); - log.log( - `[ingest] saved cursor=${lastCursor} (advanced ${ - cursor !== null ? lastCursor - cursor : "n/a" - }us)` - ); - } else { - log.log(`[ingest] no cursor returned from subscription; not saving`); - } - - // Prune feed items hourly - if (config.feeds && Date.now() - s.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { - const maxItems = Math.max( - ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) - ); - const pruned = await pruneFeedItems(db, maxItems); - if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); - s.lastFeedPruneMs = Date.now(); - } - - log.log(`[ingest] cycle complete. stored=${events.length}`); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/apply.ts b/packages/contrail/src/core/labels/apply.ts index cbf5558..551c495 100644 --- a/packages/contrail/src/core/labels/apply.ts +++ b/packages/contrail/src/core/labels/apply.ts @@ -1,64 +1 @@ -import type { Database, Statement } from "../types"; - -/** Wire shape of a single `com.atproto.label.defs#label` entry. Field names - * match the spec exactly. We accept the spec's ISO-8601 strings and - * convert to unix seconds at the storage boundary. */ -export interface IncomingLabel { - src: string; - uri: string; - val: string; - cid?: string; - neg?: boolean; - exp?: string; - cts: string; - sig?: Uint8Array; -} - -/** Upsert a batch of labels. Idempotent on `(src, uri, val, cts)`. Bad rows - * (missing required fields, unparseable timestamps) are dropped silently; - * we don't want one malformed label to abort an entire labeler frame. */ -export async function applyLabels( - db: Database, - labels: IncomingLabel[], -): Promise<number> { - if (labels.length === 0) return 0; - const stmts: Statement[] = []; - let kept = 0; - for (const l of labels) { - if (!l.src || !l.uri || !l.val || !l.cts) continue; - const cts = isoToUnixSec(l.cts); - if (cts == null) continue; - const exp = l.exp ? isoToUnixSec(l.exp) : null; - stmts.push( - db - .prepare( - `INSERT INTO labels (src, uri, val, cid, neg, exp, cts, sig) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT(src, uri, val, cts) DO UPDATE SET - cid = excluded.cid, - neg = excluded.neg, - exp = excluded.exp, - sig = excluded.sig`, - ) - .bind( - l.src, - l.uri, - l.val, - l.cid ?? null, - l.neg ? 1 : 0, - exp, - cts, - l.sig ?? null, - ), - ); - kept++; - } - if (stmts.length > 0) await db.batch(stmts); - return kept; -} - -function isoToUnixSec(iso: string): number | null { - const ms = Date.parse(iso); - if (!Number.isFinite(ms)) return null; - return Math.floor(ms / 1000); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/hydrate.ts b/packages/contrail/src/core/labels/hydrate.ts index 02b947a44ef764f5c31f64fefd468727f76fd7d1..551c49528106738c1198722f3089b7f856050fe3 100644 GIT binary patch literal 44 zcmYeTD9A4=QP5IIE6UGRP;y8t$<5bINiEY)&d)0;O3cjBO)MxV%S<g-vgQH+V0;gI literal 3547 zcmd1IEyyn_Q7EY_NL8p-a7iplOiC<HRj5@+E6UGRP}0-W2T2txS#v>ERx2bWmLzAS zrg-KBmZla}A}cM*FD*$e($7fEDM&2>>EY7X(ozV|EK1ca&PXgsRY=P(Qb<(Ds7xtJ zEJ;mK$Vp5}%~5#NFhwC3Y-X`SLUMktUSdf>QGQ9j9#~p0B{i*B8O%)JQqWRRNXtyk zNzqNqFVY3G6pHf|lJoP5OLJ3;iWQO*^Arjai;ER9OB9MriV`!^GfEUnGK%s`(=!xG z@)e*)D+IfE7wai#>2sx4fc>ADSCU$kmYAHX-~n}<57=$hTnY-sMafnm!!q;It+^Bw zN{cd)xn+qt$lT=26nkVoC;<HPaw-)Hic*VH^GX!TGg9*uN{UKT!PYA%<fW$DTY&{Y zhJ%!Q2K(z;m>C%ADCFm6mVm+utS2vDA+@3)v#3%bvsj@xwFIg^wW0vSLYN5($wjG& zC7Jno3MHAjsl_FUxdl*N$tA@ISJrZIA^abdSPu4jP=2`rIlhM(t5BnmSDKRp@>E`G zx|Kp+X>L+#5r~^wQ2^s2DFV9)rl^(+oO0Y!OOi7nsiYX1c8XI=6!Oy)ic6ESQj<%H zbrjMvb4pT+KyjO#n3I!Qq+YC$n4Fwi017v7s!1&>)&VD+<oukR#Dd~fg#?Y_qGTOV zMCvG%CFW=*D3p{I<fIlWBvmTpB$lKWmnbA8mlP-HC?w{kD5MnS7r+#hXXF>Bf(?VJ z%Pdw%R7l86O}7O_X@WvgYDrOIGAPLP6oOJqN{jM93JOw-bfGR*$W1KJ<5JK98xjol z1vuFzr79$r=!OP)7AxczDI_Hpr7F01x)g)6M@dGiLUCelszOO(QckKuMt)98u|iUQ zNd`DNQz2<MvA8lXSs|@74{Sdymw_X*Sc6MJAtlKQp22lM=^AP~EXJZ^L87q0LlrGb zP0lY$an4L}sthg4w6{|5O)Rh}E-A{)OV@$s4QO22>2PUUDFhVd=VlhC+6001!BwJU zuIN}hJ8%-qOjFQ+22ZhGPHJ9yNrr;0t*wHALQM@kGEpQnK>=M_l&4T#Yt6-_prEg> z;F6kBT967)7^uPK;_0H0n^>ukSWu9fSfr4dr;w16nNpkpj^M;%1yJ}x0v}>nVrfZ6 zevvg;931WtNw68o`MIf((lIBqxI{r47SMWJ3gCjMM4>b<vk)G&whGaDdU|=O<qE;6 zC2((Q#)5JJsN8}mcZ4NbROPV904awXUyzfSoC-<;paRbpY#U6wUT$K6hK8nst(}6B zy^^M0R(@ulhLVnwCc?J7{PN(`WCdFV-^7v(y|kSC{2~pP#FA9Ky!>(vO$B`gLjwZ? zO<1TpW#*-1=A{=alw~HO7yOAusa9am6%?hWWmf1Y=;`StXO!k;$Lc8rl!A)gRE31p zih=}4m_jqER#IwOeo?9xIBXN*<30W2;}aB+`~)h{LGh6Uv05R&C?&NBRzX0F0!ar4 z`k?3nD?`LVQGR)`f~`Vgd17V>n!|07ia0w^#sFmvP|{aWNC<ZIadi#>75RwrK}R7u zGerkfQ0st7U>$|zl41q7Ab($I$;71q0pT95L9QUJ3Z8xn8Y<ODek!ijgsD<caP)Ig z&`7N)Q1A>^@C)_vQSc7}nX6!@V2`G%v?vp779^B36CjQRr6*7lgGIWI0whks*;fIa z9$={tqyZk60h!6!;82H_T}Aoj3I(Y}n8l%<LP%z6u|j?xq!?5vN-fUMDN9vIs)PoA zVs>U;I@l<X86_EsB?^f}MVV#bDjifBXXb%r3&9P9qSWHjoDxVumI6{$q??jjkeZj0 znpXlbFsZaep$JsqW#%aqfXmP1RBMIeg4AS%l>F4<JoOTVl+?7$yi|pf%v9Z^qExU0 zK#>h<oG5swRw~#+OCdx;hxpQ|5+V!AfS@wVJGD|r0VM%|^9HD4q5;W{ATQ;oDS#4? zCb)P~fM)A#ur&!P)kXQ`dc{S_wG0quX;CJOS(cbnn*eeGOiw{kW`2=^EwowzJ4P=( zwL~L3wGwQvf<k7Rf`%eQSxpVdRy|Piva?kHi-9<rD7uRwx^+NG!A8`A3P6a{^Gi#t z6bKZ#3bx=vQvqC`qIwxA#PrG%b4pW-H8f#C0CFbC-+8I&nxOC~$;>NFwFVWu&?+xA z1>_EJQS9vLqEL{TXQhw`$sr2h6q1*in+mE7LD8C-mYJ*osZ$g{suK$eav(`Ar&6IL zADmb8^&$GoQj3Z+^Yio+Jk!953R2C3TY0G|3W-o-bdxhvKy@p)L<f}*Mfv6G#V~_F zcBEt$C#DyrrWWf!s`ShfcpU+5C%}^$$T1*SfNg`?4l}4cRlO(`TKQz=rRym;CMV~Y z=9TE?q?V=TK)hS5paH6Hpf#GNLULkJQKdp&KFFsqa|$x^K(!Dev6rVR<b#@t$r-81 z+0aHJsM3ej(F)0+%D+6bBtsz)WDF?jLy`_CPC#B)Q-cNstd53794sY5LfbhrMZp$L zB{)HX%qcC(1g9uaaR!cWaODlsqG$`MXCQenMFH6^1&|a-b8==1Qo;cH6Pz+(_JhON zN&%$>07{EsmC!~u$Q)493@Qe0X@kT{i!wna8OUAW<~B&IEHMWr25vf*6o=${7Uye# z<dRE@H6fC<kjx17O@6r^*pXnDfS90=fC!?h0+r2*w&3O~DDNeu=IDWhY!#60NUbP< z29`#CX-PE5+R~!TSOt4~TZQOYO}&EB;tUP2RbXY{uqXnz2o>^6ORTwSxwzm>HJG`j wd6^Z#smZX`u_mky1}=u6nn48_sJ_;KDpt^f)Yy6@`JTc4!QdvDh9<~-0O=5WfdBvi diff --git a/packages/contrail/src/core/labels/resolve.ts b/packages/contrail/src/core/labels/resolve.ts index 694bfec..551c495 100644 --- a/packages/contrail/src/core/labels/resolve.ts +++ b/packages/contrail/src/core/labels/resolve.ts @@ -1,134 +1 @@ -import { - CompositeDidDocumentResolver, - PlcDidDocumentResolver, - WebDidDocumentResolver, -} from "@atcute/identity-resolver"; -import type { Did } from "@atcute/lexicons"; -import type { Database } from "../types"; - -/** Reject endpoint URLs that point to private/internal addresses or non-HTTPS. - * Mirrors the validator in core/client.ts — labeler endpoints should be - * publicly reachable for the same reasons PDS endpoints should. */ -function validateEndpointUrl(url: string): boolean { - try { - const parsed = new URL(url); - if (parsed.protocol !== "https:") return false; - const host = parsed.hostname; - if (host === "localhost" || host === "127.0.0.1" || host === "[::1]") return false; - if (host.startsWith("10.")) return false; - if (host.startsWith("192.168.")) return false; - if (host.startsWith("169.254.")) return false; - if (/^172\.(1[6-9]|2\d|3[01])\./.test(host)) return false; - return true; - } catch { - return false; - } -} - -const didResolver = new CompositeDidDocumentResolver({ - methods: { - plc: new PlcDidDocumentResolver(), - web: new WebDidDocumentResolver(), - }, -}); - -/** Look up the labeler service endpoint from a DID. - * Reads the DID doc's `service[id="#atproto_labeler"].serviceEndpoint`. */ -export async function resolveLabelerEndpoint(did: string): Promise<string | null> { - if (!did.startsWith("did:plc:") && !did.startsWith("did:web:")) return null; - try { - const doc = await didResolver.resolve(did as Did<"plc"> | Did<"web">); - const endpoint = doc.service - ?.find((s) => s.id === "#atproto_labeler") - ?.serviceEndpoint?.toString(); - if (!endpoint) return null; - if (!validateEndpointUrl(endpoint)) return null; - return endpoint; - } catch { - return null; - } -} - -/** State row for a labeler — the per-DID equivalent of the singleton - * jetstream `cursor` table, with cached endpoint to avoid repeated DID-doc - * fetches. */ -export interface LabelerState { - did: string; - cursor: number; - endpoint: string | null; - resolved_at: number | null; -} - -const ENDPOINT_TTL_MS = 6 * 60 * 60 * 1000; // 6h, matches the recommended client cache for label-defs - -/** Get cached `(endpoint, cursor)` for a labeler. Resolves endpoint on - * cache miss or staleness; persists endpoint + resolved_at back to the DB - * so subsequent ingest cycles avoid the network round-trip. */ -export async function getLabelerState( - db: Database, - did: string, - endpointOverride: string | undefined, -): Promise<LabelerState | null> { - const row = await db - .prepare( - "SELECT did, cursor, endpoint, resolved_at FROM labeler_cursors WHERE did = ?", - ) - .bind(did) - .first<LabelerState>(); - - let endpoint = endpointOverride ?? row?.endpoint ?? null; - const stale = - !row?.resolved_at || Date.now() - row.resolved_at > ENDPOINT_TTL_MS; - - if (!endpoint || (!endpointOverride && stale)) { - endpoint = await resolveLabelerEndpoint(did); - if (!endpoint) return null; - const now = Date.now(); - await db - .prepare( - `INSERT INTO labeler_cursors (did, cursor, endpoint, resolved_at) - VALUES (?, ?, ?, ?) - ON CONFLICT(did) DO UPDATE SET endpoint = excluded.endpoint, resolved_at = excluded.resolved_at`, - ) - .bind(did, row?.cursor ?? 0, endpoint, now) - .run(); - return { - did, - cursor: row?.cursor ?? 0, - endpoint, - resolved_at: now, - }; - } - - return row ?? { did, cursor: 0, endpoint, resolved_at: null }; -} - -/** Persist the highest seen seq number for a labeler. Idempotent; - * the next ingest cycle resumes from `cursor + 1` via the `?cursor=` param. */ -export async function saveLabelerCursor( - db: Database, - did: string, - cursor: number, -): Promise<void> { - await db - .prepare( - `INSERT INTO labeler_cursors (did, cursor) - VALUES (?, ?) - ON CONFLICT(did) DO UPDATE SET cursor = excluded.cursor`, - ) - .bind(did, cursor) - .run(); -} - -/** Reset cursor to 0 — used in response to `#info { name: "OutdatedCursor" }` - * frames, which signal that the labeler's seq history was rewound. */ -export async function resetLabelerCursor(db: Database, did: string): Promise<void> { - await db - .prepare( - `INSERT INTO labeler_cursors (did, cursor) - VALUES (?, 0) - ON CONFLICT(did) DO UPDATE SET cursor = 0`, - ) - .bind(did) - .run(); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/schema.ts b/packages/contrail/src/core/labels/schema.ts index 486ce27..551c495 100644 --- a/packages/contrail/src/core/labels/schema.ts +++ b/packages/contrail/src/core/labels/schema.ts @@ -1,30 +1 @@ -import type { SqlDialect } from "../dialect"; - -/** DDL for the labels module. Single `labels` table covers record-level - * (uri starts with `at://`) and account-level (uri is a bare DID) entries — - * the spec collapses both into the same row shape. `labeler_cursors` - * mirrors the role of the singleton `cursor` table for jetstream, but - * per-labeler. */ -export function buildLabelsSchema(dialect: SqlDialect): string[] { - return [ - `CREATE TABLE IF NOT EXISTS labels ( - src TEXT NOT NULL, - uri TEXT NOT NULL, - val TEXT NOT NULL, - cid TEXT, - neg INTEGER NOT NULL DEFAULT 0, - exp ${dialect.bigintType}, - cts ${dialect.bigintType} NOT NULL, - sig BLOB, - PRIMARY KEY (src, uri, val, cts) - )`, - `CREATE INDEX IF NOT EXISTS idx_labels_uri ON labels(uri)`, - `CREATE INDEX IF NOT EXISTS idx_labels_src_cts ON labels(src, cts DESC)`, - `CREATE TABLE IF NOT EXISTS labeler_cursors ( - did TEXT PRIMARY KEY, - cursor ${dialect.bigintType} NOT NULL DEFAULT 0, - endpoint TEXT, - resolved_at ${dialect.bigintType} - )`, - ]; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/select.ts b/packages/contrail/src/core/labels/select.ts index da89d9c..551c495 100644 --- a/packages/contrail/src/core/labels/select.ts +++ b/packages/contrail/src/core/labels/select.ts @@ -1,64 +1 @@ -import type { LabelsConfig } from "./types"; -import { DEFAULT_LABELS_MAX_PER_REQUEST } from "./types"; - -/** Pick which labelers to honor for this request. - * - * Order of precedence: - * 1. `atproto-accept-labelers` header (atproto spec) - * 2. `?labelers=` query param (fallback for SSE/WS where headers are awkward) - * 3. `config.defaults` (operator policy) - * 4. every entry in `config.sources` - * - * Each candidate DID is checked against `config.sources`. Unknowns are - * dropped — we only have rows for labelers we've subscribed to. - * - * Header values can carry `;param` modifiers (e.g. `did:plc:...;redact`); - * v1 strips and ignores those — only the bare DID is honored. */ -export interface SelectedLabelers { - /** DIDs to use for hydration this request. */ - accepted: string[]; -} - -export function selectAcceptedLabelers( - headerValue: string | null | undefined, - paramValue: string | null | undefined, - cfg: LabelsConfig, -): SelectedLabelers { - const cap = cfg.maxPerRequest ?? DEFAULT_LABELS_MAX_PER_REQUEST; - const known = new Set(cfg.sources.map((s) => s.did)); - - const fromCaller = parseLabelerList(headerValue) ?? parseLabelerList(paramValue); - - let candidates: string[]; - if (fromCaller && fromCaller.length > 0) { - candidates = fromCaller; - } else { - candidates = (cfg.defaults ?? cfg.sources.map((s) => s.did)).slice(); - } - - const accepted: string[] = []; - const seen = new Set<string>(); - for (const did of candidates) { - if (seen.has(did)) continue; - seen.add(did); - if (known.has(did)) accepted.push(did); - if (accepted.length >= cap) break; - } - - return { accepted }; -} - -/** Parse a comma-separated DID list. Returns null when the input is empty - * or undefined so callers can distinguish "absent" from "empty list" (the - * latter — `atproto-accept-labelers: ` — is technically valid and means - * "no labelers"; we treat it the same as absent for ergonomics). */ -function parseLabelerList(value: string | null | undefined): string[] | null { - if (!value) return null; - const out: string[] = []; - for (const raw of value.split(",")) { - // Drop `;param` modifiers from the spec (e.g. `;redact`). v1 ignores them. - const head = raw.split(";")[0]!.trim(); - if (head.startsWith("did:")) out.push(head); - } - return out.length > 0 ? out : null; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/subscribe.ts b/packages/contrail/src/core/labels/subscribe.ts index c80ed2b..551c495 100644 --- a/packages/contrail/src/core/labels/subscribe.ts +++ b/packages/contrail/src/core/labels/subscribe.ts @@ -1,315 +1 @@ -import { decodeFirst } from "@atcute/cbor"; -import type { ContrailConfig, Database, Logger } from "../types"; -import type { LabelerSource } from "./types"; -import { applyLabels, type IncomingLabel } from "./apply"; -import { - getLabelerState, - resetLabelerCursor, - saveLabelerCursor, -} from "./resolve"; - -const DEFAULT_CYCLE_TIMEOUT_MS = 25_000; -const DEFAULT_BATCH_SIZE = 100; -const DEFAULT_FLUSH_INTERVAL_MS = 5_000; - -function getLogger(config: ContrailConfig): Logger { - return config.logger ?? console; -} - -/** One catch-up cycle for every configured labeler. Designed to fit inside a - * Cloudflare Workers cron tick — we drain frames until the labeler has no - * more buffered events for us, or `timeoutMs` is reached, then save cursor - * and disconnect. Mirrors the shape of `runIngestCycle` for jetstream. */ -export async function runLabelIngestCycle( - db: Database, - config: ContrailConfig, - timeoutMs = DEFAULT_CYCLE_TIMEOUT_MS, -): Promise<void> { - if (!config.labels) return; - const log = getLogger(config); - const deadline = Date.now() + timeoutMs; - - for (const source of config.labels.sources) { - if (Date.now() >= deadline) { - log.log(`[labels] cycle deadline hit before processing ${source.did}`); - break; - } - const remaining = Math.max(2_000, deadline - Date.now()); - try { - await pumpOneLabeler(db, source, log, remaining, /* persistent */ false); - } catch (err) { - log.warn(`[labels] cycle for ${source.did} failed: ${err}`); - } - } -} - -export interface PersistentLabelsOptions { - signal?: AbortSignal; - batchSize?: number; - flushIntervalMs?: number; - logger?: Logger; -} - -/** Long-lived equivalent — keeps one socket per labeler open forever, with - * exponential backoff reconnect. Mirrors `runPersistent` for jetstream. */ -export async function runPersistentLabels( - db: Database, - config: ContrailConfig, - options: PersistentLabelsOptions = {}, -): Promise<void> { - if (!config.labels) return; - const log = options.logger ?? config.logger ?? console; - const signal = options.signal; - - const tasks = config.labels.sources.map((source) => - runOneLabelerForever(db, source, log, signal, options), - ); - await Promise.all(tasks); -} - -async function runOneLabelerForever( - db: Database, - source: LabelerSource, - log: Logger, - signal: AbortSignal | undefined, - options: PersistentLabelsOptions, -): Promise<void> { - let attempts = 0; - while (!signal?.aborted) { - try { - await pumpOneLabeler(db, source, log, /* timeoutMs */ Infinity, true, { - signal, - batchSize: options.batchSize ?? DEFAULT_BATCH_SIZE, - flushIntervalMs: options.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS, - }); - attempts = 0; - } catch (err) { - if (signal?.aborted) break; - log.error(`[labels] ${source.did} stream error: ${err}`); - const delay = Math.min(1_000 * 2 ** attempts, 30_000); - attempts++; - log.log(`[labels] ${source.did} reconnecting in ${delay}ms (attempt ${attempts})`); - await new Promise((r) => setTimeout(r, delay)); - } - } -} - -interface PumpOptions { - signal?: AbortSignal; - batchSize?: number; - flushIntervalMs?: number; -} - -/** Open a `subscribeLabels` WebSocket, drain frames into a buffer, flush - * the buffer to `labels` in batches, and persist the seq cursor. Returns - * when: - * - the labeler closes the socket cleanly (caught up + no more events) - * - `timeoutMs` is reached (cron mode) - * - `signal` is aborted (persistent mode) - * - an error tears the socket down (caller may retry) */ -async function pumpOneLabeler( - db: Database, - source: LabelerSource, - log: Logger, - timeoutMs: number, - persistent: boolean, - pumpOpts: PumpOptions = {}, -): Promise<void> { - const state = await getLabelerState(db, source.did, source.endpoint); - if (!state) { - log.warn(`[labels] could not resolve labeler endpoint for ${source.did}; skipping`); - return; - } - - // First-time policy: cursor 0 = "from the beginning" if backfill is on - // (default), null = "from now" otherwise. After the first cycle we always - // resume from the saved cursor — `backfill` only flips the start point. - const isFirstRun = state.cursor === 0 && state.resolved_at === null; - const backfill = source.backfill !== false; - const startCursor = isFirstRun && !backfill ? null : state.cursor; - - const url = buildWsUrl(state.endpoint!, startCursor); - log.log(`[labels] connecting to ${source.did} (cursor=${startCursor ?? "now"})`); - - const ws = new WebSocket(url); - ws.binaryType = "arraybuffer"; - - const buffer: IncomingLabel[] = []; - let highestSeq = state.cursor; - let flushing = false; - let resolveDone!: () => void; - let rejectDone!: (err: unknown) => void; - const done = new Promise<void>((res, rej) => { - resolveDone = res; - rejectDone = rej; - }); - - const flush = async () => { - if (buffer.length === 0 || flushing) return; - flushing = true; - const batch = buffer.splice(0); - try { - const kept = await applyLabels(db, batch); - if (highestSeq > state.cursor) { - await saveLabelerCursor(db, source.did, highestSeq); - state.cursor = highestSeq; - } - log.log( - `[labels] ${source.did} flushed ${kept}/${batch.length} labels, cursor=${highestSeq}`, - ); - } catch (err) { - log.error(`[labels] ${source.did} flush failed: ${err}`); - } finally { - flushing = false; - } - }; - - const batchSize = pumpOpts.batchSize ?? DEFAULT_BATCH_SIZE; - const flushInterval = pumpOpts.flushIntervalMs ?? DEFAULT_FLUSH_INTERVAL_MS; - const flushTimer = setInterval(() => { - flush().catch(() => {}); - }, flushInterval); - - const cleanup = () => { - clearInterval(flushTimer); - try { - ws.close(); - } catch { - /* already closed */ - } - }; - - // External abort (persistent mode) — close socket gracefully. - const abortHandler = () => { - cleanup(); - flush().finally(() => resolveDone()); - }; - pumpOpts.signal?.addEventListener("abort", abortHandler, { once: true }); - - // Cron-mode time budget — close socket gracefully when reached. - let deadlineTimer: ReturnType<typeof setTimeout> | undefined; - if (Number.isFinite(timeoutMs)) { - deadlineTimer = setTimeout(() => { - log.log(`[labels] ${source.did} cycle deadline reached, closing`); - cleanup(); - flush().finally(() => resolveDone()); - }, timeoutMs); - } - - ws.addEventListener("error", (ev) => { - cleanup(); - if (deadlineTimer) clearTimeout(deadlineTimer); - pumpOpts.signal?.removeEventListener("abort", abortHandler); - rejectDone(new Error(`WebSocket error: ${(ev as ErrorEvent)?.message ?? "unknown"}`)); - }); - - ws.addEventListener("close", () => { - if (deadlineTimer) clearTimeout(deadlineTimer); - pumpOpts.signal?.removeEventListener("abort", abortHandler); - flush().finally(() => { - clearInterval(flushTimer); - resolveDone(); - }); - }); - - ws.addEventListener("message", async (ev) => { - let bytes: Uint8Array; - if (ev.data instanceof ArrayBuffer) { - bytes = new Uint8Array(ev.data); - } else if (ev.data instanceof Uint8Array) { - bytes = ev.data; - } else { - // Binary-only protocol — text frames shouldn't arrive. - return; - } - const frame = decodeFrame(bytes); - if (!frame) return; - - if (frame.t === "#labels") { - const seq = Number(frame.payload?.seq ?? 0); - const rawLabels = Array.isArray(frame.payload?.labels) ? frame.payload.labels : []; - for (const raw of rawLabels) { - const lab = normalizeLabel(raw, source.did); - if (lab) buffer.push(lab); - } - if (Number.isFinite(seq) && seq > highestSeq) highestSeq = seq; - if (buffer.length >= batchSize) { - flush().catch(() => {}); - } - } else if (frame.t === "#info") { - const name = String(frame.payload?.name ?? ""); - log.log(`[labels] ${source.did} info: ${name}`); - if (name === "OutdatedCursor") { - // Labeler rewound its log — discard our cursor and let the next - // run start from the beginning. We don't reconnect here; the - // caller (or the persistent loop) will pick up the reset on retry. - await resetLabelerCursor(db, source.did); - cleanup(); - } - } else if (frame.op === -1) { - log.warn(`[labels] ${source.did} error frame: ${JSON.stringify(frame.payload)}`); - cleanup(); - } - }); - - // Workers WebSocket doesn't always emit `open`; just await `done` directly. - await done; -} - -function buildWsUrl(httpEndpoint: string, cursor: number | null): string { - const u = new URL("/xrpc/com.atproto.label.subscribeLabels", httpEndpoint); - // wss:// for HTTPS endpoints — the protocol on the labeler service is - // expected to be HTTPS already (validated at resolution time). - u.protocol = u.protocol === "https:" ? "wss:" : "ws:"; - if (cursor !== null) u.searchParams.set("cursor", String(cursor)); - return u.toString(); -} - -interface DecodedFrame { - op: number; - t: string | undefined; - payload: Record<string, unknown>; -} - -/** Decode an atproto subscription frame: two consecutive CBOR objects. - * Header `{ op, t? }`, payload — shape depends on `t`. Returns null on - * decode failure or non-object frames. */ -function decodeFrame(bytes: Uint8Array): DecodedFrame | null { - try { - const [header, rest] = decodeFirst(bytes); - if (!header || typeof header !== "object") return null; - const op = typeof (header as { op?: number }).op === "number" ? (header as { op: number }).op : 1; - const t = typeof (header as { t?: string }).t === "string" ? (header as { t: string }).t : undefined; - const [payload] = decodeFirst(rest); - if (!payload || typeof payload !== "object") return null; - return { op, t, payload: payload as Record<string, unknown> }; - } catch { - return null; - } -} - -/** Coerce a wire `Label` object into our `IncomingLabel` shape. Returns - * null when required fields are missing — we'd rather skip a row than - * insert one with placeholder values. */ -function normalizeLabel(raw: unknown, expectedSrc: string): IncomingLabel | null { - if (!raw || typeof raw !== "object") return null; - const r = raw as Record<string, unknown>; - const src = typeof r.src === "string" ? r.src : null; - const uri = typeof r.uri === "string" ? r.uri : null; - const val = typeof r.val === "string" ? r.val : null; - const cts = typeof r.cts === "string" ? r.cts : null; - if (!src || !uri || !val || !cts) return null; - // A labeler shouldn't emit labels under a different `src` than its own - // DID — drop them rather than poison our table with cross-issuer rows. - if (src !== expectedSrc) return null; - return { - src, - uri, - val, - cts, - cid: typeof r.cid === "string" ? r.cid : undefined, - neg: r.neg === true, - exp: typeof r.exp === "string" ? r.exp : undefined, - sig: r.sig instanceof Uint8Array ? r.sig : undefined, - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/labels/types.ts b/packages/contrail/src/core/labels/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/labels/types.ts +++ b/packages/contrail/src/core/labels/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/persistent.ts b/packages/contrail/src/core/persistent.ts index c1e1c18..551c495 100644 --- a/packages/contrail/src/core/persistent.ts +++ b/packages/contrail/src/core/persistent.ts @@ -1,243 +1 @@ -import type { JetstreamSubscription } from "@atcute/jetstream"; -import type { ContrailConfig, IngestEvent, Database, Logger, ResolvedContrailConfig } from "./types"; -import { getCollectionNsids, getDependentNsids, DEFAULT_FEED_MAX_ITEMS, resolveConfig } from "./types"; -import { initSchema, getLastCursor, saveCursor, applyEvents, pruneFeedItems } from "./db"; -import { refreshStaleIdentities } from "./identity"; -import { createIngestState } from "./jetstream"; -import type { IngestState } from "./jetstream"; - -const FEED_PRUNE_INTERVAL_MS = 60 * 60 * 1000; - -export interface PersistentIngestOptions { - batchSize?: number; - flushIntervalMs?: number; - signal?: AbortSignal; - /** Override subscription creation for testing */ - createSubscription?: (cursor: number | null) => JetstreamSubscription; - logger?: Logger; - /** Publish `collection:<nsid>` / `actor:<did>` events for each applied - * public record. Usually supplied by the Contrail instance. */ - pubsub?: import("./realtime/types").PubSub; -} - -function getLogger(config: ContrailConfig, options?: PersistentIngestOptions): Logger { - return options?.logger ?? config.logger ?? console; -} - -export async function runPersistent( - db: Database, - config: ContrailConfig, - options?: PersistentIngestOptions, -): Promise<void> { - // Internals (applyEvents, count updates, query planning) read `_resolved` - // and silently skip features when it's missing. The Contrail class resolves - // in its constructor; callers using this raw export must also get a resolved - // config, so do it defensively here. resolveConfig is idempotent. - if (!(config as ResolvedContrailConfig)._resolved) { - config = resolveConfig(config); - } - const log = getLogger(config, options); - const batchSize = options?.batchSize ?? 50; - const flushIntervalMs = options?.flushIntervalMs ?? 5_000; - const signal = options?.signal; - const state = createIngestState(); - - // Init schema once - if (!state.schemaInitialized) { - await initSchema(db, config); - state.schemaInitialized = true; - } - - // Load known DIDs for dependent collection filtering - const dependentCollections: Set<string> = new Set(getDependentNsids(config)); - let knownDids: Set<string> | undefined; - if (dependentCollections.size > 0) { - const result = await db - .prepare("SELECT did FROM identities") - .all<{ did: string }>(); - knownDids = new Set((result.results ?? []).map((r) => r.did)); - state.cachedKnownDids = knownDids; - log.log(`Loaded ${knownDids.size} known DIDs from database`); - } - - const collections = getCollectionNsids(config); - let reconnectAttempts = 0; - - while (!signal?.aborted) { - const cursor = await getLastCursor(db); - log.log(`Starting persistent ingestion. Cursor: ${cursor ?? "none"}, Collections: ${collections.join(", ")}`); - - try { - await streamAndFlush(db, config, cursor, { - batchSize, - flushIntervalMs, - signal, - collections, - dependentCollections, - knownDids, - state, - log, - createSubscription: options?.createSubscription, - pubsub: options?.pubsub, - }); - reconnectAttempts = 0; - } catch (err) { - if (signal?.aborted) break; - log.error(`Jetstream connection error: ${err}`); - const delay = Math.min(1000 * Math.pow(2, reconnectAttempts), 30_000); - reconnectAttempts++; - log.log(`Reconnecting in ${delay}ms (attempt ${reconnectAttempts})...`); - await new Promise((r) => setTimeout(r, delay)); - } - } - - log.log("Persistent ingestion stopped"); -} - -interface StreamOptions { - batchSize: number; - flushIntervalMs: number; - signal?: AbortSignal; - collections: string[]; - dependentCollections: Set<string>; - knownDids?: Set<string>; - state: IngestState; - log: Logger; - createSubscription?: (cursor: number | null) => any; - pubsub?: import("./realtime/types").PubSub; -} - -async function streamAndFlush( - db: Database, - config: ContrailConfig, - cursor: number | null, - opts: StreamOptions, -): Promise<void> { - const { batchSize, flushIntervalMs, signal, collections, dependentCollections, knownDids, state, log } = opts; - - const subscription = opts.createSubscription - ? opts.createSubscription(cursor) - : new (await import("@atcute/jetstream")).JetstreamSubscription({ - url: config.jetstreams ?? [], - wantedCollections: collections, - ...(cursor !== null ? { cursor } : {}), - onConnectionOpen() { log.log("Connected to Jetstream"); }, - onConnectionClose(event: any) { log.log(`Disconnected: ${event.code} ${event.reason}`); }, - onConnectionError(event: any) { log.error("Jetstream error:", event.error); }, - }); - - const buffer: IngestEvent[] = []; - // Guards against overlap between the periodic timer flush and a main-loop - // batchSize-driven flush. The main loop only ever awaits flush() sequentially, - // but the setInterval callback is a second entry point on another tick. - let flushing = false; - - const flush = async () => { - if (buffer.length === 0 || flushing) return; - flushing = true; - const batch = buffer.splice(0); - - try { - await applyEvents(db, batch, config, { pubsub: opts.pubsub }); - - const lastTimeUs = Math.max(...batch.map((e) => e.time_us)); - await saveCursor(db, lastTimeUs); - - const uniqueDids = [...new Set(batch.map((e) => e.did))]; - if (uniqueDids.length > 0) { - try { - await refreshStaleIdentities(db, uniqueDids); - } catch (err) { - log.warn(`Identity refresh failed: ${err}`); - } - } - - if (config.feeds && Date.now() - state.lastFeedPruneMs > FEED_PRUNE_INTERVAL_MS) { - const maxItems = Math.max( - ...Object.values(config.feeds).map((f) => f.maxItems ?? DEFAULT_FEED_MAX_ITEMS) - ); - const pruned = await pruneFeedItems(db, maxItems); - if (pruned > 0) log.log(`Pruned ${pruned} old feed items`); - state.lastFeedPruneMs = Date.now(); - } - - log.log(`Flushed ${batch.length} events. Cursor: ${lastTimeUs}`); - } finally { - flushing = false; - } - }; - - // Periodic flush decoupled from the main loop. Runs even when Jetstream is - // idle, which is the whole point — without it, buffered events strand until - // the next event or abort. Errors log and retry next interval rather than - // propagate, so transient DB hiccups don't force a reconnect. - const flushTimer = setInterval(() => { - flush().catch((err) => log.error(`Timer flush failed: ${err}`)); - }, flushIntervalMs); - - const onAbort = () => { - clearInterval(flushTimer); - }; - signal?.addEventListener("abort", onAbort, { once: true }); - - const iterator = subscription[Symbol.asyncIterator](); - - try { - while (!signal?.aborted) { - // Per-iteration abort race so the handler can be removed synchronously - // after the race settles — otherwise addEventListener calls accumulate on - // the signal across the streamAndFlush lifetime. - let abortHandler!: () => void; - const abortPromise = new Promise<IteratorResult<any>>((resolve) => { - abortHandler = () => resolve({ value: undefined, done: true }); - signal?.addEventListener("abort", abortHandler, { once: true }); - }); - - let result: IteratorResult<any>; - try { - result = await Promise.race([iterator.next(), abortPromise]); - } finally { - signal?.removeEventListener("abort", abortHandler); - } - - if (result.done) break; - const event = result.value; - - if (event.kind === "commit") { - const { commit } = event; - - if (dependentCollections.has(commit.collection) && knownDids) { - if (!knownDids.has(event.did)) continue; - } - - const now = Date.now(); - const uri = `at://${event.did}/${commit.collection}/${commit.rkey}`; - - buffer.push({ - uri, - did: event.did, - time_us: event.time_us, - collection: commit.collection, - operation: commit.operation as "create" | "update" | "delete", - rkey: commit.rkey, - cid: commit.operation === "delete" ? null : commit.cid, - record: commit.operation === "delete" ? null : JSON.stringify(commit.record), - indexed_at: now * 1000, - }); - - if (knownDids && !dependentCollections.has(commit.collection)) { - knownDids.add(event.did); - } - } - - if (buffer.length >= batchSize) { - await flush(); - } - } - } finally { - clearInterval(flushTimer); - signal?.removeEventListener("abort", onAbort); - await iterator.return?.({ value: undefined, done: true }); - await flush(); - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/durable-object.ts b/packages/contrail/src/core/realtime/durable-object.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/durable-object.ts +++ b/packages/contrail/src/core/realtime/durable-object.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/in-memory.ts b/packages/contrail/src/core/realtime/in-memory.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/in-memory.ts +++ b/packages/contrail/src/core/realtime/in-memory.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/index.ts b/packages/contrail/src/core/realtime/index.ts index 05e39ec..551c495 100644 --- a/packages/contrail/src/core/realtime/index.ts +++ b/packages/contrail/src/core/realtime/index.ts @@ -1,37 +1 @@ -export { registerRealtimeRoutes } from "./router"; -export type { RealtimeRoutesOptions } from "./router"; -export { InMemoryPubSub } from "./in-memory"; -export { DurableObjectPubSub, RealtimePubSubDO } from "./durable-object"; -export type { - DurableObjectId, - DurableObjectNamespace, - DurableObjectStub, - DurableObjectState, -} from "./durable-object"; -export { TicketSigner } from "./ticket"; -export type { TicketPayload } from "./ticket"; -export { wrapWithPublishing } from "./publishing-adapter"; -export { sseResponse } from "./sse"; -export { pumpWebSocket } from "./websocket"; -export type { WebSocketLike } from "./websocket"; -export { mergeAsyncIterables } from "./merge"; -export { resolveTopicForCaller } from "./resolve"; -export type { TopicResolution, TopicResolutionContext, TopicResolutionError } from "./resolve"; -export type { - PubSub, - RealtimeConfig, - RealtimeEvent, - RealtimeEventKind, -} from "./types"; -export { - actorTopic, - collectionTopic, - communityTopic, - parseCommunityTopic, - parseSpaceTopic, - spaceTopic, - isCommunityTopic, - DEFAULT_KEEPALIVE_MS, - DEFAULT_QUEUE_BOUND, - DEFAULT_TICKET_TTL_MS, -} from "./types"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/merge.ts b/packages/contrail/src/core/realtime/merge.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/merge.ts +++ b/packages/contrail/src/core/realtime/merge.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/publishing-adapter.ts b/packages/contrail/src/core/realtime/publishing-adapter.ts index c5dc61f..551c495 100644 --- a/packages/contrail/src/core/realtime/publishing-adapter.ts +++ b/packages/contrail/src/core/realtime/publishing-adapter.ts @@ -1,157 +1 @@ -/** Decorator that wraps a spaces StorageAdapter and publishes realtime events - * after successful writes. Spaces and community modules stay unaware of - * realtime; the decorator is the only integration seam. */ - -import type { StorageAdapter, SpaceMemberRow } from "../spaces/types"; -import type { PubSub, RealtimeEvent } from "./types"; -import { communityTopic, spaceTopic } from "./types"; - -export interface PublishingAdapterOptions { - /** Optional lookup: given a space's ownerDid, return true if that DID is a - * community in the local `communities` table. When provided, writes also - * publish to `community:<ownerDid>` so subscribers who expanded that alias - * at ticket-mint time receive the event. - * - * The lookup is expected to be cheap (cached in the caller) — the decorator - * calls it on every write. */ - isCommunityDid?: (did: string) => Promise<boolean> | boolean; -} - -export function wrapWithPublishing( - inner: StorageAdapter, - pubsub: PubSub, - opts: PublishingAdapterOptions = {} -): StorageAdapter { - const publishSpaceAndCommunity = async ( - spaceUri: string, - ownerDid: string | null, - build: (topic: string) => RealtimeEvent - ): Promise<void> => { - await pubsub.publish(build(spaceTopic(spaceUri))); - if (ownerDid && opts.isCommunityDid && (await opts.isCommunityDid(ownerDid))) { - await pubsub.publish(build(communityTopic(ownerDid))); - } - }; - - const ownerOf = async (spaceUri: string): Promise<string | null> => { - const s = await inner.getSpace(spaceUri); - return s?.ownerDid ?? null; - }; - - const wrapped: StorageAdapter = { - ...inner, - createSpace: inner.createSpace.bind(inner), - getSpace: inner.getSpace.bind(inner), - listSpaces: inner.listSpaces.bind(inner), - deleteSpace: inner.deleteSpace.bind(inner), - updateSpaceAppPolicy: inner.updateSpaceAppPolicy.bind(inner), - getMember: inner.getMember.bind(inner), - listMembers: inner.listMembers.bind(inner), - createInvite: inner.createInvite.bind(inner), - listInvites: inner.listInvites.bind(inner), - revokeInvite: inner.revokeInvite.bind(inner), - getInvite: inner.getInvite.bind(inner), - redeemInvite: inner.redeemInvite.bind(inner), - enroll: inner.enroll.bind(inner), - getEnrollment: inner.getEnrollment.bind(inner), - listEnrollments: inner.listEnrollments.bind(inner), - removeEnrollment: inner.removeEnrollment.bind(inner), - getRecord: inner.getRecord.bind(inner), - listRecords: inner.listRecords.bind(inner), - listCollections: inner.listCollections.bind(inner), - putBlobMeta: inner.putBlobMeta.bind(inner), - getBlobMeta: inner.getBlobMeta.bind(inner), - listBlobMeta: inner.listBlobMeta.bind(inner), - deleteBlobMeta: inner.deleteBlobMeta.bind(inner), - findOrphanBlobs: inner.findOrphanBlobs.bind(inner), - - async addMember(spaceUri, did, addedBy) { - await inner.addMember(spaceUri, did, addedBy); - const owner = await ownerOf(spaceUri); - const now = Date.now(); - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.added", - payload: { space: spaceUri, did }, - ts: now, - })); - }, - - async removeMember(spaceUri, did) { - await inner.removeMember(spaceUri, did); - const owner = await ownerOf(spaceUri); - const now = Date.now(); - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.removed", - payload: { space: spaceUri, did }, - ts: now, - })); - }, - - async applyMembershipDiff(spaceUri, adds, removes, addedBy) { - await inner.applyMembershipDiff(spaceUri, adds, removes, addedBy); - if (adds.length === 0 && removes.length === 0) return; - const owner = await ownerOf(spaceUri); - const now = Date.now(); - for (const did of adds) { - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.added", - payload: { space: spaceUri, did }, - ts: now, - })); - } - for (const did of removes) { - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "member.removed", - payload: { space: spaceUri, did }, - ts: now, - })); - } - }, - - async putRecord(record) { - await inner.putRecord(record); - const owner = await ownerOf(record.spaceUri); - const now = Date.now(); - // Space records use ms timestamps; listRecords surface uses microseconds - // (time_us). Convert here so subscribers can render a row identically. - const time_us = record.createdAt * 1000; - const uri = `at://${record.authorDid}/${record.collection}/${record.rkey}`; - await publishSpaceAndCommunity(record.spaceUri, owner, (topic) => ({ - topic, - kind: "record.created", - payload: { - uri, - did: record.authorDid, - collection: record.collection, - rkey: record.rkey, - cid: record.cid, - record: record.record, - time_us, - space: record.spaceUri, - }, - ts: now, - })); - }, - - async deleteRecord(spaceUri, collection, authorDid, rkey) { - await inner.deleteRecord(spaceUri, collection, authorDid, rkey); - const owner = await ownerOf(spaceUri); - const now = Date.now(); - const uri = `at://${authorDid}/${collection}/${rkey}`; - await publishSpaceAndCommunity(spaceUri, owner, (topic) => ({ - topic, - kind: "record.deleted", - payload: { uri, did: authorDid, collection, rkey, space: spaceUri }, - ts: now, - })); - }, - }; - return wrapped; -} - -// Keep this import hint for types that downstream code might pull from here. -export type { SpaceMemberRow }; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/query-filter.ts b/packages/contrail/src/core/realtime/query-filter.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/query-filter.ts +++ b/packages/contrail/src/core/realtime/query-filter.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/resolve.ts b/packages/contrail/src/core/realtime/resolve.ts index c16abe5..551c495 100644 --- a/packages/contrail/src/core/realtime/resolve.ts +++ b/packages/contrail/src/core/realtime/resolve.ts @@ -1,92 +1 @@ -/** Resolve a raw topic request (as given by a caller) to the concrete set of - * delivery topics they are authorized to subscribe to. - * - * Rules (v1): - * - `space:<uri>` → allowed iff caller is owner or a member of the space. - * - `community:<did>` → expanded to `space:<uri>` for every space in the - * community reachable by the caller (direct grants - * or via delegation). `resolveReachableSpaces` - * already has exactly this semantics. - * - `actor:<did>` → self-only in v1. - * - `collection:<nsid>` → rejected unless the deployment opts in - * (not yet implemented). */ - -import type { StorageAdapter } from "../spaces/types"; -import type { CommunityProbe } from "../community-integration"; -import { spaceTopic, parseCommunityTopic, parseSpaceTopic } from "./types"; - -export interface TopicResolutionContext { - /** May be null when the deployment has no spaces module — in that case - * `space:` and `community:` topics are NotSupported. Public topics - * (`collection:`, `actor:`) still resolve. */ - spaces: StorageAdapter | null; - /** May be null if the community module is not enabled. */ - community: CommunityProbe | null; -} - -export interface TopicResolution { - ok: true; - topics: string[]; -} - -export interface TopicResolutionError { - ok: false; - error: "Forbidden" | "InvalidRequest" | "NotFound" | "NotSupported"; - reason: string; -} - -export async function resolveTopicForCaller( - rawTopic: string, - callerDid: string, - ctx: TopicResolutionContext -): Promise<TopicResolution | TopicResolutionError> { - // space:<uri> - const spaceUri = parseSpaceTopic(rawTopic); - if (spaceUri) { - if (!ctx.spaces) { - return { ok: false, error: "NotSupported", reason: "spaces-module-disabled" }; - } - const space = await ctx.spaces.getSpace(spaceUri); - if (!space) return { ok: false, error: "NotFound", reason: "space-not-found" }; - if (space.ownerDid === callerDid) return { ok: true, topics: [rawTopic] }; - const member = await ctx.spaces.getMember(spaceUri, callerDid); - if (!member) return { ok: false, error: "Forbidden", reason: "not-member" }; - return { ok: true, topics: [rawTopic] }; - } - - // community:<did> - const communityDid = parseCommunityTopic(rawTopic); - if (communityDid) { - if (!ctx.community || !ctx.spaces) { - return { ok: false, error: "NotSupported", reason: "community-module-disabled" }; - } - const row = await ctx.community.getCommunity(communityDid); - if (!row) return { ok: false, error: "NotFound", reason: "community-not-found" }; - const reachable = await ctx.community.resolveReachableSpaces(callerDid); - // Filter to spaces owned by THIS community — reachable may include spaces - // from other communities via cross-community delegation. - const ownedList = await ctx.spaces.listSpaces({ ownerDid: communityDid, limit: 1000 }); - const owned: Set<string> = new Set(ownedList.spaces.map((s) => s.uri)); - const topics: string[] = []; - for (const uri of reachable) { - if (owned.has(uri)) topics.push(spaceTopic(uri)); - } - if (topics.length === 0) { - return { ok: false, error: "Forbidden", reason: "no-reachable-spaces-in-community" }; - } - return { ok: true, topics }; - } - - // actor:<did> — public stream of records authored by this DID. - // Any caller can subscribe (parallels listRecords with an `actor` filter). - if (rawTopic.startsWith("actor:")) { - return { ok: true, topics: [rawTopic] }; - } - - // collection:<nsid> — public firehose for this collection. - if (rawTopic.startsWith("collection:")) { - return { ok: true, topics: [rawTopic] }; - } - - return { ok: false, error: "InvalidRequest", reason: "unknown-topic" }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/router.ts b/packages/contrail/src/core/realtime/router.ts index d10e45e..551c495 100644 --- a/packages/contrail/src/core/realtime/router.ts +++ b/packages/contrail/src/core/realtime/router.ts @@ -1,255 +1 @@ -/** Realtime XRPC routes: ticket mint + subscribe (SSE | WS). */ - -import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig } from "../types"; -import type { ServiceAuth } from "../spaces/auth"; -import type { StorageAdapter } from "../spaces/types"; -import type { CommunityProbe } from "../community-integration"; -import { InMemoryPubSub } from "./in-memory"; -import { TicketSigner } from "./ticket"; -import { sseResponse } from "./sse"; -import { pumpWebSocket, type WebSocketLike } from "./websocket"; -import { mergeAsyncIterables } from "./merge"; -import { resolveTopicForCaller } from "./resolve"; -import type { PubSub, RealtimeEvent } from "./types"; -import { DEFAULT_TICKET_TTL_MS, DEFAULT_KEEPALIVE_MS } from "./types"; - -export interface RealtimeRoutesOptions { - /** Auth middleware for `<ns>.realtime.ticket` and for JWT-based bot - * subscriptions to private topics. Null when no JWT verifier is available - * (deployments without a spaces config) — in that case, private-topic - * subscribe paths return NotSupported and public topics still work without - * auth. */ - authMiddleware: MiddlewareHandler | null; - pubsub?: PubSub; -} - -/** Public topics: subscribable without any auth. Mirrors listRecords - * semantics — no JWT means "public records only". */ -function isPublicTopic(topic: string): boolean { - return topic.startsWith("collection:") || topic.startsWith("actor:"); -} - -/** WebSocketPair exists on Cloudflare Workers; on Node/Bun it's absent. - * When absent, a platform-provided WebSocket accept hook is used instead. */ -interface WebSocketPairCtor { - new (): { 0: WebSocketLike & { accept?: () => void }; 1: WebSocketLike & { accept?: () => void } }; -} - -export function registerRealtimeRoutes( - app: Hono, - config: ContrailConfig, - spaces: StorageAdapter | null, - community: CommunityProbe | null, - options: RealtimeRoutesOptions -): void { - const cfg = config.realtime; - if (!cfg) return; - - const pubsub: PubSub = options.pubsub ?? cfg.pubsub ?? new InMemoryPubSub({ queueBound: cfg.queueBound }); - const signer = new TicketSigner(cfg.ticketSecret); - const ticketTtl = cfg.ticketTtlMs ?? DEFAULT_TICKET_TTL_MS; - const keepaliveMs = cfg.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; - - const NS = `${config.namespace}.realtime`; - - // POST /<ns>.realtime.ticket — { topic } → { ticket, topics, expiresAt } - // Ticket-minting exists so browsers (which can't set Authorization on - // EventSource) can subscribe to *private* topics. Public topics - // (collection:, actor:) don't need tickets — subscribe with `?topic=` directly. - if (options.authMiddleware) { - const authMw = options.authMiddleware; - app.post(`/xrpc/${NS}.ticket`, authMw, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { topic?: string } | null; - if (!body?.topic) { - return c.json({ error: "InvalidRequest", message: "topic required" }, 400); - } - const resolved = await resolveTopicForCaller(body.topic, sa.issuer, { spaces, community }); - if (!resolved.ok) { - const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; - return c.json({ error: resolved.error, reason: resolved.reason }, status); - } - const ticket = await signer.sign({ - topics: resolved.topics, - did: sa.issuer, - ttlMs: ticketTtl, - }); - return c.json({ - ticket, - topics: resolved.topics, - expiresAt: Date.now() + ticketTtl, - }); - }); - } - - // GET /<ns>.realtime.subscribe — SSE or WS. - // - // Three access paths, all land on the same stream: - // - `?topic=collection:<nsid>` or `?topic=actor:<did>` — *public*, no auth. - // Mirrors listRecords semantics. - // - `?ticket=<jwt>` — presented by browsers, minted via `.ticket` after - // a JWT-authenticated call. Only used for private topics. - // - `Authorization: Bearer <jwt>` + `?topic=space:<uri>` — server-side - // bots can skip the ticket dance and go straight to subscribe. - app.get(`/xrpc/${NS}.subscribe`, async (c) => { - const url = new URL(c.req.url); - const ticketParam = url.searchParams.get("ticket"); - const collectionFilter = url.searchParams.get("collection"); - const topicParam = url.searchParams.get("topic"); - - let callerDid: string | null = null; - let topics: string[]; - - if (ticketParam) { - const payload = await signer.verify(ticketParam); - if (!payload) { - return c.json({ error: "AuthRequired", reason: "invalid-or-expired-ticket" }, 401); - } - callerDid = payload.did; - topics = payload.topics; - // Optional: narrow to topics the query explicitly requests. - if (topicParam) { - if (!payload.topics.includes(topicParam)) { - return c.json({ error: "Forbidden", reason: "topic-not-in-ticket" }, 403); - } - topics = [topicParam]; - } - } else if (topicParam && isPublicTopic(topicParam)) { - // Public subscribe — no auth required. Jetstream ingestion publishes - // record events to collection:/actor: topics directly. - topics = [topicParam]; - } else { - // JWT path for private-topic bots. If no auth middleware is available - // (deployment has no spaces config), private topics aren't offered. - if (!options.authMiddleware) { - return c.json( - { - error: "InvalidRequest", - reason: "private-topic-without-auth", - message: - "Subscribing to space:/community: topics requires a JWT verifier; only public topics (collection:, actor:) are available on this deployment.", - }, - 400 - ); - } - let authed = false; - await options.authMiddleware(c, async () => { - authed = true; - }); - if (!authed) return c.res; // middleware already responded with 401 - const sa = getAuth(c); - callerDid = sa.issuer; - if (!topicParam) { - return c.json({ error: "InvalidRequest", message: "topic required" }, 400); - } - const resolved = await resolveTopicForCaller(topicParam, callerDid, { spaces, community }); - if (!resolved.ok) { - const status = resolved.error === "NotFound" ? 404 : resolved.error === "Forbidden" ? 403 : 400; - return c.json({ error: resolved.error, reason: resolved.reason }, status); - } - topics = resolved.topics; - } - - if (topics.length === 0) { - return c.json({ error: "InvalidRequest", reason: "no-topics" }, 400); - } - - // Build the merged iterable, with an inline filter that closes the stream - // on a matching `member.removed` event (self-kick on revocation). - const ac = new AbortController(); - const signals: AbortSignal[] = [ac.signal]; - const reqSignal = c.req.raw.signal; - if (reqSignal) signals.push(reqSignal); - const combined = anySignal(signals); - - const sources = topics.map((t) => pubsub.subscribe(t, combined)); - const merged = withSelfKickAndFilter( - mergeAsyncIterables(sources, combined), - callerDid, - collectionFilter, - ac - ); - - // Content negotiation: Upgrade: websocket → WS, else SSE. - if (c.req.header("Upgrade")?.toLowerCase() === "websocket") { - const Pair = (globalThis as unknown as { WebSocketPair?: WebSocketPairCtor }) - .WebSocketPair; - if (!Pair) { - return c.json( - { error: "NotSupported", reason: "websockets-require-worker-or-ws-adapter" }, - 426 - ); - } - const pair = new Pair(); - const clientWs = pair[0]; - const serverWs = pair[1]; - serverWs.accept?.(); - // Pump in the background; don't await. - void pumpWebSocket(serverWs, merged, combined, { keepaliveMs }); - return new Response(null, { - status: 101, - // Hono/undici-compat: some runtimes honor `webSocket` on the init. - // @ts-expect-error - Workers-specific init field - webSocket: clientWs, - }); - } - - return sseResponse(merged, combined, { keepaliveMs }); - }); -} - -// ============================================================================ -// Helpers -// ============================================================================ - -function getAuth(c: Context): ServiceAuth { - const a = c.get("serviceAuth") as ServiceAuth | undefined; - if (!a) throw new Error("service auth not set"); - return a; -} - -/** Merge multiple AbortSignals into one. Aborts when any source aborts. */ -function anySignal(signals: AbortSignal[]): AbortSignal { - const ac = new AbortController(); - for (const s of signals) { - if (s.aborted) { - ac.abort(); - return ac.signal; - } - s.addEventListener("abort", () => ac.abort(), { once: true }); - } - return ac.signal; -} - -/** Wrap an iterable: drop events that don't pass the collection filter (if - * any), and close the outer controller as soon as we see a `member.removed` - * for the caller's own DID. `callerDid` may be null on public subscriptions - * (anonymous) — in that case self-kick is not applicable. */ -function withSelfKickAndFilter( - source: AsyncIterable<RealtimeEvent>, - callerDid: string | null, - collectionFilter: string | null, - ac: AbortController -): AsyncIterable<RealtimeEvent> { - return { - async *[Symbol.asyncIterator]() { - for await (const event of source) { - if (event.kind === "member.removed" && event.payload.did === callerDid) { - // Deliver the kick event so the client sees why, then close. - yield event; - ac.abort(); - return; - } - if ( - collectionFilter && - (event.kind === "record.created" || event.kind === "record.deleted") && - event.payload.collection !== collectionFilter - ) { - continue; - } - yield event; - } - }, - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/sse.ts b/packages/contrail/src/core/realtime/sse.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/sse.ts +++ b/packages/contrail/src/core/realtime/sse.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/ticket.ts b/packages/contrail/src/core/realtime/ticket.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/ticket.ts +++ b/packages/contrail/src/core/realtime/ticket.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/types.ts b/packages/contrail/src/core/realtime/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/types.ts +++ b/packages/contrail/src/core/realtime/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/realtime/websocket.ts b/packages/contrail/src/core/realtime/websocket.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/realtime/websocket.ts +++ b/packages/contrail/src/core/realtime/websocket.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/refresh.ts b/packages/contrail/src/core/refresh.ts index 7b2fd21..551c495 100644 --- a/packages/contrail/src/core/refresh.ts +++ b/packages/contrail/src/core/refresh.ts @@ -1,267 +1 @@ -import type {} from "@atcute/atproto"; -/** - * Fresh refresh: re-walk every known DID's PDS for every configured collection - * and reconcile against what's in our DB. Unlike `backfillPending`, this - * ignores the `backfills` state machine — it's a "check what we might have - * missed" pass, not a resumable bulk load. - * - * Two categories of delta are counted: - * - missing — the PDS has a record we don't - * - staleUpdates — we have the same URI but a different CID, *and* our - * copy's `indexed_at` is older than `ignoreWindowMs` - * - * The ignore window exists because Jetstream can run ~seconds behind the - * PDS; without the window, "in-sync but racy" writes would show up as - * misses every run. Records inside the window are still applied (they - * might be legit updates), just not counted toward stats. - * - * Typical uses: - * - dev: "I ran backfillAll on Monday, haven't touched it for a week, - * how much did jetstream miss?" - * - prod: "we had jetstream outage yesterday, what did we drop?" - */ -import { type Did, type Nsid } from "@atcute/lexicons"; -import { isDid, isNsid } from "@atcute/lexicons/syntax"; - -import type { Client } from "@atcute/client"; -import type { ContrailConfig, Database, IngestEvent } from "./types.js"; -import { applyEvents, lookupExistingRecords } from "./db/records.js"; -import { getClient } from "./client.js"; - -const PAGE_SIZE = 100; -const REQUEST_TIMEOUT_MS = 10_000; - -async function withTimeout<T>(fn: () => Promise<T>, ms: number): Promise<T> { - return Promise.race([ - fn(), - new Promise<never>((_, rej) => - setTimeout(() => rej(new Error(`timeout after ${ms}ms`)), ms) - ), - ]); -} - -export interface CollectionStats { - /** Record exists on PDS but was absent from our DB. */ - missing: number; - /** Record exists in our DB with a different CID than the PDS, and our - * copy was written before the ignore window. */ - staleUpdates: number; - /** Record is present and matches (same CID, or within ignore window). */ - inSync: number; -} - -export interface RefreshProgress { - usersComplete: number; - usersTotal: number; - usersFailed: number; - recordsScanned: number; -} - -export interface RefreshResult { - /** Per-NSID stats. */ - byCollection: Record<string, CollectionStats>; - /** Sum across every NSID. */ - total: CollectionStats; - usersScanned: number; - usersFailed: number; - /** Effective ignore window used for classification, in ms. */ - ignoreWindowMs: number; - /** Wall-clock runtime, in ms. */ - elapsedMs: number; -} - -export interface RefreshOptions { - /** How many DIDs to fan out against in parallel. Default: 50. */ - concurrency?: number; - /** Records whose local `indexed_at` is within this window of `now` are - * still upserted but excluded from `staleUpdates` counts — guards - * against jetstream being briefly behind the PDS. Default: 60_000 ms. */ - ignoreWindowMs?: number; - /** Override which NSIDs to walk. Default: every `config.collections[*].collection`. */ - nsids?: string[]; - /** Optional progress callback (fires per completed DID). */ - onProgress?: (p: RefreshProgress) => void; - /** Max attempts per listRecords request. Default: 3. */ - maxRetries?: number; - /** Per-request timeout in ms. Default: 10000. */ - requestTimeout?: number; -} - -function emptyStats(): CollectionStats { - return { missing: 0, staleUpdates: 0, inSync: 0 }; -} - -export async function refresh( - db: Database, - config: ContrailConfig, - options?: RefreshOptions -): Promise<RefreshResult> { - const concurrency = options?.concurrency ?? 50; - const ignoreWindowMs = options?.ignoreWindowMs ?? 60_000; - const requestTimeout = options?.requestTimeout ?? REQUEST_TIMEOUT_MS; - const maxRetries = options?.maxRetries ?? 3; - const startedAt = Date.now(); - - // Default to every configured collection NSID. Profiles are already - // included because `resolveConfig` adds them to `config.collections`. - const nsids = - options?.nsids ?? - Object.values(config.collections).map((c) => c.collection); - - const byCollection: Record<string, CollectionStats> = {}; - for (const nsid of nsids) byCollection[nsid] = emptyStats(); - const total: CollectionStats = emptyStats(); - - // Known DIDs = every author we've ever written for. `backfills` is a - // superset (it also includes failed/pending users that we never got - // records from), which is actually what we want — if we tried and - // failed before, we might succeed now. - const didRows = await db - .prepare("SELECT DISTINCT did FROM backfills") - .all<{ did: string }>(); - const dids = (didRows.results ?? []) - .map((r) => r.did) - .filter((d) => isDid(d)); - - const usersTotal = dids.length; - let usersComplete = 0; - let usersFailed = 0; - let recordsScanned = 0; - - const ignoreBeforeUs = (Date.now() - ignoreWindowMs) * 1000; - - const processDid = async (did: string): Promise<void> => { - let client: Client; - try { - client = await withTimeout( - () => getClient(did as Did, db), - requestTimeout - ); - } catch { - usersFailed++; - return; - } - - for (const nsid of nsids) { - if (!isNsid(nsid)) continue; - let cursor: string | undefined; - while (true) { - let pageRecords: Array<{ uri: string; cid: string; value: unknown }>; - let nextCursor: string | undefined; - try { - // Retry listRecords: transient PDS failures are expected during refresh - let attempt = 0; - // eslint-disable-next-line no-constant-condition - while (true) { - try { - const res = await withTimeout( - () => - client.get("com.atproto.repo.listRecords", { - params: { - repo: did as Did, - collection: nsid as Nsid, - limit: PAGE_SIZE, - cursor, - }, - }), - requestTimeout - ); - if (!res.ok) { - // 400s on a collection the user doesn't have are fine; stop - // paging this collection for this user. - pageRecords = []; - nextCursor = undefined; - break; - } - pageRecords = res.data.records; - nextCursor = res.data.cursor ?? undefined; - break; - } catch (err) { - if (attempt >= maxRetries) throw err; - attempt++; - await new Promise((r) => setTimeout(r, 500 * 2 ** attempt)); - } - } - } catch { - // Give up on this collection for this user; keep going. - break; - } - - if (pageRecords.length === 0) break; - - const now = Date.now(); - const events: IngestEvent[] = pageRecords.map((r) => ({ - uri: r.uri, - did, - collection: nsid, - rkey: r.uri.split("/").pop()!, - operation: "create" as const, - cid: r.cid, - record: JSON.stringify(r.value), - time_us: now * 1000, - indexed_at: now * 1000, - })); - - const existing = await lookupExistingRecords( - db, - events.map((e) => ({ uri: e.uri, collection: e.collection })), - false, - config - ); - - for (const ev of events) { - const ex = existing.get(ev.uri); - if (!ex) { - byCollection[nsid].missing++; - total.missing++; - } else if (ex.cid !== ev.cid) { - const inWindow = - ex.indexed_at !== null && ex.indexed_at >= ignoreBeforeUs; - if (inWindow) { - byCollection[nsid].inSync++; - total.inSync++; - } else { - byCollection[nsid].staleUpdates++; - total.staleUpdates++; - } - } else { - byCollection[nsid].inSync++; - total.inSync++; - } - } - - // Upsert everything — even records "inside the ignore window" - // might genuinely have a new CID; we just don't count them as a - // miss-signal. Skip feed fanout since this is a catch-up, not a - // user-visible write. - await applyEvents(db, events, config, { skipFeedFanout: true }); - recordsScanned += events.length; - - cursor = nextCursor; - if (!cursor) break; - } - } - - usersComplete++; - options?.onProgress?.({ - usersComplete, - usersTotal, - usersFailed, - recordsScanned, - }); - }; - - for (let i = 0; i < dids.length; i += concurrency) { - const batch = dids.slice(i, i + concurrency); - await Promise.allSettled(batch.map(processDid)); - } - - return { - byCollection, - total, - usersScanned: usersComplete, - usersFailed, - ignoreWindowMs, - elapsedMs: Date.now() - startedAt, - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/admin.ts b/packages/contrail/src/core/router/admin.ts index a3b754b..551c495 100644 --- a/packages/contrail/src/core/router/admin.ts +++ b/packages/contrail/src/core/router/admin.ts @@ -1,44 +1 @@ -import type { Hono } from "hono"; -import type { ContrailConfig, Database } from "../types"; -import { getCollectionShortNames, recordsTableName, nsidForShortName } from "../types"; -import { getLastCursor } from "../db"; - -export function registerAdminRoutes( - app: Hono, - db: Database, - config: ContrailConfig -): void { - const ns = config.namespace; - - app.get(`/xrpc/${ns}.getCursor`, async (c) => { - const cursor = await getLastCursor(db); - if (cursor === null) return c.json({ cursor: null }); - - const dateMs = Math.floor(cursor / 1000); - return c.json({ - time_us: cursor, - date: new Date(dateMs).toISOString(), - seconds_ago: Math.floor((Date.now() - dateMs) / 1000), - }); - }); - - app.get(`/xrpc/${ns}.getOverview`, async (c) => { - const collections: { collection: string; records: number; unique_users: number }[] = []; - - for (const short of getCollectionShortNames(config)) { - const table = recordsTableName(short); - const nsid = nsidForShortName(config, short) ?? short; - const row = await db - .prepare(`SELECT COUNT(*) as records, COUNT(DISTINCT did) as unique_users FROM ${table}`) - .first<{ records: number; unique_users: number }>(); - if (row) { - collections.push({ collection: nsid, records: row.records, unique_users: row.unique_users }); - } - } - - return c.json({ - total_records: collections.reduce((sum, col) => sum + col.records, 0), - collections, - }); - }); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/collection.ts b/packages/contrail/src/core/router/collection.ts index 6d43d00..551c495 100644 --- a/packages/contrail/src/core/router/collection.ts +++ b/packages/contrail/src/core/router/collection.ts @@ -1,1225 +1 @@ -import type { Context, Hono } from "hono"; -import type { ContrailConfig, ResolvedContrailConfig, Database, RecordRow, QueryableField, RecordSource, RelationConfig } from "../types"; -import { - getCollectionShortNames, - countColumnName, - groupedCountColumnName, - recordsTableName, - nsidForShortName, - getCollectionMethods, -} from "../types"; -import { queryRecords, queryAcrossSources } from "../db"; -import type { SortOption } from "../db/records"; -import { backfillUser } from "../backfill"; -import { resolveHydrates, resolveReferences, parseHydrateParams } from "./hydrate"; -import { resolveProfiles, collectDids } from "./profiles"; -import { resolveActor } from "../identity"; -import type { FormattedRecord } from "./helpers"; -import { formatRecord, parseIntParam, fieldToParam } from "./helpers"; -import { selectAcceptedLabelers } from "../labels/select"; -import { hydrateLabels } from "../labels/hydrate"; -import { verifyServiceAuthRequest, extractInviteToken, checkInviteReadGrant } from "../spaces/auth"; -import { checkAccess } from "../spaces/acl"; -import { hashInviteToken } from "../invite/token"; -import type { SpacesContext } from "."; -import type { Nsid } from "@atcute/lexicons"; -import type { RealtimeEvent } from "../realtime/types"; -import { sseResponse } from "../realtime/sse"; -import { spaceTopic, communityTopic, parseSpaceTopic } from "../realtime/types"; -import type { SubscriberQuerySpec } from "../realtime/durable-object"; -import { DurableObjectPubSub } from "../realtime/durable-object"; -import { TicketSigner, type TicketQuerySpec } from "../realtime/ticket"; -import { resolveTopicForCaller } from "../realtime/resolve"; -import { mergeAsyncIterables } from "../realtime/merge"; -import type { CommunityProbe } from "../community-integration"; -import { getRelationField, getNestedValue } from "../types"; - -/** Scope of a watch stream. - * - `space`: single permissioned space — one `space:<uri>` topic. - * - `actor`: records authored by `actor` across multiple spaces — the - * resolver expanded these to a per-caller subset of space topics (plus - * `actor:<did>` for public records). Events outside `allowedSpaces` - * are filtered out. */ -type WatchScope = - | { kind: "space"; spaceUri: string } - | { - kind: "actor"; - actor: string; - /** Concrete pubsub topics to subscribe to (from resolveTopicForCaller). */ - topics: string[]; - /** Space URIs the caller can see. Events with `space` outside this - * set are dropped. Undefined `space` on an event (public record) - * is allowed only when `actor` topic is in `topics`. */ - allowedSpaces: Set<string>; - }; - -/** Shared implementation of the watchRecords snapshot+live loop. Called by - * both transport branches (SSE and Worker-terminated WS). The caller owns - * the actual socket/stream and provides a `send(kind, data)` closure. */ -async function runQueryStream(opts: { - send: (kind: string, data: unknown) => void; - abort: AbortController; - scope: WatchScope; - callerDid: string | undefined; - params: URLSearchParams; - db: Database; - config: ContrailConfig; - collection: string; - colNsid: string; - pubsub: import("../realtime/types").PubSub; - relations: Record<string, import("../types").RelationConfig>; - references: Record<string, import("../types").ReferenceConfig>; - childCollectionMap: Map< - string, - { relName: string; matchField: string; matchMode: "uri" | "did" } - >; -}): Promise<void> { - const { - send, - abort, - scope, - callerDid, - params, - db, - config, - collection, - colNsid, - pubsub, - relations, - references, - childCollectionMap - } = opts; - - // Predicate: does this event belong in the caller's scope? - const inScope = (space: string | undefined): boolean => { - if (scope.kind === "space") return space === scope.spaceUri; - if (space == null) return false; // actor mode: require space for now (app topic) - return scope.allowedSpaces.has(space); - }; - - const hydrateSpec = parseHydrateParams(params, relations, references); - const trackHydration = Object.keys(hydrateSpec.relations).length > 0; - const parentUris = new Set<string>(); - const parentDids = new Set<string>(); - const childToParent = new Map<string, { parentUri: string; relName: string }>(); - - const primaryUri = (payload: { uri: string }) => payload.uri; - - const handleChildEvent = (event: RealtimeEvent) => { - if (!trackHydration) return; - if (event.kind !== "record.created" && event.kind !== "record.deleted") return; - const meta = childCollectionMap.get(event.payload.collection); - if (!meta) return; - if (!(hydrateSpec.relations as Record<string, number>)[meta.relName]) return; - if (!inScope(event.payload.space)) return; - // Actor mode: additionally require the record's author match our actor - // (the caller might share spaces with other authors — we only surface - // records by the actor under watch). - if (scope.kind === "actor" && event.payload.did !== scope.actor) return; - - if (event.kind === "record.created") { - const matched = getNestedValue(event.payload.record, meta.matchField); - if (matched == null) return; - const parent = - meta.matchMode === "did" - ? parentDids.has(String(matched)) - ? `at://${String(matched)}/${colNsid}/_` - : null - : parentUris.has(String(matched)) - ? String(matched) - : null; - if (!parent) return; - childToParent.set(event.payload.rkey, { - parentUri: parent, - relName: meta.relName - }); - send("hydration.added", { - parentUri: parent, - relation: meta.relName, - child: { - uri: primaryUri(event.payload), - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - value: event.payload.record, - space: event.payload.space - } - }); - } else { - const info = childToParent.get(event.payload.rkey); - if (!info) return; - childToParent.delete(event.payload.rkey); - send("hydration.removed", { - parentUri: info.parentUri, - relation: info.relName, - childRkey: event.payload.rkey, - childDid: event.payload.did - }); - } - }; - - const handleLive = (event: RealtimeEvent) => { - if (abort.signal.aborted) return; - if (event.kind === "member.removed" && event.payload.did === callerDid) { - send("member.removed", event.payload); - abort.abort(); - return; - } - if (event.kind !== "record.created" && event.kind !== "record.deleted") return; - if (!inScope(event.payload.space)) return; - if (scope.kind === "actor" && event.payload.did !== scope.actor) return; - - if (event.payload.collection !== colNsid) { - handleChildEvent(event); - return; - } - - const nowUs = event.ts * 1000; - const uri = primaryUri(event.payload); - if (event.kind === "record.created") { - parentUris.add(uri); - parentDids.add(event.payload.did); - send("record.created", { - record: { - uri, - did: event.payload.did, - rkey: event.payload.rkey, - collection: event.payload.collection, - cid: event.payload.cid, - value: event.payload.record, - time_us: nowUs, - indexed_at: event.ts, - space: event.payload.space - } - }); - } else { - parentUris.delete(uri); - send("record.deleted", { - uri, - did: event.payload.did, - rkey: event.payload.rkey - }); - } - }; - - // Subscribe: one topic for space-scoped, merge across all topics for - // actor-scoped. `mergeAsyncIterables` exists for exactly this case. - let iter: AsyncIterable<RealtimeEvent>; - if (scope.kind === "space") { - iter = pubsub.subscribe(spaceTopic(scope.spaceUri), abort.signal); - } else { - const sources = scope.topics.map((t) => pubsub.subscribe(t, abort.signal)); - iter = mergeAsyncIterables(sources, abort.signal); - } - - const buffered: RealtimeEvent[] = []; - let snapshotDone = false; - - const pump = (async () => { - try { - for await (const event of iter) { - if (abort.signal.aborted) break; - if (!snapshotDone) buffered.push(event); - else handleLive(event); - } - } catch { - /* aborted or errored */ - } - })(); - - try { - send( - "snapshot.start", - scope.kind === "space" - ? { spaceUri: scope.spaceUri, collection: colNsid } - : { actor: scope.actor, collection: colNsid } - ); - const snapshotSpaces = - scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); - const result = await runPipeline(db, config, collection, params, undefined, snapshotSpaces); - for (const record of result.records) { - if (abort.signal.aborted) break; - if (typeof record.uri === "string") parentUris.add(record.uri); - if (typeof record.did === "string") parentDids.add(record.did); - for (const [relName] of Object.entries(hydrateSpec.relations)) { - const hydratedGroups = (record as Record<string, unknown>)[relName]; - if (!hydratedGroups) continue; - const flat: Array<{ rkey?: string }> = Array.isArray(hydratedGroups) - ? (hydratedGroups as Array<{ rkey?: string }>) - : (Object.values(hydratedGroups as Record<string, unknown>).flat() as Array<{ - rkey?: string; - }>); - for (const child of flat) { - if (child?.rkey) { - childToParent.set(child.rkey, { - parentUri: record.uri as string, - relName - }); - } - } - } - send("snapshot.record", { record }); - } - send("snapshot.end", { cursor: result.cursor }); - snapshotDone = true; - for (const event of buffered) handleLive(event); - } catch (err) { - send("error", { - message: err instanceof Error ? err.message : String(err) - }); - abort.abort(); - } - - await pump.catch(() => {}); -} - -export async function runPipeline( - db: Database, - config: ContrailConfig, - collection: string, - params: URLSearchParams, - source?: RecordSource, - spaceUris?: string[], - /** Optional headers from the originating request — used for label - * hydration (`atproto-accept-labelers`). Other entry points pass nothing - * and labels are gated by `?labelers=` / config defaults. */ - headers?: Headers -): Promise<{ records: FormattedRecord[]; cursor?: string; profiles?: any[]; labelersApplied?: string[] }> { - const colConfig = config.collections[collection]; - if (!colConfig) throw new Error(`Unknown collection: ${collection}`); - - const relations = colConfig.relations ?? {}; - const references = colConfig.references ?? {}; - const queryableFields: Record<string, QueryableField> = - (config as ResolvedContrailConfig)._resolved?.queryable[collection] ?? colConfig.queryable ?? {}; - - const limit = parseIntParam(params.get("limit"), 50); - const cursor = params.get("cursor") || undefined; - const actor = params.get("actor") || params.get("did") || undefined; - const wantProfiles = params.get("profiles") === "true"; - - let did: string | undefined; - if (actor) { - const resolved = await resolveActor(db, actor); - if (!resolved) throw new Error("Could not resolve actor"); - did = resolved; - // backfillUser expects the record NSID (for PDS calls), not the short name. - const nsid = nsidForShortName(config, collection) ?? collection; - await backfillUser(db, did, nsid, Date.now() + 3_000, config, { - maxRetries: 0, - requestTimeout: 3_000, - }); - } - - const filters: Record<string, string> = {}; - const rangeFilters: Record<string, { min?: string; max?: string }> = {}; - for (const [field, fieldConfig] of Object.entries(queryableFields)) { - const param = fieldToParam(field); - if (fieldConfig.type === "range") { - const min = params.get(`${param}Min`); - const max = params.get(`${param}Max`); - if (min || max) { - rangeFilters[field] = {}; - if (min) rangeFilters[field].min = min; - if (max) rangeFilters[field].max = max; - } - } else { - const value = params.get(param); - if (value) filters[field] = value; - } - } - - const countFilters: Record<string, number> = {}; - const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; - for (const [relName, rel] of Object.entries(relations)) { - const totalMin = parseIntParam(params.get(`${relName}CountMin`)); - if (totalMin != null) countFilters[rel.collection] = totalMin; - const mapping = relMap[relName]; - if (mapping) { - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - for (const [shortName, fullToken] of Object.entries(mapping.groups)) { - const val = parseIntParam(params.get(`${relName}${capitalize(shortName)}CountMin`)); - if (val != null) countFilters[fullToken] = val; - } - } - } - - let sort: SortOption | undefined; - const sortParam = params.get("sort"); - if (sortParam) { - const orderParam = params.get("order"); - - const fieldEntry = Object.entries(queryableFields).find( - ([field]) => fieldToParam(field) === sortParam - ); - if (fieldEntry) { - const defaultDir = fieldEntry[1].type === "range" ? "desc" : "asc"; - const direction = orderParam === "asc" ? "asc" as const : orderParam === "desc" ? "desc" as const : defaultDir as "asc" | "desc"; - sort = { recordField: fieldEntry[0], direction }; - } else { - const direction = orderParam === "asc" ? "asc" as const : "desc" as const; - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - for (const [relName, rel] of Object.entries(relations)) { - if (sortParam === `${relName}Count`) { - sort = { countType: rel.collection, direction }; - break; - } - const mapping = relMap[relName]; - if (mapping) { - for (const [shortName, fullToken] of Object.entries(mapping.groups)) { - if (sortParam === `${relName}${capitalize(shortName)}Count`) { - sort = { countType: fullToken, direction }; - break; - } - } - if (sort) break; - } - } - } - } - - const search = params.get("search") || undefined; - const spaceUri = params.get("spaceUri") || undefined; - - const queryOpts = { - collection, - did, - limit, - cursor, - filters, - rangeFilters, - countFilters, - sort, - search, - source, - spaceUri, - }; - const result = spaceUris && spaceUris.length > 0 && !spaceUri - ? await queryAcrossSources(db, config, queryOpts, spaceUris) - : await queryRecords(db, config, queryOpts); - - const rows = result.records; - const hydrateRequested = parseHydrateParams(params, relations, references); - const hydrates = await resolveHydrates( - db, - relations, - hydrateRequested.relations, - rows, - config - ); - const refs = await resolveReferences( - db, - references, - hydrateRequested.references, - rows, - config - ); - - const formattedRecords: FormattedRecord[] = rows.map((row) => { - const formatted = formatRecord(row); - flattenCounts(formatted, row.counts, relations); - const h = hydrates[row.uri]; - if (h) { - for (const [relName, groups] of Object.entries(h)) { - formatted[relName] = groups; - } - } - const r = refs[row.uri]; - if (r) { - for (const [refName, record] of Object.entries(r)) { - formatted[refName] = record; - } - } - return formatted; - }); - - const allDids = collectDids(rows, hydrates); - const profileMap = wantProfiles - ? await resolveProfiles(db, config, allDids) - : undefined; - - let labelersApplied: string[] | undefined; - if (config.labels) { - const sel = selectAcceptedLabelers( - headers?.get("atproto-accept-labelers") ?? null, - params.get("labelers"), - config.labels, - ); - if (sel.accepted.length > 0) { - const subjects: string[] = [ - ...formattedRecords.map((r) => r.uri), - ...allDids, - ]; - const cidByUri = new Map<string, string | null>(); - for (const r of formattedRecords) cidByUri.set(r.uri, r.cid); - const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); - for (const fr of formattedRecords) { - const ls = labelsByUri[fr.uri]; - if (ls && ls.length > 0) fr.labels = ls; - } - if (profileMap) { - for (const entries of Object.values(profileMap)) { - for (const entry of entries) { - const ls = labelsByUri[entry.did]; - if (ls && ls.length > 0) entry.labels = ls; - } - } - } - labelersApplied = sel.accepted; - } - } - - return { - records: formattedRecords, - cursor: result.cursor, - ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), - ...(labelersApplied ? { labelersApplied } : {}), - }; -} - -/** Serialize a runPipeline result as JSON, echoing - * `atproto-content-labelers` when labels were applied. The result's - * `labelersApplied` field never appears in the response body — it's a - * side channel for the route to read and turn into a header. */ -function jsonWithLabelers(c: Context, result: { labelersApplied?: string[] } & Record<string, unknown>) { - const { labelersApplied, ...body } = result; - if (labelersApplied && labelersApplied.length > 0) { - c.header("atproto-content-labelers", labelersApplied.join(",")); - } - return c.json(body); -} - -export function registerCollectionRoutes( - app: Hono, - db: Database, - config: ContrailConfig, - spacesCtx?: SpacesContext | null, - options: { - pubsub?: import("../realtime/types").PubSub | null; - community?: CommunityProbe | null; - } = {} -): void { - const ns = config.namespace; - const pubsub = options.pubsub ?? null; - const community = options.community ?? null; - - /** When a per-collection endpoint receives `?spaceUri=...`, verify the JWT, - * resolve membership, run the space ACL, and return the caller DID if allowed. - * Returns null if the spaces subsystem isn't available; the handler should - * then treat the spaceUri as invalid. - * Throws by returning a Response (caller checks via `instanceof Response`). */ - async function gateSpaceAccess( - c: Context, - spaceUri: string, - op: "read" - ): Promise<Response | { callerDid?: string; clientId?: string; viaInviteToken?: boolean }> { - if (!spacesCtx) { - return c.json( - { error: "InvalidRequest", message: "spaces not configured on this service" }, - 501 - ); - } - - // Read-token path: anonymous bearer access via `?inviteToken=...` (or - // `Authorization: Bearer atmo-invite:<token>`). Token must exist, be - // unexpired/unrevoked, scoped to this space, and have a kind that grants - // read (`read` or `read-join`). Token kind cannot grant write — caller must - // separately redeem to become a member for any non-read op. - if (op === "read") { - const rawToken = extractInviteToken(c.req.raw); - if (rawToken) { - const ok = await checkInviteReadGrant( - spacesCtx.adapter, - rawToken, - spaceUri, - hashInviteToken - ); - if (ok) { - const space = await spacesCtx.adapter.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - return { viaInviteToken: true }; - } - return c.json( - { error: "Forbidden", reason: "invalid-invite-token" }, - 403 - ); - } - } - - const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; - const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); - if (!auth) { - return c.json( - { error: "AuthRequired", message: "spaceUri requires a valid service-auth JWT or read-grant invite token" }, - 401 - ); - } - const space = await spacesCtx.adapter.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await spacesCtx.adapter.getMember(spaceUri, auth.issuer); - const result = checkAccess({ - op, - space, - callerDid: auth.issuer, - member, - clientId: auth.clientId, - }); - if (!result.allow) { - return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - return { callerDid: auth.issuer, clientId: auth.clientId }; - } - - for (const collection of getCollectionShortNames(config)) { - const colConfig = config.collections[collection]; - const methods = getCollectionMethods(colConfig); - - if (methods.includes("listRecords")) { - app.get(`/xrpc/${ns}.${collection}.listRecords`, async (c) => { - const params = new URL(c.req.url).searchParams; - const spaceUri = params.get("spaceUri") || undefined; - - if (spaceUri) { - const gated = await gateSpaceAccess(c, spaceUri, "read"); - if (gated instanceof Response) return gated; - // Route through runPipeline with a single-element space list so the - // full filter / sort / hydrate / reference surface works on per-space - // queries too, not just on the cross-space union path. - try { - const result = await runPipeline(db, config, collection, params, undefined, [spaceUri], c.req.raw.headers); - return jsonWithLabelers(c, result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - } - - // Union path: when the caller is authenticated, fold in records from - // spaces they're a member of. Anonymous callers just get public results. - let spaceUris: string[] | undefined; - const hasAuthHeader = !!c.req.header("Authorization"); - if (spacesCtx) { - const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; - const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); - if (auth) { - const { spaces } = await spacesCtx.adapter.listSpaces({ - memberDid: auth.issuer, - limit: 200, - }); - spaceUris = spaces.map((s) => s.uri); - } else if (hasAuthHeader) { - // Had an auth header but it was invalid — reject rather than - // silently downgrading to public results. - return c.json( - { error: "AuthRequired", message: "invalid service-auth JWT" }, - 401 - ); - } - } - - try { - const result = await runPipeline(db, config, collection, params, undefined, spaceUris, c.req.raw.headers); - return jsonWithLabelers(c, result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - }); - - // Streaming variant — same query shape, SSE'd forever. Opted in via the - // presence of the realtime module; no explicit method config needed. - if (pubsub && spacesCtx) { - const colNsid = colConfig.collection; - const relations = colConfig.relations ?? {}; - const references = colConfig.references ?? {}; - // Map child-NSID → { relName, matchField } so we can route child - // events to hydration deltas without re-parsing config per event. - const childCollectionMap = new Map< - string, - { relName: string; matchField: string; matchMode: "uri" | "did" } - >(); - for (const [relName, rel] of Object.entries(relations)) { - const childNsid = nsidForShortName(config, rel.collection) ?? rel.collection; - childCollectionMap.set(childNsid, { - relName, - matchField: getRelationField(rel), - matchMode: rel.match ?? "uri" - }); - } - - // TicketSigner for watch-scoped tickets. Minted on `mode=ws` handshake - // so the subsequent WS upgrade can auth with just `?ticket=...` (no - // cookie or JWT needed — enables cross-origin + stateless clients). - const ticketSigner = config.realtime?.ticketSecret - ? new TicketSigner(config.realtime.ticketSecret) - : null; - const ticketTtl = config.realtime?.ticketTtlMs ?? 120_000; - - app.get(`/xrpc/${ns}.${collection}.watchRecords`, async (c) => { - const params = new URL(c.req.url).searchParams; - const spaceUri = params.get("spaceUri"); - const actorParam = params.get("actor"); - - if (!spaceUri && !actorParam) { - return c.json( - { error: "InvalidRequest", message: "spaceUri or actor required" }, - 400 - ); - } - - // Resolve the caller and their scope. Two parallel paths: - // - space-scoped: single `space:<uri>` topic, per-space ACL gate. - // - actor-scoped: caller's reachable spaces in the actor's - // community (v1 only supports community DIDs as the actor). - // Events are delivered via N `space:<uri>` topics and filtered - // to `did === actor`. - let callerDid: string | undefined; - let scope: WatchScope; - let scopeTopics: string[]; // for ticket signing - let ticketSpec: TicketQuerySpec | null = null; - - const providedTicket = params.get("ticket"); - if (providedTicket && ticketSigner) { - const payload = await ticketSigner.verify(providedTicket); - if (payload?.querySpec && payload.querySpec.collection === colNsid) { - const ts = payload.querySpec; - if (spaceUri && ts.spaceUri === spaceUri) { - if (payload.topics.includes(spaceTopic(spaceUri))) { - callerDid = payload.did; - ticketSpec = { - collection: ts.collection, - spaceUri: ts.spaceUri, - ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) - }; - } - } else if (actorParam && ts.actor === actorParam) { - callerDid = payload.did; - ticketSpec = { - collection: ts.collection, - actor: ts.actor, - ...(ts.hydrate ? { hydrate: ts.hydrate } : {}) - }; - } - } - } - - const hydrateSpec = parseHydrateParams(params, relations, references); - const hydrateForSpec = Object.keys(hydrateSpec.relations).length > 0 - ? Object.fromEntries( - Object.entries(hydrateSpec.relations).map(([relName]) => { - const rel = relations[relName]!; - const childNsid = - nsidForShortName(config, rel.collection) ?? rel.collection; - return [ - relName, - { childCollection: childNsid, matchField: getRelationField(rel) } - ]; - }) - ) - : undefined; - - if (spaceUri) { - if (!ticketSpec) { - const gated = await gateSpaceAccess(c, spaceUri, "read"); - if (gated instanceof Response) return gated; - callerDid = "callerDid" in gated ? gated.callerDid : undefined; - } - scope = { kind: "space", spaceUri }; - scopeTopics = [spaceTopic(spaceUri)]; - } else { - // Actor-scoped path — v1 only supports community DIDs. - const actor = actorParam!; - if (!community || !spacesCtx) { - return c.json( - { error: "NotSupported", reason: "community-module-disabled" }, - 400 - ); - } - const isCommunity = !!(await community.getCommunity(actor)); - if (!isCommunity) { - return c.json( - { error: "InvalidRequest", reason: "actor-must-be-community-did", message: "cross-space watch currently only supports community DIDs as actor" }, - 400 - ); - } - - if (!ticketSpec) { - // Verify the caller via the same JWT/in-process path used for - // per-space queries, then resolve the community topic to the - // caller's accessible space topics. - const nsidLxm = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; - const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsidLxm); - if (!auth) { - return c.json( - { error: "AuthRequired", message: "service-auth JWT or in-process principal required" }, - 401 - ); - } - callerDid = auth.issuer; - } - const resolved = await resolveTopicForCaller(communityTopic(actor), callerDid!, { - spaces: spacesCtx.adapter, - community - }); - if (!resolved.ok) { - const status = - resolved.error === "NotFound" ? 404 : - resolved.error === "Forbidden" ? 403 : 400; - return c.json({ error: resolved.error, reason: resolved.reason }, status); - } - const allowedSpaces = new Set<string>(); - for (const t of resolved.topics) { - const uri = parseSpaceTopic(t); - if (uri) allowedSpaces.add(uri); - } - scope = { kind: "actor", actor, topics: resolved.topics, allowedSpaces }; - scopeTopics = resolved.topics; - } - - const querySpec: TicketQuerySpec = ticketSpec ?? { - collection: colNsid, - ...(spaceUri ? { spaceUri } : { actor: actorParam! }), - ...(hydrateForSpec ? { hydrate: hydrateForSpec } : {}) - }; - - // Upgrade-to-WS path — forward directly to the DO with the spec, - // so the DO terminates the socket and hibernates when idle. - // Requires snapshot to be fetched separately (see `mode=ws` JSON - // handshake below) or accepted as lossy-on-connect for a plain WS - // upgrade. - const isUpgrade = c.req.header("Upgrade")?.toLowerCase() === "websocket"; - const isWsMode = params.get("mode") === "ws"; - - if (isWsMode && !isUpgrade) { - // Handshake: return snapshot + a ticket the client uses to - // upgrade. Ticket carries the (did, topics, querySpec) signed - // so the WS-upgrade route skips any other auth. - try { - const sinceTs = Date.now(); - const snapshotSpaces = - scope.kind === "space" ? [scope.spaceUri] : Array.from(scope.allowedSpaces); - const result = await runPipeline( - db, - config, - collection, - params, - undefined, - snapshotSpaces, - c.req.raw.headers - ); - let ticket: string | undefined; - if (ticketSigner && callerDid) { - ticket = await ticketSigner.sign({ - topics: scopeTopics, - did: callerDid, - ttlMs: ticketTtl, - querySpec - }); - } - const wsUrl = (() => { - const u = new URL(c.req.url); - u.searchParams.delete("mode"); - if (ticket) u.searchParams.set("ticket", ticket); - u.searchParams.set("sinceTs", String(sinceTs)); - return u.pathname + u.search; - })(); - return c.json({ - transport: "ws", - snapshot: { records: result.records, cursor: result.cursor }, - querySpec, - ticket, - ticketTtlMs: ticketTtl, - sinceTs, - wsUrl - }); - } catch (err) { - return c.json( - { error: "SnapshotFailed", message: err instanceof Error ? err.message : String(err) }, - 500 - ); - } - } - - if (isUpgrade && pubsub instanceof DurableObjectPubSub && scope.kind === "space") { - // Forward the WS upgrade to the DO. The DO owns the socket from - // here and hibernates when idle. Replays any events buffered - // since the handshake `sinceTs` so the client closes the gap. - // - // Actor-scoped queries fall through to the worker-terminated - // path below — the DO binding is single-topic today; extending - // it to fan out over N topics is future work. - const sinceTsParam = params.get("sinceTs"); - const sinceTs = sinceTsParam ? Number(sinceTsParam) : 0; - return pubsub.forwardSubscribe(spaceTopic(scope.spaceUri), c.req.raw, { - did: callerDid, - querySpec: { - collection: querySpec.collection, - spaceUri: scope.spaceUri, - ...(querySpec.hydrate ? { hydrate: querySpec.hydrate } : {}) - }, - sinceTs: Number.isFinite(sinceTs) ? sinceTs : 0 - }); - } - - const ac = new AbortController(); - const reqSignal = c.req.raw.signal; - if (reqSignal) { - if (reqSignal.aborted) ac.abort(); - else reqSignal.addEventListener("abort", () => ac.abort(), { once: true }); - } - - // Worker-terminated WebSocket — used when pubsub isn't DO-backed - // (dev InMemoryPubSub). Same query-filter loop as SSE; different - // transport. Runs in the same isolate so no cost benefit, but - // matches the prod protocol. - if (isUpgrade) { - const WsPair = (globalThis as unknown as { WebSocketPair?: any }).WebSocketPair; - if (!WsPair) { - return c.json( - { error: "NotSupported", reason: "websockets-require-workers-runtime" }, - 426 - ); - } - const pair = new WsPair(); - const clientWs = pair[0] as WebSocket; - const serverWs = pair[1] as WebSocket & { accept?: () => void }; - serverWs.accept?.(); - - const sendWs = (kind: string, data: unknown) => { - try { - serverWs.send(JSON.stringify({ kind, data })); - } catch { - ac.abort(); - } - }; - serverWs.addEventListener?.("close", () => ac.abort()); - serverWs.addEventListener?.("error", () => ac.abort()); - - void runQueryStream({ - send: sendWs, - abort: ac, - scope, - callerDid, - params, - db, - config, - collection, - colNsid, - pubsub, - relations, - references, - childCollectionMap - }).finally(() => { - try { - serverWs.close(); - } catch { - /* ignore */ - } - }); - - return new Response(null, { - status: 101, - webSocket: clientWs - } as ResponseInit & { webSocket: unknown }); - } - - // SSE fallback. - const encoder = new TextEncoder(); - const stream = new ReadableStream<Uint8Array>({ - start(controller) { - let closed = false; - const close = () => { - if (closed) return; - closed = true; - try { - controller.close(); - } catch { - /* already closed */ - } - }; - ac.signal.addEventListener("abort", close, { once: true }); - - const send = (kind: string, data: unknown) => { - if (closed) return; - try { - controller.enqueue( - encoder.encode(`event: ${kind}\ndata: ${JSON.stringify(data)}\n\n`) - ); - } catch { - close(); - } - }; - - const keepalive = setInterval(() => { - if (closed) return; - try { - controller.enqueue(encoder.encode(`: keepalive\n\n`)); - } catch { - close(); - } - }, 15_000); - ac.signal.addEventListener( - "abort", - () => clearInterval(keepalive), - { once: true } - ); - - void runQueryStream({ - send, - abort: ac, - scope, - callerDid, - params, - db, - config, - collection, - colNsid, - pubsub, - relations, - references, - childCollectionMap - }).finally(() => close()); - }, - cancel() { - ac.abort(); - }, - }); - - return new Response(stream, { - status: 200, - headers: { - "content-type": "text/event-stream", - "cache-control": "no-cache, no-transform", - connection: "keep-alive", - "x-accel-buffering": "no", - }, - }); - }); - } - } - - if (!methods.includes("getRecord")) { - // Skip getRecord + custom queries unless listRecords-only was explicitly requested. - for (const [queryName, handler] of Object.entries(colConfig.queries ?? {})) { - app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { - const params = new URL(c.req.url).searchParams; - return handler(db, params, config); - }); - } - continue; - } - - app.get(`/xrpc/${ns}.${collection}.getRecord`, async (c) => { - const uri = c.req.query("uri"); - if (!uri) return c.json({ error: "uri parameter required" }, 400); - - // Spaces path — `?spaceUri=` routes to the per-space store + ACL gate. - const spaceUri = c.req.query("spaceUri") || undefined; - if (spaceUri) { - const gated = await gateSpaceAccess(c, spaceUri, "read"); - if (gated instanceof Response) return gated; - - // Parse author + rkey from the record uri `at://<did>/<collection>/<rkey>` - const m = uri.match(/^at:\/\/([^/]+)\/[^/]+\/([^/]+)$/); - if (!m) return c.json({ error: "InvalidRequest", message: "uri must be at://<did>/<collection>/<rkey>" }, 400); - const authorDid = m[1]; - const rkey = m[2]; - - const nsid = colConfig.collection; - const record = await spacesCtx!.adapter.getRecord(spaceUri, nsid, authorDid, rkey); - if (!record) return c.json({ error: "NotFound" }, 404); - return c.json({ record }); - } - - const relations = colConfig.relations ?? {}; - const references = colConfig.references ?? {}; - const relMap = (config as ResolvedContrailConfig)._resolved?.relations[collection] ?? {}; - - const table = recordsTableName(collection); - const countCols = getRelationCountColumns(relations, relMap); - const selectCols = `uri, did, rkey, cid, record, time_us, indexed_at${countCols.length > 0 ? ", " + countCols.map(c => c.column).join(", ") : ""}`; - const row = await db - .prepare(`SELECT ${selectCols} FROM ${table} WHERE uri = ?`) - .bind(uri) - .first<any>(); - - if (!row) return c.json({ error: "Record not found" }, 404); - - const nsid = nsidForShortName(config, collection) ?? collection; - const formatted = formatRecord({ ...row, collection: nsid }); - const counts = extractCounts(row, relations); - if (counts) flattenCounts(formatted, counts, relations); - - const params = new URL(c.req.url).searchParams; - const wantProfilesSingle = params.get("profiles") === "true"; - - const hydrateRequested = parseHydrateParams(params, relations, references); - const hydrates = await resolveHydrates( - db, - relations, - hydrateRequested.relations, - [row], - config - ); - const refs = await resolveReferences( - db, - references, - hydrateRequested.references, - [row], - config - ); - const h = hydrates[row.uri]; - if (h) { - for (const [relName, groups] of Object.entries(h)) { - (formatted as any)[relName] = groups; - } - } - const r = refs[row.uri]; - if (r) { - for (const [refName, record] of Object.entries(r)) { - (formatted as any)[refName] = record; - } - } - - const allDids = collectDids([row], hydrates); - const profileMap = wantProfilesSingle - ? await resolveProfiles(db, config, allDids) - : undefined; - - let labelersApplied: string[] | undefined; - if (config.labels) { - const sel = selectAcceptedLabelers( - c.req.raw.headers.get("atproto-accept-labelers"), - params.get("labelers"), - config.labels, - ); - if (sel.accepted.length > 0) { - const subjects: string[] = [row.uri, ...allDids]; - const cidByUri = new Map<string, string | null>([[row.uri, row.cid]]); - const labelsByUri = await hydrateLabels(db, subjects, sel.accepted, cidByUri); - const ls = labelsByUri[row.uri]; - if (ls && ls.length > 0) (formatted as Record<string, unknown>).labels = ls; - if (profileMap) { - for (const entries of Object.values(profileMap)) { - for (const entry of entries) { - const els = labelsByUri[entry.did]; - if (els && els.length > 0) entry.labels = els; - } - } - } - labelersApplied = sel.accepted; - } - } - if (labelersApplied) { - c.header("atproto-content-labelers", labelersApplied.join(",")); - } - - return c.json({ - ...formatted, - ...(profileMap ? { profiles: Object.values(profileMap).flat() } : {}), - }); - }); - - for (const [queryName, handler] of Object.entries( - colConfig.queries ?? {} - )) { - app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { - const params = new URL(c.req.url).searchParams; - return handler(db, params, config); - }); - } - - for (const [queryName, handler] of Object.entries( - colConfig.pipelineQueries ?? {} - )) { - app.get(`/xrpc/${ns}.${collection}.${queryName}`, async (c) => { - const params = new URL(c.req.url).searchParams; - try { - const source = await handler(db, params, config); - const result = await runPipeline(db, config, collection, params, source, undefined, c.req.raw.headers); - return jsonWithLabelers(c, result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - }); - } - } -} - -function getRelationCountColumns( - relations: Record<string, RelationConfig>, - relMap: Record<string, any> -): { column: string }[] { - const cols: { column: string }[] = []; - for (const [relName, rel] of Object.entries(relations)) { - if (rel.count === false) continue; - cols.push({ column: countColumnName(rel.collection) }); - const mapping = relMap[relName]; - if (mapping?.groups) { - for (const groupKey of Object.keys(mapping.groups as Record<string, string>)) { - cols.push({ column: groupedCountColumnName(rel.collection, groupKey) }); - } - } - } - return cols; -} - -function extractCounts( - row: any, - relations: Record<string, any> -): Record<string, number> | undefined { - const counts: Record<string, number> = {}; - - for (const [, rel] of Object.entries(relations)) { - if (rel.count === false) continue; - const totalCol = countColumnName(rel.collection); - const val = row[totalCol]; - if (val != null && val !== 0) counts[rel.collection] = val; - - if (rel.groups) { - for (const [groupKey, fullToken] of Object.entries(rel.groups as Record<string, string>)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - const gval = row[groupCol]; - if (gval != null && gval !== 0) counts[fullToken] = gval; - } - } - } - - return Object.keys(counts).length > 0 ? counts : undefined; -} - -function flattenCounts( - formatted: FormattedRecord, - counts: Record<string, number> | undefined, - relations: Record<string, any> -): void { - if (!counts) return; - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - - const collectionToRelName: Record<string, string> = {}; - const tokenToField: Record<string, string> = {}; - for (const [relName, rel] of Object.entries(relations)) { - collectionToRelName[rel.collection] = relName; - if (rel.groups) { - for (const [shortName, fullToken] of Object.entries(rel.groups as Record<string, string>)) { - tokenToField[fullToken] = `${relName}${capitalize(shortName)}Count`; - } - } - } - - for (const [type, count] of Object.entries(counts)) { - if (collectionToRelName[type]) { - formatted[`${collectionToRelName[type]}Count`] = count; - } else if (tokenToField[type]) { - formatted[tokenToField[type]] = count; - } - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/feed.ts b/packages/contrail/src/core/router/feed.ts index 0430292..551c495 100644 --- a/packages/contrail/src/core/router/feed.ts +++ b/packages/contrail/src/core/router/feed.ts @@ -1,134 +1 @@ -import type { Hono } from "hono"; -import type { ContrailConfig, Database, FeedConfig } from "../types"; -import { getDialect } from "../dialect"; -import { DEFAULT_FEED_MAX_ITEMS, recordsTableName } from "../types"; -import { resolveActor } from "../identity"; -import { backfillUser } from "../backfill"; -import { runPipeline } from "./collection"; - -async function maybeBackfillFeed( - db: Database, - config: ContrailConfig, - actor: string, - feedName: string, - feedConfig: FeedConfig -): Promise<void> { - const status = await db - .prepare("SELECT completed FROM feed_backfills WHERE actor = ? AND feed = ?") - .bind(actor, feedName) - .first<{ completed: number }>(); - - if (status?.completed) return; - - // Ensure the user's follow records are backfilled first - await backfillUser(db, actor, feedConfig.follow, Date.now() + 3_000, config, { - maxRetries: 0, - requestTimeout: 3_000, - }); - - // Mark as in-progress (idempotent) - await db - .prepare( - "INSERT INTO feed_backfills (actor, feed, completed) VALUES (?, ?, 0) ON CONFLICT DO NOTHING" - ) - .bind(actor, feedName) - .run(); - - const maxItems = feedConfig.maxItems ?? DEFAULT_FEED_MAX_ITEMS; - - // Populate feed from existing records by followed users - const followTable = recordsTableName(feedConfig.follow); - for (const targetCol of feedConfig.targets) { - const targetTable = recordsTableName(targetCol); - await db - .prepare( - getDialect(db).insertOrIgnore( - `INSERT INTO feed_items (actor, uri, collection, time_us) - SELECT ?, r.uri, ?, r.time_us - FROM ${targetTable} r - WHERE r.did IN ( - SELECT ${getDialect(db).jsonExtract('f.record', 'subject')} - FROM ${followTable} f - WHERE f.did = ? - ) - ORDER BY r.time_us DESC - LIMIT ?` - ) - ) - .bind(actor, targetCol, actor, maxItems) - .run(); - } - - // Prune oldest items beyond the cap - await db - .prepare( - `DELETE FROM feed_items WHERE actor = ? AND uri NOT IN ( - SELECT uri FROM feed_items WHERE actor = ? ORDER BY time_us DESC LIMIT ? - )` - ) - .bind(actor, actor, maxItems) - .run(); - - await db - .prepare("UPDATE feed_backfills SET completed = 1 WHERE actor = ? AND feed = ?") - .bind(actor, feedName) - .run(); -} - -export function registerFeedRoutes( - app: Hono, - db: Database, - config: ContrailConfig -): void { - if (!config.feeds) return; - - const ns = config.namespace; - - app.get(`/xrpc/${ns}.getFeed`, async (c) => { - const params = new URL(c.req.url).searchParams; - const feedName = params.get("feed"); - const actor = params.get("actor"); - - if (!feedName || !actor) { - return c.json({ error: "feed and actor parameters required" }, 400); - } - - const feedConfig = config.feeds![feedName]; - if (!feedConfig) { - return c.json({ error: "Unknown feed" }, 404); - } - - const did = await resolveActor(db, actor); - if (!did) return c.json({ error: "Could not resolve actor" }, 400); - - await maybeBackfillFeed(db, config, did, feedName, feedConfig); - - const collection = params.get("collection") || feedConfig.targets[0]; - if (!feedConfig.targets.includes(collection)) { - return c.json({ error: "Collection not in feed targets" }, 400); - } - - // Strip feed-specific params so runPipeline doesn't misinterpret them - // (e.g. "actor" in feeds means "whose feed", not "filter by record creator") - const pipelineParams = new URLSearchParams(params); - pipelineParams.delete("feed"); - pipelineParams.delete("actor"); - pipelineParams.delete("collection"); - - const source = { - joins: "JOIN feed_items f ON r.uri = f.uri", - conditions: ["f.actor = ?"], - params: [did], - }; - - try { - const result = await runPipeline(db, config, collection, pipelineParams, source); - return c.json(result); - } catch (e: any) { - if (e.message === "Could not resolve actor") { - return c.json({ error: e.message }, 400); - } - throw e; - } - }); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/helpers.ts b/packages/contrail/src/core/router/helpers.ts index 7ff372e..551c495 100644 --- a/packages/contrail/src/core/router/helpers.ts +++ b/packages/contrail/src/core/router/helpers.ts @@ -1,67 +1 @@ -import type { Database, RecordRow } from "../types"; - -export interface FormattedRecord { - uri: string; - cid: string | null; - value: unknown; - did: string; - collection: string; - rkey: string; - time_us: number; - [key: string]: unknown; -} - -export function formatRecord(row: RecordRow): FormattedRecord { - let value: unknown = null; - if (row.record) { - try { - value = JSON.parse(row.record); - } catch { - value = row.record; - } - } - return { - uri: row.uri, - cid: row.cid, - value, - did: row.did, - collection: row.collection, - rkey: row.rkey, - time_us: row.time_us, - ...(row.space ? { space: row.space } : {}), - }; -} - -export function parseIntParam( - value: string | null | undefined, - defaultValue?: number -): number | undefined { - if (!value) return defaultValue; - const parsed = parseInt(value, 10); - return isNaN(parsed) ? defaultValue : parsed; -} - -export function fieldToParam(field: string): string { - return field.replace(/\.(\w)/g, (_, c) => c.toUpperCase()); -} - -const BATCH_SIZE = 50; - -export async function batchedInQuery<T>( - db: Database, - sql: string, - prefixBindings: (string | number)[], - inValues: string[] -): Promise<T[]> { - const results: T[] = []; - for (let i = 0; i < inValues.length; i += BATCH_SIZE) { - const chunk = inValues.slice(i, i + BATCH_SIZE); - const query = sql.replace("__IN__", chunk.map(() => "?").join(",")); - const rows = await db - .prepare(query) - .bind(...prefixBindings, ...chunk) - .all<T>(); - results.push(...(rows.results ?? [])); - } - return results; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/hydrate.ts b/packages/contrail/src/core/router/hydrate.ts index 8c1ae63..551c495 100644 --- a/packages/contrail/src/core/router/hydrate.ts +++ b/packages/contrail/src/core/router/hydrate.ts @@ -1,223 +1 @@ -import type { RelationConfig, ReferenceConfig, RecordRow, Database, ContrailConfig } from "../types"; -import { getDialect } from "../dialect"; -import { - getNestedValue, - getRelationField, - recordsTableName, - spacesRecordsTableName, - nsidForShortName, -} from "../types"; -import { batchedInQuery, formatRecord } from "./helpers"; - -/** Group rows by their origin: public (undefined key) or a specific spaceUri. */ -function groupBySource<T extends { space?: string }>(rows: T[]): Map<string | undefined, T[]> { - const groups = new Map<string | undefined, T[]>(); - for (const r of rows) { - const key = r.space; - const g = groups.get(key); - if (g) g.push(r); - else groups.set(key, [r]); - } - return groups; -} - -// --- Hydration: embed related records --- - -export function parseHydrateParams( - params: URLSearchParams, - relations: Record<string, RelationConfig>, - references: Record<string, ReferenceConfig> -): { relations: Record<string, number>; references: Set<string> } { - const relHydrates: Record<string, number> = {}; - const refHydrates = new Set<string>(); - const capitalize = (s: string) => s.charAt(0).toUpperCase() + s.slice(1); - - for (const relName of Object.keys(relations)) { - const val = params.get(`hydrate${capitalize(relName)}`); - if (val) { - const limit = parseInt(val, 10); - if (!isNaN(limit) && limit > 0) { - relHydrates[relName] = Math.min(limit, 50); - } - } - } - - for (const refName of Object.keys(references)) { - const val = params.get(`hydrate${capitalize(refName)}`); - if (val === "true" || val === "1") { - refHydrates.add(refName); - } - } - - return { relations: relHydrates, references: refHydrates }; -} - -// Per-relation hydrate result: array for ungrouped, Record<group, array> for grouped -export type HydrateResult = Record<string, Record<string, any[] | Record<string, any[]>>>; - -export async function resolveHydrates( - db: Database, - relations: Record<string, RelationConfig>, - requested: Record<string, number>, - records: RecordRow[], - config?: ContrailConfig -): Promise<HydrateResult> { - if (Object.keys(requested).length === 0 || records.length === 0) return {}; - - const grouped: Record<string, Record<string, Record<string, any[]>>> = {}; - - const sourceGroups = groupBySource(records); - - for (const [relName, hydrateLimit] of Object.entries(requested)) { - const rel = relations[relName]; - const field = getRelationField(rel); - const matchMode = rel.match ?? "uri"; - - for (const [sourceSpace, sourceRecords] of sourceGroups) { - const matchValues = matchMode === "did" - ? [...new Set(sourceRecords.map((r) => r.did))] - : sourceRecords.map((r) => r.uri); - - if (matchValues.length === 0) continue; - - const groupCount = rel.groupBy ? 10 : 1; - const maxRows = matchValues.length * hydrateLimit * groupCount; - - const table = sourceSpace - ? spacesRecordsTableName(rel.collection) - : recordsTableName(rel.collection); - const where = sourceSpace - ? `space_uri = ? AND ${getDialect(db).jsonExtract('record', field)} IN (__IN__)` - : `${getDialect(db).jsonExtract('record', field)} IN (__IN__)`; - const prefix = sourceSpace ? [sourceSpace] : []; - - const relatedRows = await batchedInQuery<Omit<RecordRow, "collection">>( - db, - `SELECT uri, did, rkey, record, time_us FROM ${table} - WHERE ${where} - ORDER BY time_us DESC - LIMIT ${maxRows}`, - prefix, - matchValues - ); - - for (const row of relatedRows) { - const record = row.record ? JSON.parse(row.record) : null; - const matchedValue = getNestedValue(record, field); - if (!matchedValue) continue; - - const parentUris = matchMode === "did" - ? sourceRecords.filter((r) => r.did === matchedValue).map((r) => r.uri) - : [matchedValue]; - - const groupValue = rel.groupBy - ? String(getNestedValue(record, rel.groupBy) ?? "other") - : "_flat"; - - for (const parentUri of parentUris) { - const targetUri = matchMode === "did" ? parentUri : matchedValue; - - if (!grouped[targetUri]) grouped[targetUri] = {}; - if (!grouped[targetUri][relName]) grouped[targetUri][relName] = {}; - if (!grouped[targetUri][relName][groupValue]) grouped[targetUri][relName][groupValue] = []; - - const group = grouped[targetUri][relName][groupValue]; - if (group.length < hydrateLimit) { - const childNsid = config - ? nsidForShortName(config, rel.collection) ?? rel.collection - : rel.collection; - group.push( - formatRecord({ - ...(row as any), - collection: childNsid, - ...(sourceSpace ? { space: sourceSpace } : {}), - } as RecordRow) - ); - } - } - } - } - } - - const result: HydrateResult = {}; - for (const [uri, rels] of Object.entries(grouped)) { - result[uri] = {}; - for (const [relName, groups] of Object.entries(rels)) { - if (relations[relName].groupBy) { - result[uri][relName] = groups; - } else { - result[uri][relName] = groups["_flat"] ?? []; - } - } - } - - return result; -} - -// --- References: embed records that our records point at --- - -export type ReferenceResult = Record<string, Record<string, any>>; - -export async function resolveReferences( - db: Database, - references: Record<string, ReferenceConfig>, - requested: Set<string>, - records: RecordRow[], - config?: ContrailConfig -): Promise<ReferenceResult> { - if (requested.size === 0 || records.length === 0) return {}; - - const result: ReferenceResult = {}; - - const sourceGroups = groupBySource(records); - - for (const refName of requested) { - const ref = references[refName]; - if (!ref) continue; - - const refNsid = config - ? nsidForShortName(config, ref.collection) ?? ref.collection - : ref.collection; - - for (const [sourceSpace, sourceRecords] of sourceGroups) { - const targetMap = new Map<string, string[]>(); - for (const r of sourceRecords) { - const parsed = r.record ? JSON.parse(r.record) : null; - const targetValue = parsed ? getNestedValue(parsed, ref.field) : null; - if (!targetValue) continue; - if (!targetMap.has(targetValue)) targetMap.set(targetValue, []); - targetMap.get(targetValue)!.push(r.uri); - } - - const targetUris = [...targetMap.keys()]; - if (targetUris.length === 0) continue; - - const table = sourceSpace - ? spacesRecordsTableName(ref.collection) - : recordsTableName(ref.collection); - const where = sourceSpace ? `space_uri = ? AND uri IN (__IN__)` : `uri IN (__IN__)`; - const prefix = sourceSpace ? [sourceSpace] : []; - - const rows = await batchedInQuery<Omit<RecordRow, "collection">>( - db, - `SELECT uri, did, rkey, record, time_us FROM ${table} WHERE ${where}`, - prefix, - targetUris - ); - - for (const row of rows) { - const parentUris = targetMap.get(row.uri) ?? []; - for (const parentUri of parentUris) { - if (!result[parentUri]) result[parentUri] = {}; - result[parentUri][refName] = formatRecord({ - ...(row as any), - collection: refNsid, - ...(sourceSpace ? { space: sourceSpace } : {}), - } as RecordRow); - } - } - } - } - - return result; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/index.ts b/packages/contrail/src/core/router/index.ts index 0c5e058..551c495 100644 --- a/packages/contrail/src/core/router/index.ts +++ b/packages/contrail/src/core/router/index.ts @@ -1,247 +1 @@ -import { Hono } from "hono"; -import { cors } from "hono/cors"; -import type { Database, ContrailConfig } from "../types"; -import { normalizeProfileConfig } from "../types"; -import { registerAdminRoutes } from "./admin"; -import { registerCollectionRoutes } from "./collection"; -import { registerFeedRoutes } from "./feed"; -import { registerNotifyRoute } from "./notify"; -import { registerSpacesRoutes } from "../spaces/router"; -import type { SpacesRoutesOptions } from "../spaces/router"; -import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; -import { HostedAdapter } from "../spaces/adapter"; -import type { StorageAdapter } from "../spaces/types"; -import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; -import type { CommunityIntegration } from "../community-integration"; -import { registerRealtimeRoutes } from "../realtime/router"; -import type { RealtimeRoutesOptions } from "../realtime/router"; -import { registerInviteRoutes } from "../invite/router"; -import { InMemoryPubSub } from "../realtime/in-memory"; -import { wrapWithPublishing } from "../realtime/publishing-adapter"; -import type { PubSub } from "../realtime/types"; -import { resolveActor } from "../identity"; -import { resolveProfiles } from "./profiles"; -import { backfillUser } from "../backfill"; -import { selectAcceptedLabelers } from "../labels/select"; -import { hydrateLabels } from "../labels/hydrate"; -import type { MiddlewareHandler } from "hono"; - -export interface SpacesContext { - adapter: StorageAdapter; - verifier: ServiceJwtVerifier; -} - -export interface CreateAppOptions { - spaces?: SpacesRoutesOptions; - /** Pre-built community integration. Construct via the community package's - * `createCommunityIntegration({ ... })`. When set, contrail wires - * community whoami extension, invite handler, route registration, etc. - * When omitted, deployment runs without community features. */ - community?: CommunityIntegration | null; - /** Auth middleware override for community routes (rare — mostly for tests). */ - communityAuthMiddleware?: MiddlewareHandler; - realtime?: Partial<RealtimeRoutesOptions>; - /** Separate DB for the spaces tables. Defaults to `db`. */ - spacesDb?: Database; - /** Full spaces context override (escape hatch for tests). */ - spacesCtx?: SpacesContext | null; - /** Lexicon JSONs to serve at `/lexicons` so consumer apps can fetch + - * typegen against this deployment. Emit with `contrail-lex generate` — - * its `lexicons/generated/index.ts` exports the right shape. If omitted, - * the endpoint returns `404`. */ - lexicons?: object[]; -} - -export function createApp( - db: Database, - config: ContrailConfig, - options: CreateAppOptions = {} -): Hono { - const app = new Hono(); - app.use("*", cors()); - - app.get("/", (c) => c.json({ status: "ok" })); - app.get("/health", (c) => c.json({ status: "ok" })); - app.get("/xrpc/_health", (c) => c.json({ status: "ok" })); - - const ns = config.namespace; - - // Lexicon manifest — lets consumer apps fetch every lexicon this - // deployment speaks (generated + pulled + custom) over HTTP and - // typegen clients, without needing a PDS or DNS resolution. Only - // registered when the caller passed bundled lexicons at build time - // via `contrail-lex generate`. - if (options.lexicons && options.lexicons.length > 0) { - const lexicons = options.lexicons; - app.get(`/xrpc/${ns}.lexicons`, (c) => c.json({ lexicons })); - } - - app.get(`/xrpc/${ns}.getProfile`, async (c) => { - const actor = c.req.query("actor"); - if (!actor) return c.json({ error: "actor parameter required" }, 400); - - const did = await resolveActor(db, actor); - if (!did) return c.json({ error: "Could not resolve actor" }, 400); - - // Ensure profile records are backfilled - const profileConfigs = (config.profiles ?? []).map(normalizeProfileConfig); - for (const pc of profileConfigs) { - await backfillUser(db, did, pc.collection, Date.now() + 3_000, config, { - maxRetries: 0, - requestTimeout: 3_000, - }); - } - - const profileMap = await resolveProfiles(db, config, [did]); - const profiles = profileMap[did]; - if (!profiles || profiles.length === 0) return c.json({ error: "Profile not found" }, 404); - - if (config.labels) { - const params = new URL(c.req.url).searchParams; - const sel = selectAcceptedLabelers( - c.req.raw.headers.get("atproto-accept-labelers"), - params.get("labelers"), - config.labels, - ); - if (sel.accepted.length > 0) { - const labelsByUri = await hydrateLabels(db, [did], sel.accepted); - const ls = labelsByUri[did]; - if (ls && ls.length > 0) { - for (const entry of profiles) { - entry.labels = ls; - } - } - c.header("atproto-content-labelers", sel.accepted.join(",")); - } - } - - return c.json({ profiles }); - }); - - // Shared spaces context — verifier + adapter — reused by both the per-collection - // routes (for `?spaceUri=...` dispatch) and the `<ns>.space.*` routes. - // Built when an authority is configured (spaces are gated on the authority, - // not the record host — a record-host-only deployment still needs an - // authority somewhere, just possibly external). - const spacesDb = options.spacesDb ?? db; - let spacesCtx: SpacesContext | null = - options.spacesCtx !== undefined - ? options.spacesCtx - : config.spaces?.authority - ? { - adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), - verifier: buildVerifier(config.spaces.authority), - } - : null; - - // Community is provided as a pre-built integration — contrail core never - // imports from the community package. The integration object is opaque; - // we just pass through its probe / whoamiExtension / inviteHandler / - // registerRoutes hooks at the right wiring points. - const community = options.community ?? null; - - // Realtime pubsub is built whenever realtime is configured — independent of - // spaces. With spaces, the spaces adapter is wrapped so private record/member - // events publish to space:/community: topics. Without spaces, only public - // topics (collection:/actor:) see traffic — those are published from - // applyEvents (jetstream ingestion), not from here. - let realtimePubsub: PubSub | null = null; - if (config.realtime) { - realtimePubsub = - options.realtime?.pubsub ?? config.realtime.pubsub ?? new InMemoryPubSub({ - queueBound: config.realtime.queueBound, - }); - if (spacesCtx) { - const isCommunityDid = community - ? cachedIsCommunityDid(community.probe) - : undefined; - spacesCtx = { - ...spacesCtx, - adapter: wrapWithPublishing(spacesCtx.adapter, realtimePubsub, { isCommunityDid }), - }; - } - } - - registerAdminRoutes(app, db, config); - - registerCollectionRoutes(app, db, config, spacesCtx, { - pubsub: realtimePubsub, - community: community?.probe ?? null, - }); - registerFeedRoutes(app, db, config); - registerNotifyRoute(app, db, config); - - // Spaces routes — get a whoami extension from the community integration - // when one's wired so community-owned spaces get an `accessLevel` field. - const spacesOptions = { - ...options.spaces, - whoamiExtension: - options.spaces?.whoamiExtension ?? community?.whoamiExtension, - }; - registerSpacesRoutes(app, spacesDb, config, spacesOptions, spacesCtx); - - if (community && spacesCtx) { - // Community routes reuse the spaces service-auth middleware (same JWT verifier). - const authMiddleware = - options.communityAuthMiddleware ?? - options.spaces?.authMiddleware ?? - createServiceAuthMiddleware(spacesCtx.verifier); - community.registerRoutes(app, { authMiddleware }); - } - - if (config.spaces?.authority && spacesCtx) { - // Unified invite surface: one `<ns>.invite.*` family that dispatches on - // space ownership (user-owned → addMember; community-owned → grant via - // the integration's invite handler). - const authMiddleware = - options.spaces?.authMiddleware ?? - createServiceAuthMiddleware(spacesCtx.verifier); - registerInviteRoutes( - app, - config, - spacesCtx.adapter, - community?.inviteHandler ?? null, - { authMiddleware } - ); - } - - if (config.realtime && realtimePubsub) { - // The ticket endpoint still needs a JWT verifier — but that verifier only - // exists when spaces is configured. Without spaces, private-topic ticket - // minting simply isn't offered; public subscriptions (collection:/actor:) - // require no auth and still work. - const authMiddleware = spacesCtx - ? options.realtime?.authMiddleware ?? - options.spaces?.authMiddleware ?? - createServiceAuthMiddleware(spacesCtx.verifier) - : null; - registerRealtimeRoutes( - app, - config, - spacesCtx?.adapter ?? null, - community?.probe ?? null, - { - authMiddleware, - pubsub: realtimePubsub, - } - ); - } - - return app; -} - -function cachedIsCommunityDid( - probe: import("../community-integration").CommunityProbe -): (did: string) => Promise<boolean> { - const TTL = 60_000; - const cache = new Map<string, { value: boolean; expires: number }>(); - return async (did: string) => { - const now = Date.now(); - const hit = cache.get(did); - if (hit && hit.expires > now) return hit.value; - const row = await probe.getCommunity(did); - const value = row != null; - cache.set(did, { value, expires: now + TTL }); - return value; - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/notify.ts b/packages/contrail/src/core/router/notify.ts index 5ff630b..551c495 100644 --- a/packages/contrail/src/core/router/notify.ts +++ b/packages/contrail/src/core/router/notify.ts @@ -1,179 +1 @@ -import type { Hono } from "hono"; -import type { Database, ContrailConfig, IngestEvent } from "../types"; -import { shortNameForNsid } from "../types"; -import { applyEvents, lookupExistingRecords } from "../db/records"; -import { getPDS } from "../client"; -import type { Did } from "@atcute/lexicons"; - -/** Parse an AT URI into its components. */ -export function parseAtUri(uri: string): { did: string; collection: string; rkey: string } | null { - const match = uri.match(/^at:\/\/(did:[^/]+)\/([^/]+)\/([^/]+)$/); - if (!match) return null; - return { did: match[1], collection: match[2], rkey: match[3] }; -} - -/** - * Fetch a single record from the user's PDS. - * Returns the record + cid on success, null if not found. - */ -async function fetchRecordFromPDS( - pds: string, - did: string, - collection: string, - rkey: string -): Promise<{ value: unknown; cid: string } | null> { - const url = new URL(`/xrpc/com.atproto.repo.getRecord`, pds); - url.searchParams.set("repo", did); - url.searchParams.set("collection", collection); - url.searchParams.set("rkey", rkey); - - const res = await fetch(url.toString()); - if (!res.ok) return null; - - const data = (await res.json()) as { value?: unknown; cid?: string }; - if (!data.value || !data.cid) return null; - return { value: data.value, cid: data.cid }; -} - -export interface NotifyResult { - indexed: number; - deleted: number; - errors?: string[]; -} - -/** - * Process notify URIs: fetch from PDS, detect changes, apply events. - * Shared by both the Hono route and the Contrail.notify() method. - */ -export async function processNotifyUris( - db: Database, - config: ContrailConfig, - uris: string[] -): Promise<NotifyResult> { - const events: IngestEvent[] = []; - const errors: string[] = []; - - // Validate and parse all URIs first - const validUris: { uri: string; parsed: { did: string; collection: string; rkey: string } }[] = []; - for (const uri of uris) { - const parsed = parseAtUri(uri); - if (!parsed) { - errors.push(`invalid AT URI: ${uri}`); - continue; - } - // `parsed.collection` is an NSID; look up the matching short name. - if (!shortNameForNsid(config, parsed.collection)) { - errors.push(`collection not tracked: ${parsed.collection}`); - continue; - } - validUris.push({ uri, parsed }); - } - - // Single batch lookup for all existing records (cid + record in one query) - const existing = await lookupExistingRecords( - db, - validUris.map(({ uri, parsed }) => ({ uri, collection: parsed.collection })), - true, - config - ); - - for (const { uri, parsed } of validUris) { - const pds = await getPDS(parsed.did as Did, db); - if (!pds) { - errors.push(`could not resolve PDS for ${parsed.did}`); - continue; - } - - const result = await fetchRecordFromPDS( - pds, - parsed.did, - parsed.collection, - parsed.rkey - ); - - const now = Date.now() * 1000; // microseconds - const existingInfo = existing.get(uri); - - if (result) { - if (existingInfo?.cid === result.cid) { - // Same CID — nothing changed - continue; - } - - events.push({ - uri, - did: parsed.did, - collection: parsed.collection, - rkey: parsed.rkey, - operation: existingInfo ? "update" : "create", - cid: result.cid, - record: JSON.stringify(result.value), - time_us: now, - indexed_at: now, - }); - } else if (existingInfo) { - // Record gone from PDS but exists locally — delete it. - events.push({ - uri, - did: parsed.did, - collection: parsed.collection, - rkey: parsed.rkey, - operation: "delete", - cid: null, - record: existingInfo.record, - time_us: now, - indexed_at: now, - }); - } - } - - if (events.length > 0) { - // Pass pre-fetched existing records so applyEvents skips re-querying - await applyEvents(db, events, config, { existing }); - } - - return { - indexed: events.filter((e) => e.operation === "create" || e.operation === "update").length, - deleted: events.filter((e) => e.operation === "delete").length, - errors: errors.length > 0 ? errors : undefined, - }; -} - -export function registerNotifyRoute( - app: Hono, - db: Database, - config: ContrailConfig -) { - // Endpoint is off by default. Set config.notify to true or a secret string to enable. - if (!config.notify) return; - - const ns = config.namespace; - const secret = typeof config.notify === "string" ? config.notify : null; - - app.post(`/xrpc/${ns}.notifyOfUpdate`, async (c) => { - if (secret) { - const auth = c.req.header("Authorization"); - if (auth !== `Bearer ${secret}`) { - return c.json({ error: "unauthorized" }, 401); - } - } - - const body = await c.req.json<{ uri?: string; uris?: string[] }>().catch(() => null); - const uris: string[] = []; - - if (body?.uris && Array.isArray(body.uris)) { - uris.push(...body.uris); - } else if (body?.uri) { - uris.push(body.uri); - } else { - return c.json({ error: "uri or uris required" }, 400); - } - - if (uris.length > 25) { - return c.json({ error: "max 25 URIs per request" }, 400); - } - - const result = await processNotifyUris(db, config, uris); - return c.json(result); - }); -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/router/profiles.ts b/packages/contrail/src/core/router/profiles.ts index 5552357..551c495 100644 --- a/packages/contrail/src/core/router/profiles.ts +++ b/packages/contrail/src/core/router/profiles.ts @@ -1,181 +1 @@ -import type { Database, ContrailConfig, RecordRow, ProfileConfig } from "../types"; -import { recordsTableName, normalizeProfileConfig } from "../types"; -import { resolveIdentities } from "../identity"; -import { getPDS } from "../client"; -import type { Did } from "@atcute/lexicons"; -import { batchedInQuery } from "./helpers"; - -export interface ProfileEntry { - did: string; - handle: string | null; - uri?: string; - cid?: string | null; - value?: unknown; - collection?: string; - rkey?: string; - /** Hydrated by the labels module when the caller has accepted-labelers - * active and there are matching labels on this DID. */ - labels?: unknown; -} - -export function collectDids( - records: RecordRow[], - hydrates: Record<string, Record<string, any[] | Record<string, any[]>>> -): string[] { - const dids = new Set(records.map((r) => r.did)); - for (const rels of Object.values(hydrates)) { - for (const value of Object.values(rels)) { - const items = Array.isArray(value) - ? value - : Object.values(value).flat(); - for (const item of items) { - if (item.did) dids.add(item.did); - } - } - } - return [...dids]; -} - -export async function resolveProfiles( - db: Database, - config: ContrailConfig, - dids: string[] -): Promise<Record<string, ProfileEntry[]>> { - if (dids.length === 0 || !config.profiles || config.profiles.length === 0) { - return {}; - } - - const profileConfigs = config.profiles.map(normalizeProfileConfig); - const result: Record<string, ProfileEntry[]> = {}; - - // Batch-lookup profile records for each configured profile collection - for (const pc of profileConfigs) { - const { collection, rkey: configRkey, shortName } = pc; - const rkey = configRkey ?? "self"; - const table = recordsTableName(shortName ?? collection); - const uris = dids.map((did) => `at://${did}/${collection}/${rkey}`); - - const rows = await batchedInQuery<Omit<RecordRow, "collection">>( - db, - `SELECT uri, did, rkey, cid, record FROM ${table} WHERE uri IN (__IN__)`, - [], - uris - ); - - for (const row of rows) { - let value: unknown = null; - if (row.record) { - try { - value = JSON.parse(row.record); - } catch { - value = row.record; - } - } - if (!result[row.did]) result[row.did] = []; - result[row.did].push({ - did: row.did, - handle: null, // filled below - uri: row.uri, - collection, - rkey: row.rkey, - cid: row.cid, - value, - }); - } - } - - // Resolve identities for all DIDs - const identities = await resolveIdentities(db, dids); - - // Fetch missing profile records from PDS on demand - const missingDids = dids.filter((d) => !result[d]); - if (missingDids.length > 0 && profileConfigs.length > 0) { - const fetched = await fetchMissingProfiles(db, config, missingDids); - for (const [did, entries] of Object.entries(fetched)) { - if (!result[did]) result[did] = []; - result[did].push(...entries); - } - } - - // Fill in handles and create entries for DIDs without profile records - for (const did of dids) { - const identity = identities.get(did); - const handle = identity?.handle ?? null; - - if (result[did]) { - for (const entry of result[did]) { - entry.handle = handle; - } - } else { - result[did] = [{ did, handle }]; - } - } - - return result; -} - -/** - * Fetch profile records from PDS for DIDs not yet in the index. - * Fetches in parallel across all configured profile collections, - * indexes the results into D1 for future requests. - */ -async function fetchMissingProfiles( - db: Database, - config: ContrailConfig, - dids: string[] -): Promise<Record<string, ProfileEntry[]>> { - const result: Record<string, ProfileEntry[]> = {}; - const profileConfigs = config.profiles!.map(normalizeProfileConfig); - - await Promise.all( - dids.flatMap((did) => - profileConfigs.map(async (pc) => { - const { collection, rkey: configRkey, shortName } = pc; - const rkey = configRkey ?? "self"; - const table = recordsTableName(shortName ?? collection); - try { - const pds = await getPDS(did as Did, db); - if (!pds) return; - - const url = new URL("/xrpc/com.atproto.repo.getRecord", pds); - url.searchParams.set("repo", did); - url.searchParams.set("collection", collection); - url.searchParams.set("rkey", rkey); - - const res = await fetch(url.toString()); - if (!res.ok) return; - - const data = (await res.json()) as { uri?: string; value?: unknown; cid?: string }; - if (!data.value || !data.cid) return; - - const uri = data.uri ?? `at://${did}/${collection}/${rkey}`; - const record = data.value; - const cid = data.cid; - - // Index into D1 for future requests - await db - .prepare( - `INSERT INTO ${table} (uri, did, rkey, cid, record, time_us, indexed_at) VALUES (?, ?, ?, ?, ?, ?, ?) ON CONFLICT(uri) DO UPDATE SET cid = excluded.cid, record = excluded.record, indexed_at = excluded.indexed_at` - ) - .bind(uri, did, rkey, cid, JSON.stringify(record), Date.now() * 1000, Date.now()) - .run(); - - if (!result[did]) result[did] = []; - result[did].push({ - did, - handle: null, - uri, - collection, - rkey, - cid, - value: record, - }); - } catch { - // Skip failures silently - } - }) - ) - ); - - return result; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/search.ts b/packages/contrail/src/core/search.ts index a33dc5e..551c495 100644 --- a/packages/contrail/src/core/search.ts +++ b/packages/contrail/src/core/search.ts @@ -1,31 +1 @@ -import type { CollectionConfig } from "./types"; -import { getNestedValue } from "./types"; - -/** - * Resolve which fields are searchable for a collection. - * Returns null if search is disabled or no fields found. - */ -export function getSearchableFields( - collection: string, - colConfig: CollectionConfig -): string[] | null { - if (!Array.isArray(colConfig.searchable)) return null; - return colConfig.searchable.length > 0 ? colConfig.searchable : null; -} - -/** Sanitized FTS table name for a collection. */ -export function ftsTableName(collection: string): string { - return `fts_${collection.replace(/[^a-zA-Z0-9]/g, "_")}`; -} - -/** Extract searchable field values from a record and join them into a single string. */ -export function buildFtsContent(record: unknown, fields: string[]): string | null { - const parts: string[] = []; - for (const field of fields) { - const value = getNestedValue(record, field); - if (typeof value === "string" && value.length > 0) { - parts.push(value); - } - } - return parts.length > 0 ? parts.join(" ") : null; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/acl.ts b/packages/contrail/src/core/spaces/acl.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/acl.ts +++ b/packages/contrail/src/core/spaces/acl.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/adapter.ts b/packages/contrail/src/core/spaces/adapter.ts index 50fb33d..551c495 100644 --- a/packages/contrail/src/core/spaces/adapter.ts +++ b/packages/contrail/src/core/spaces/adapter.ts @@ -1,505 +1 @@ -/** Contrail's all-in-one default adapter — extends the authority package's - * {@link HostedAuthorityAdapter} (which owns space lifecycle, member list, - * invites) and adds the record-host methods (records, blobs, enrollment). - * - * Phase 7a step 3 will lift the record-host methods into a separate - * HostedRecordHostAdapter, at which point this class becomes a thin - * composition / re-export. For now we keep both roles in one class so - * consumers can wire a single object that satisfies the full StorageAdapter. */ - -import type { ContrailConfig, RelationConfig, ResolvedContrailConfig } from "../types"; -import { - shortNameForNsid, - spacesRecordsTableName, - countColumnName, - groupedCountColumnName, - getRelationField, - getNestedValue, -} from "../types"; -import { getDialect } from "../dialect"; -import type { - BlobMetaRow, - CollectionCount, - EnrollmentRow, - ListBlobsOptions, - ListBlobsResult, - ListOptions, - ListResult, - StorageAdapter, - StoredRecord, -} from "./types"; -import type { Database } from "../types"; -import { buildRecordUri } from "./uri"; -import { HostedAuthorityAdapter, parseJson, toNum } from "@atmo-dev/contrail-authority"; - -function mapBlobMetaRow(row: any): BlobMetaRow { - return { - spaceUri: row.space_uri, - cid: row.cid, - mimeType: row.mime_type, - size: Number(row.size), - authorDid: row.author_did, - createdAt: toNum(row.created_at), - }; -} - -function mapEnrollmentRow(row: any): EnrollmentRow { - return { - spaceUri: row.space_uri, - authorityDid: row.authority_did, - enrolledAt: toNum(row.enrolled_at), - enrolledBy: row.enrolled_by, - }; -} - -/** Row mapper for per-collection spaces_records_<short> tables. - * `collection` is injected by the caller (known from the table name). */ -function mapRecordRow(row: any, collection: string): StoredRecord { - return { - spaceUri: row.space_uri, - collection, - authorDid: row.did, - rkey: row.rkey, - cid: row.cid ?? null, - record: parseJson<Record<string, unknown>>(row.record) ?? {}, - createdAt: toNum(row.time_us), - }; -} - -export class HostedAdapter extends HostedAuthorityAdapter implements StorageAdapter { - /** Resolve the per-collection spaces table name, or throw if the collection - * isn't configured (and therefore has no table). */ - private tableFor(collection: string): string { - if (!this.config) { - throw new Error( - `HostedAdapter: config not provided; cannot resolve table for collection ${collection}` - ); - } - const short = shortNameForNsid(this.config, collection); - if (!short) { - throw new Error( - `HostedAdapter: collection ${collection} is not configured in this deployment` - ); - } - return spacesRecordsTableName(short); - } - - // ---- Enrollment ---- - - async enroll(input: EnrollmentRow): Promise<void> { - await this.db - .prepare( - `INSERT INTO record_host_enrollments (space_uri, authority_did, enrolled_at, enrolled_by) - VALUES (?, ?, ?, ?) - ON CONFLICT (space_uri) DO UPDATE SET - authority_did = excluded.authority_did, - enrolled_at = excluded.enrolled_at, - enrolled_by = excluded.enrolled_by` - ) - .bind(input.spaceUri, input.authorityDid, input.enrolledAt, input.enrolledBy) - .run(); - } - - async getEnrollment(spaceUri: string): Promise<EnrollmentRow | null> { - const row = await this.db - .prepare(`SELECT * FROM record_host_enrollments WHERE space_uri = ?`) - .bind(spaceUri) - .first<any>(); - return row ? mapEnrollmentRow(row) : null; - } - - async listEnrollments( - options: { authorityDid?: string; limit?: number } = {} - ): Promise<EnrollmentRow[]> { - const limit = Math.min(options.limit ?? 200, 1000); - if (options.authorityDid) { - const { results } = await this.db - .prepare( - `SELECT * FROM record_host_enrollments WHERE authority_did = ? ORDER BY enrolled_at DESC LIMIT ?` - ) - .bind(options.authorityDid, limit) - .all<any>(); - return results.map(mapEnrollmentRow); - } - const { results } = await this.db - .prepare(`SELECT * FROM record_host_enrollments ORDER BY enrolled_at DESC LIMIT ?`) - .bind(limit) - .all<any>(); - return results.map(mapEnrollmentRow); - } - - async removeEnrollment(spaceUri: string): Promise<void> { - await this.db - .prepare(`DELETE FROM record_host_enrollments WHERE space_uri = ?`) - .bind(spaceUri) - .run(); - } - - // ---- Records ---- - - async putRecord(record: StoredRecord): Promise<void> { - const table = this.tableFor(record.collection); - const uri = buildRecordUri(record.authorDid, record.collection, record.rkey); - - const childShort = this.config ? shortNameForNsid(this.config, record.collection) : null; - const prev = childShort - ? await this.db - .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) - .bind(record.spaceUri, record.authorDid, record.rkey) - .first<{ record: unknown } | null>() - : null; - const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); - - await this.db - .prepare( - `INSERT INTO ${table} (space_uri, uri, did, rkey, cid, record, time_us, indexed_at) - VALUES (?, ?, ?, ?, ?, ?, ?, ?) - ON CONFLICT (space_uri, did, rkey) DO UPDATE SET - uri = excluded.uri, - cid = excluded.cid, - record = excluded.record, - time_us = excluded.time_us, - indexed_at = excluded.indexed_at` - ) - .bind( - record.spaceUri, - uri, - record.authorDid, - record.rkey, - record.cid, - JSON.stringify(record.record), - record.createdAt, - Date.now() - ) - .run(); - - if (childShort && this.config) { - await this.recountParentsForSpace( - record.spaceUri, - childShort, - beforeRecord, - record.record, - record.authorDid - ); - } - } - - async getRecord( - spaceUri: string, - collection: string, - authorDid: string, - rkey: string - ): Promise<StoredRecord | null> { - const table = this.tableFor(collection); - const row = await this.db - .prepare( - `SELECT * FROM ${table} - WHERE space_uri = ? AND did = ? AND rkey = ?` - ) - .bind(spaceUri, authorDid, rkey) - .first<any>(); - return row ? mapRecordRow(row, collection) : null; - } - - async listRecords( - spaceUri: string, - collection: string, - options: ListOptions = {} - ): Promise<ListResult> { - const table = this.tableFor(collection); - const limit = Math.min(options.limit ?? 50, 200); - const clauses: string[] = ["space_uri = ?"]; - const params: any[] = [spaceUri]; - - if (options.byUser) { - clauses.push("did = ?"); - params.push(options.byUser); - } - if (options.cursor) { - clauses.push("time_us < ?"); - params.push(Number(options.cursor)); - } - - const sql = `SELECT * FROM ${table} - WHERE ${clauses.join(" AND ")} - ORDER BY time_us DESC - LIMIT ?`; - params.push(limit + 1); - - const { results } = await this.db.prepare(sql).bind(...params).all<any>(); - const records = results.map((r) => mapRecordRow(r, collection)); - let cursor: string | undefined; - if (records.length > limit) { - const next = records.pop()!; - cursor = String(next.createdAt); - } - return { records, cursor }; - } - - async deleteRecord( - spaceUri: string, - collection: string, - authorDid: string, - rkey: string - ): Promise<void> { - const table = this.tableFor(collection); - - const childShort = this.config ? shortNameForNsid(this.config, collection) : null; - const prev = childShort - ? await this.db - .prepare(`SELECT record FROM ${table} WHERE space_uri = ? AND did = ? AND rkey = ?`) - .bind(spaceUri, authorDid, rkey) - .first<{ record: unknown } | null>() - : null; - const beforeRecord = parseJson<Record<string, unknown>>(prev?.record ?? null); - - await this.db - .prepare( - `DELETE FROM ${table} - WHERE space_uri = ? AND did = ? AND rkey = ?` - ) - .bind(spaceUri, authorDid, rkey) - .run(); - - if (childShort && this.config) { - await this.recountParentsForSpace(spaceUri, childShort, beforeRecord, null, authorDid); - } - } - - /** Recompute count columns on parent records in the same space, scoped to the - * targets derived from before/after versions of the written/deleted child record. */ - private async recountParentsForSpace( - spaceUri: string, - childShort: string, - before: Record<string, unknown> | null, - after: Record<string, unknown> | null, - childDid: string - ): Promise<void> { - if (!this.config) return; - const config = this.config; - const resolved = (config as ResolvedContrailConfig)._resolved; - const childTable = spacesRecordsTableName(childShort); - - type Inbound = { parentShort: string; relationName: string; rel: RelationConfig }; - const inbound: Inbound[] = []; - for (const [parentShort, parentCfg] of Object.entries(config.collections)) { - if (parentCfg.allowInSpaces === false) continue; - for (const [relName, rel] of Object.entries(parentCfg.relations ?? {})) { - if (rel.count === false) continue; - if (rel.collection !== childShort) continue; - inbound.push({ parentShort, relationName: relName, rel }); - } - } - if (inbound.length === 0) return; - - // Deduplicate (parent, relation, target) across before/after. - const keyed = new Map<string, { parentShort: string; relationName: string; rel: RelationConfig; target: string }>(); - for (const { parentShort, relationName, rel } of inbound) { - const field = getRelationField(rel); - const collectTarget = (rec: Record<string, unknown> | null) => { - if (!rec) return; - if (rel.match === "did") { - keyed.set(`${parentShort}:${relationName}:${childDid}`, { - parentShort, relationName, rel, target: childDid, - }); - return; - } - const v = getNestedValue(rec, field); - if (typeof v === "string" && v.length > 0) { - keyed.set(`${parentShort}:${relationName}:${v}`, { - parentShort, relationName, rel, target: v, - }); - } - }; - collectTarget(before); - collectTarget(after); - } - if (keyed.size === 0) return; - - const dialect = getDialect(this.db); - const stmts: ReturnType<Database["prepare"]>[] = []; - - for (const { parentShort, relationName, rel, target } of keyed.values()) { - const parentTable = spacesRecordsTableName(parentShort); - const matchColumn = rel.match === "did" ? "did" : "uri"; - const field = getRelationField(rel); - const countExpr = rel.countDistinct - ? `COUNT(DISTINCT ${rel.countDistinct})` - : "COUNT(*)"; - - const setClauses: string[] = []; - const binds: (string | number)[] = []; - - const totalCol = countColumnName(rel.collection); - setClauses.push( - `${totalCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ?)` - ); - binds.push(spaceUri, target); - - if (rel.groupBy) { - const mapping = resolved?.relations[parentShort]?.[relationName]; - if (mapping?.groups) { - for (const [groupKey, fullToken] of Object.entries(mapping.groups)) { - const groupCol = groupedCountColumnName(rel.collection, groupKey); - setClauses.push( - `${groupCol} = (SELECT ${countExpr} FROM ${childTable} WHERE space_uri = ? AND ${dialect.jsonExtract("record", field)} = ? AND ${dialect.jsonExtract("record", rel.groupBy)} = ?)` - ); - binds.push(spaceUri, target, fullToken); - } - } - } - - binds.push(spaceUri, target); - stmts.push( - this.db - .prepare( - `UPDATE ${parentTable} SET ${setClauses.join(", ")} WHERE space_uri = ? AND ${matchColumn} = ?` - ) - .bind(...binds) - ); - } - - if (stmts.length > 0) await this.db.batch(stmts); - } - - async listCollections( - spaceUri: string, - options: { byUser?: string } = {} - ): Promise<CollectionCount[]> { - if (!this.config) return []; - const results: CollectionCount[] = []; - for (const [short, colConfig] of Object.entries(this.config.collections)) { - if (colConfig.allowInSpaces === false) continue; - const table = spacesRecordsTableName(short); - const clauses: string[] = ["space_uri = ?"]; - const params: any[] = [spaceUri]; - if (options.byUser) { - clauses.push("did = ?"); - params.push(options.byUser); - } - try { - const row = await this.db - .prepare(`SELECT COUNT(*) AS count FROM ${table} WHERE ${clauses.join(" AND ")}`) - .bind(...params) - .first<{ count: number }>(); - const count = Number(row?.count ?? 0); - if (count > 0) results.push({ collection: colConfig.collection, count }); - } catch { - // table doesn't exist (collection added after init, or allowInSpaces toggled) — skip - } - } - return results; - } - - // ---- Blobs ---- - - async putBlobMeta(row: BlobMetaRow): Promise<void> { - const sql = `INSERT INTO spaces_blobs (space_uri, cid, mime_type, size, author_did, created_at) - VALUES (?, ?, ?, ?, ?, ?) - ON CONFLICT (space_uri, cid) DO NOTHING`; - await this.db - .prepare(sql) - .bind(row.spaceUri, row.cid, row.mimeType, row.size, row.authorDid, row.createdAt) - .run(); - } - - async getBlobMeta(spaceUri: string, cid: string): Promise<BlobMetaRow | null> { - const r = await this.db - .prepare(`SELECT * FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) - .bind(spaceUri, cid) - .first<any>(); - return r ? mapBlobMetaRow(r) : null; - } - - async listBlobMeta( - spaceUri: string, - options: ListBlobsOptions = {} - ): Promise<ListBlobsResult> { - const limit = Math.min(options.limit ?? 50, 200); - const clauses: string[] = ["space_uri = ?"]; - const params: any[] = [spaceUri]; - if (options.byUser) { - clauses.push("author_did = ?"); - params.push(options.byUser); - } - if (options.cursor) { - clauses.push("created_at < ?"); - params.push(Number(options.cursor)); - } - const sql = `SELECT * FROM spaces_blobs - WHERE ${clauses.join(" AND ")} - ORDER BY created_at DESC - LIMIT ?`; - params.push(limit + 1); - const { results } = await this.db.prepare(sql).bind(...params).all<any>(); - const blobs = results.map(mapBlobMetaRow); - let cursor: string | undefined; - if (blobs.length > limit) { - const next = blobs.pop()!; - cursor = String(next.createdAt); - } - return { blobs, cursor }; - } - - async deleteBlobMeta(spaceUri: string, cid: string): Promise<void> { - await this.db - .prepare(`DELETE FROM spaces_blobs WHERE space_uri = ? AND cid = ?`) - .bind(spaceUri, cid) - .run(); - } - - async findOrphanBlobs( - spaceUri: string, - cutoff: number, - limit: number - ): Promise<BlobMetaRow[]> { - if (!this.config) return []; - // Gather candidate blobs older than cutoff, then filter out any whose CID - // appears in any record JSON in this space. We use a cheap substring probe - // (LIKE) per collection — false positives are OK because an orphan that - // survives GC just gets collected next cycle; false negatives (deleting - // a referenced blob) would be a bug, and substring search over the full - // CID is safe enough for that. - const { results } = await this.db - .prepare( - `SELECT * FROM spaces_blobs - WHERE space_uri = ? AND created_at < ? - ORDER BY created_at ASC - LIMIT ?` - ) - .bind(spaceUri, cutoff, limit) - .all<any>(); - const candidates = results.map(mapBlobMetaRow); - if (candidates.length === 0) return []; - - const tables: string[] = []; - for (const [short, colConfig] of Object.entries(this.config.collections)) { - if (colConfig.allowInSpaces === false) continue; - tables.push(spacesRecordsTableName(short)); - } - - const orphans: BlobMetaRow[] = []; - for (const blob of candidates) { - let referenced = false; - const pattern = `%${blob.cid}%`; - for (const table of tables) { - try { - const row = await this.db - .prepare( - `SELECT 1 FROM ${table} WHERE space_uri = ? AND record LIKE ? LIMIT 1` - ) - .bind(spaceUri, pattern) - .first<any>(); - if (row) { - referenced = true; - break; - } - } catch { - // table missing — ignore - } - } - if (!referenced) orphans.push(blob); - } - return orphans; - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/auth.ts b/packages/contrail/src/core/spaces/auth.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/auth.ts +++ b/packages/contrail/src/core/spaces/auth.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/binding.ts b/packages/contrail/src/core/spaces/binding.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/binding.ts +++ b/packages/contrail/src/core/spaces/binding.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/blob-adapter.ts b/packages/contrail/src/core/spaces/blob-adapter.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/blob-adapter.ts +++ b/packages/contrail/src/core/spaces/blob-adapter.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/blob-gc.ts b/packages/contrail/src/core/spaces/blob-gc.ts index d523fcd..551c495 100644 --- a/packages/contrail/src/core/spaces/blob-gc.ts +++ b/packages/contrail/src/core/spaces/blob-gc.ts @@ -1,38 +1 @@ -import type { BlobAdapter } from "./blob-adapter"; -import { blobKey } from "./blob-adapter"; -import type { StorageAdapter } from "./types"; - -export interface BlobGcOptions { - /** Orphan rows created before this timestamp are eligible for deletion. */ - olderThan: number; - /** Maximum number of blobs to delete in this pass. Defaults to 500. */ - batchSize?: number; -} - -export interface BlobGcResult { - deleted: number; - cids: string[]; -} - -/** Delete blob bytes + metadata for any blob older than `olderThan` that - * is not referenced by any record in the space. Safe to run periodically. */ -export async function gcOrphanBlobs( - storage: StorageAdapter, - blobs: BlobAdapter, - spaceUri: string, - options: BlobGcOptions -): Promise<BlobGcResult> { - const batchSize = options.batchSize ?? 500; - const orphans = await storage.findOrphanBlobs(spaceUri, options.olderThan, batchSize); - if (orphans.length === 0) return { deleted: 0, cids: [] }; - - const keys: string[] = []; - for (const row of orphans) { - keys.push(await blobKey(row.spaceUri, row.cid)); - } - await blobs.delete(keys); - for (const row of orphans) { - await storage.deleteBlobMeta(row.spaceUri, row.cid); - } - return { deleted: orphans.length, cids: orphans.map((o) => o.cid) }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/blob-refs.ts b/packages/contrail/src/core/spaces/blob-refs.ts index 62cf693..551c495 100644 --- a/packages/contrail/src/core/spaces/blob-refs.ts +++ b/packages/contrail/src/core/spaces/blob-refs.ts @@ -1,27 +1 @@ -/** - * Walk a record JSON and collect every atproto blob ref. - * - * Blob refs look like: - * { "$type": "blob", "ref": { "$link": "<cid>" }, "mimeType": "...", "size": N } - * - * We return the CID strings. - */ -export function collectBlobCids(value: unknown, out: Set<string> = new Set()): Set<string> { - if (value == null) return out; - if (Array.isArray(value)) { - for (const v of value) collectBlobCids(v, out); - return out; - } - if (typeof value !== "object") return out; - - const obj = value as Record<string, unknown>; - if (obj["$type"] === "blob") { - const ref = obj["ref"] as { $link?: unknown } | undefined; - if (ref && typeof ref["$link"] === "string") out.add(ref["$link"]); - // Don't descend — a blob ref's own shape has no nested blobs. - return out; - } - - for (const v of Object.values(obj)) collectBlobCids(v, out); - return out; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/credentials.ts b/packages/contrail/src/core/spaces/credentials.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/credentials.ts +++ b/packages/contrail/src/core/spaces/credentials.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/in-process.ts b/packages/contrail/src/core/spaces/in-process.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/in-process.ts +++ b/packages/contrail/src/core/spaces/in-process.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/router.ts b/packages/contrail/src/core/spaces/router.ts index dadb7c9..551c495 100644 --- a/packages/contrail/src/core/spaces/router.ts +++ b/packages/contrail/src/core/spaces/router.ts @@ -1,1027 +1 @@ -import type { Context, Hono, MiddlewareHandler } from "hono"; -import type { ContrailConfig, Database } from "../types"; -import { HostedAdapter } from "./adapter"; -import { checkAccess } from "./acl"; -import type { ServiceAuth } from "./auth"; -import { - buildVerifier, - checkInviteReadGrant, - createServiceAuthMiddleware, - extractInviteToken, - extractSpaceCredential, -} from "./auth"; -import { nextTid } from "./tid"; -import { hashInviteToken } from "../invite/token"; -import { buildSpaceUri, parseSpaceUri } from "./uri"; -import { - DEFAULT_BLOB_MAX_SIZE, - DEFAULT_CREDENTIAL_TTL_MS, - type AuthorityConfig, - type RecordHostConfig, - type RecordHost, - type SpaceAuthority, - type SpaceRow, - type StorageAdapter, -} from "./types"; -import { blobKey } from "./blob-adapter"; -import { collectBlobCids } from "./blob-refs"; -import { - createBindingCredentialVerifier, - decodeUnverifiedClaims, - issueCredential, - verifyCredential, - type CredentialClaims, - type CredentialScope, - type CredentialVerifier, -} from "./credentials"; -import { - createCompositeBindingResolver, - createEnrollmentBindingResolver, - createLocalBindingResolver, - createLocalKeyResolver, -} from "./binding"; -import { create as createCid, toString as cidToString } from "@atcute/cid"; - -/** Optional hook to extend `<ns>.spaceExt.whoami` with extra fields when a - * module above spaces (e.g. community) wants to override the default - * binary-membership response. If the hook returns a non-null object, that - * object is the entire response body. If null, falls through to the - * default behavior (just `isOwner`/`isMember`). - * - * Spaces stays community-agnostic: any consumer can plug in here. */ -export type WhoamiExtension = (input: { - spaceUri: string; - callerDid: string; - isOwner: boolean; - ownerDid: string; -}) => Promise<Record<string, unknown> | null>; - -export interface SpacesRoutesOptions { - /** Provide a custom middleware (e.g. for tests). If omitted and authority is set, a real one is built. */ - authMiddleware?: MiddlewareHandler; - /** Storage adapter override. Defaults to HostedAdapter(db). */ - adapter?: StorageAdapter; - /** Optional whoami extension; see {@link WhoamiExtension}. */ - whoamiExtension?: WhoamiExtension; - /** Optional credential verifier for the record host. When omitted, a - * default in-process binding verifier is built from the authority's - * signing config (Local binding + Local key resolvers). Override to - * accept credentials from external authorities — wire in PDS-record / - * DID-doc binding resolvers and a DID-doc key resolver. */ - credentialVerifier?: CredentialVerifier; -} - -/** Umbrella registration: wires both the authority and the record-host - * routes against the same adapter. Today's deployments enable both via - * `config.spaces.authority` and `config.spaces.recordHost`. Either may be - * omitted in future split deployments — phase 5 lifts the assumption that - * one process runs both. */ -export function registerSpacesRoutes( - app: Hono, - db: Database, - config: ContrailConfig, - options: SpacesRoutesOptions = {}, - ctx?: { adapter: StorageAdapter; verifier: import("@atcute/xrpc-server/auth").ServiceJwtVerifier } | null -): void { - const spacesConfig = config.spaces; - if (!spacesConfig) return; - const authorityConfig = spacesConfig.authority; - if (!authorityConfig) return; - - const adapter = options.adapter ?? ctx?.adapter ?? new HostedAdapter(db, config); - const verifier = ctx?.verifier ?? buildVerifier(authorityConfig); - const auth = options.authMiddleware ?? createServiceAuthMiddleware(verifier); - - // When the record host is colocated, hand it to the authority so - // createSpace can auto-enroll. This is the in-process default — - // single-call createSpace gives you a usable space without requiring an - // explicit `recordHost.enroll` afterward. Split deployments don't get - // this convenience; their createSpace caller (or operator) has to call - // enroll on the remote host explicitly. - const localRecordHost = spacesConfig.recordHost ? adapter : null; - registerAuthorityRoutes( - app, - adapter, - authorityConfig, - config, - auth, - options.whoamiExtension, - localRecordHost - ); - - if (spacesConfig.recordHost) { - // Default in-process verifier: enrollment is the canonical binding - // source (matches the host's local consent), with Local-binding as a - // fallback for the case where the authority's createSpace ran but - // auto-enroll was bypassed. Local key resolver knows the local - // authority's public key. Caller overrides via - // `options.credentialVerifier` to accept external authorities — wire - // in DID-doc key resolvers there. - const verifier = - options.credentialVerifier ?? - (authorityConfig.signing - ? createBindingCredentialVerifier({ - bindings: createCompositeBindingResolver([ - createEnrollmentBindingResolver({ recordHost: adapter }), - createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), - ]), - keys: createLocalKeyResolver({ - authorityDid: authorityConfig.serviceDid, - publicKey: authorityConfig.signing.publicKey, - }), - }) - : undefined); - registerRecordHostRoutes(app, adapter, adapter, spacesConfig.recordHost, config, auth, verifier); - } -} - -/** Register the **space authority** XRPC surface — space lifecycle, member - * list, app policy, whoami. Does NOT touch records or blobs. - * - * When a `localRecordHost` is passed, the authority's `createSpace` handler - * also enrolls the new space on that host — convenient for in-process - * deployments where the same operator runs both roles. Split deployments - * pass `null` and arrange enrollment explicitly via `recordHost.enroll`. */ -export function registerAuthorityRoutes( - app: Hono, - authority: SpaceAuthority, - authorityConfig: AuthorityConfig, - config: ContrailConfig, - auth: MiddlewareHandler, - whoamiExtension?: WhoamiExtension, - localRecordHost?: RecordHost | null -): void { - /** Space endpoints are emitted per-deployment under the configured namespace; - * the deployment owns and publishes its own lexicons. The library ships - * templates at `lexicon-templates/spaces/*` that the generator instantiates - * under `<ns>.space.*` (spec-aligned) and `<ns>.spaceExt.*` (contrail - * extras — invites, whoami — that the permissioned-data spec doesn't cover). */ - const SPACE = `${config.namespace}.space`; - const SPACE_EXT = `${config.namespace}.spaceExt`; - - // ---- Read endpoints ---- - - app.get(`/xrpc/${SPACE}.listSpaces`, auth, async (c) => { - const sa = getAuth(c); - const scope = c.req.query("scope") ?? "member"; // "member" | "owner" - const type = c.req.query("type") ?? undefined; - const owner = c.req.query("owner") ?? undefined; - const cursor = c.req.query("cursor") ?? undefined; - const limit = c.req.query("limit") ? Number(c.req.query("limit")) : undefined; - - const opts: Parameters<typeof authority.listSpaces>[0] = { type, cursor, limit }; - if (scope === "owner") opts.ownerDid = sa.issuer; - else { - opts.memberDid = sa.issuer; - if (owner) opts.ownerDid = owner; // narrow to spaces owned by this DID - } - - const result = await authority.listSpaces(opts); - return c.json({ - spaces: result.spaces.map((s) => publicSpaceView(s, s.ownerDid === sa.issuer)), - cursor: result.cursor, - }); - }); - - app.get(`/xrpc/${SPACE}.listMembers`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - const member = isOwner ? null : await authority.getMember(spaceUri, sa.issuer); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - const members = await authority.listMembers(spaceUri); - return c.json({ members }); - }); - - /** Read-route auth: skip the JWT middleware when an `?inviteToken=` is - * present so anonymous bearer reads don't 401 before the route handler can - * validate the token. */ - const readAuth: MiddlewareHandler = async (c, next) => { - if (extractInviteToken(c.req.raw)) { - await next(); - return; - } - return auth(c, next); - }; - - app.get(`/xrpc/${SPACE}.getSpace`, readAuth, async (c) => { - const uri = c.req.query("uri"); - if (!uri) return c.json({ error: "InvalidRequest", message: "uri required" }, 400); - const space = await authority.getSpace(uri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const authz = await authorizeRead(c, authority, uri); - if (authz instanceof Response) return authz; - - if (authz.via === "token") { - // Anonymous read-token bearer — show non-owner space view. - return c.json({ space: publicSpaceView(space, false) }); - } - - if (authz.via === "credential") { - // Credential proves membership; derive isOwner from sub vs ownerDid. - const isOwner = authz.claims.sub === space.ownerDid; - return c.json({ space: publicSpaceView(space, isOwner) }); - } - - const sa = authz.sa; - const isOwner = sa.issuer === space.ownerDid; - const member = isOwner ? null : await authority.getMember(uri, sa.issuer); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - return c.json({ space: publicSpaceView(space, isOwner) }); - }); - - // ---- Space management (owner-gated) ---- - - app.post(`/xrpc/${SPACE}.createSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => ({}))) as { - type?: string; - key?: string; - appPolicy?: SpaceRow["appPolicy"]; - appPolicyRef?: string; - }; - - const type = body.type ?? authorityConfig.type; - const key = body.key ?? nextTid(); - const uri = buildSpaceUri({ ownerDid: sa.issuer, type, key }); - - const existing = await authority.getSpace(uri); - if (existing) return c.json({ error: "AlreadyExists", uri }, 409); - - const space = await authority.createSpace({ - uri, - ownerDid: sa.issuer, - type, - key, - serviceDid: authorityConfig.serviceDid, - appPolicyRef: body.appPolicyRef ?? null, - appPolicy: body.appPolicy ?? authorityConfig.defaultAppPolicy ?? null, - }); - // Owner is implicit; we still write a row so membership queries are uniform. - await authority.addMember(uri, sa.issuer, sa.issuer); - - // Auto-enroll the space on the colocated record host (if any). Without - // this, putRecord/listRecords would 404 with "not-enrolled" until the - // caller explicitly hit recordHost.enroll. Idempotent (ON CONFLICT - // UPDATE), so calling enroll again later is harmless. - if (localRecordHost) { - await localRecordHost.enroll({ - spaceUri: uri, - authorityDid: authorityConfig.serviceDid, - enrolledAt: Date.now(), - enrolledBy: sa.issuer, - }); - } - - return c.json({ space: publicSpaceView(space, true) }); - }); - - app.post(`/xrpc/${SPACE}.addMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - await authority.addMember(body.spaceUri, body.did, sa.issuer); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.removeMember`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; did?: string } - | null; - if (!body?.spaceUri || !body.did) { - return c.json({ error: "InvalidRequest", message: "spaceUri and did required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid !== sa.issuer) { - return c.json({ error: "Forbidden", reason: "not-owner" }, 403); - } - if (body.did === space.ownerDid) { - return c.json({ error: "InvalidRequest", reason: "cannot-remove-owner" }, 400); - } - await authority.removeMember(body.spaceUri, body.did); - return c.json({ ok: true }); - }); - - app.post(`/xrpc/${SPACE}.leaveSpace`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - if (space.ownerDid === sa.issuer) { - return c.json( - { error: "InvalidRequest", reason: "owner-cannot-leave", message: "Owner cannot leave; delete the space instead" }, - 400 - ); - } - await authority.removeMember(body.spaceUri, sa.issuer); - return c.json({ ok: true }); - }); - - // Unified whoami — `<ns>.spaceExt.whoami?spaceUri=X` → { isOwner, isMember, - // ... }. Extra fields (e.g. accessLevel for community-owned spaces) come - // from the optional whoamiExtension hook; without one, response is binary. - app.get(`/xrpc/${SPACE_EXT}.whoami`, auth, async (c) => { - const sa = getAuth(c); - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - - if (whoamiExtension) { - const ext = await whoamiExtension({ - spaceUri, - callerDid: sa.issuer, - isOwner, - ownerDid: space.ownerDid, - }); - if (ext) return c.json(ext); - } - - // Default: binary membership. - if (isOwner) return c.json({ isOwner: true, isMember: true }); - const member = await authority.getMember(spaceUri, sa.issuer); - return c.json({ isOwner: false, isMember: !!member }); - }); - - // ---- Credential endpoints ---- - - /** Mint a space credential for a member of `spaceUri`. Caller is identified - * by the JWT issuer; the credential's `sub` is set to that DID. */ - app.post(`/xrpc/${SPACE}.getCredential`, auth, async (c) => { - if (!authorityConfig.signing) { - return c.json( - { error: "NotImplemented", message: "authority is not configured to sign credentials" }, - 501 - ); - } - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as { spaceUri?: string } | null; - if (!body?.spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - - const isOwner = space.ownerDid === sa.issuer; - const member = isOwner ? null : await authority.getMember(body.spaceUri, sa.issuer); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - - // App policy is checked at credential-issuance time. Existing credentials - // remain valid until expiry — that's the spec contract (revocation - // bounded by TTL, not synchronous). - if (space.appPolicy) { - const allowed = checkClientId(space.appPolicy, sa.clientId); - if (!allowed) return c.json({ error: "Forbidden", reason: "app-not-allowed" }, 403); - } - - const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; - const { credential, expiresAt } = await issueCredential( - { - iss: authorityConfig.serviceDid, - sub: sa.issuer, - space: body.spaceUri, - scope: "rw", - ttlMs: ttl, - }, - authorityConfig.signing - ); - return c.json({ credential, expiresAt }); - }); - - /** Refresh an unexpired credential. Used by long-running clients to extend - * their access without going back through the JWT mint dance. The current - * credential must verify; the bearer must still be a member. */ - app.post(`/xrpc/${SPACE}.refreshCredential`, async (c) => { - if (!authorityConfig.signing) { - return c.json( - { error: "NotImplemented", message: "authority is not configured to sign credentials" }, - 501 - ); - } - const body = (await c.req.json().catch(() => null)) as { credential?: string } | null; - if (!body?.credential) { - return c.json({ error: "InvalidRequest", message: "credential required" }, 400); - } - const signing = authorityConfig.signing; - const claims = await verifyAndAuthorizeRefresh(body.credential, authorityConfig); - if ("error" in claims) return c.json(claims, claims.status); - - const space = await authority.getSpace(claims.space); - if (!space) return c.json({ error: "NotFound" }, 404); - const isOwner = space.ownerDid === claims.sub; - const member = isOwner ? null : await authority.getMember(claims.space, claims.sub); - if (!isOwner && !member) { - return c.json({ error: "Forbidden", reason: "not-member" }, 403); - } - - const ttl = authorityConfig.credentialTtlMs ?? DEFAULT_CREDENTIAL_TTL_MS; - const { credential, expiresAt } = await issueCredential( - { - iss: authorityConfig.serviceDid, - sub: claims.sub, - space: claims.space, - scope: claims.scope, - ttlMs: ttl, - }, - signing - ); - return c.json({ credential, expiresAt }); - }); -} - -/** Verify a credential presented at refreshCredential. Returns the claims, or - * an error envelope ready to relay. Different from the record-host verifier - * in two ways: (a) we don't have the expectedSpace yet — we read it from the - * credential itself; (b) we don't enforce a scope. */ -async function verifyAndAuthorizeRefresh( - credential: string, - authorityConfig: AuthorityConfig -): Promise<CredentialClaims | { error: string; reason?: string; message?: string; status: 400 | 401 }> { - const peek = decodeUnverifiedClaims(credential); - if (!peek) return { error: "InvalidRequest", reason: "malformed", status: 400 }; - if (peek.iss !== authorityConfig.serviceDid) { - return { error: "Forbidden", reason: "wrong-issuer", status: 401 }; - } - if (!authorityConfig.signing) { - return { error: "InvalidState", status: 401 }; - } - const signing = authorityConfig.signing; - const result = await verifyCredential(credential, { - expectedSpace: peek.space, - resolveKey: async (iss) => (iss === authorityConfig.serviceDid ? signing.publicKey : null), - }); - if (!result.ok) { - return { error: "InvalidCredential", reason: result.reason, status: 401 }; - } - return result.claims; -} - -/** App-policy check using just `clientId`. Mirrors `acl.ts:checkAppPolicy` - * but inlined here so the credential-issuance path doesn't need to construct - * a full AclInput. */ -function checkClientId( - appPolicy: NonNullable<SpaceRow["appPolicy"]>, - clientId: string | undefined -): boolean { - const listed = clientId ? appPolicy.apps.includes(clientId) : false; - if (appPolicy.mode === "allow") return !listed; - return listed; -} - -/** Register the **record host** XRPC surface — record + blob CRUD. - * - * Auth precedence on every route: - * 1. `X-Space-Credential` header (if a verifier is wired and the credential - * is valid) — caller DID = credential `sub`, no clientId. - * 2. Read-route invite token (`?inviteToken=` or `Bearer atmo-invite:...`). - * 3. Service-auth JWT (existing behavior) — caller DID = JWT issuer. - * - * When a credential is presented, the record host trusts it: no member - * check, no app-policy check (those happen at issuance time on the - * authority side). Service-auth requests still consult the authority — that - * bridge is what phase 5 cuts when the host/authority split goes runtime. */ -export function registerRecordHostRoutes( - app: Hono, - recordHost: RecordHost, - authority: SpaceAuthority, - recordHostConfig: RecordHostConfig, - config: ContrailConfig, - auth: MiddlewareHandler, - /** Optional credential verifier. When present, the record host accepts - * `X-Space-Credential` as an alternative to a service-auth JWT. */ - credentialVerifier?: CredentialVerifier -): void { - const SPACE = `${config.namespace}.space`; - - /** Auth wrapper: tries credential first, then delegates to JWT auth. */ - const authWithCredential: MiddlewareHandler = async (c, next) => { - const credToken = extractSpaceCredential(c.req.raw); - if (credToken) { - if (!credentialVerifier) { - return c.json( - { error: "AuthRequired", reason: "credential-verifier-not-configured" }, - 401 - ); - } - const result = await credentialVerifier.verify(credToken); - if (!result.ok) { - return c.json({ error: "AuthRequired", reason: result.reason }, 401); - } - c.set("spaceCredential", result.claims); - await next(); - return; - } - return auth(c, next); - }; - - /** Read-route auth: like {@link authWithCredential} but also short-circuits - * on a read-grant invite token. Token presence skips both credential and - * JWT middlewares; the route handler validates the token via authorizeRead. */ - const readAuth: MiddlewareHandler = async (c, next) => { - if (extractInviteToken(c.req.raw)) { - await next(); - return; - } - return authWithCredential(c, next); - }; - - /** Hard gate on every record-host operation: the space must be enrolled. - * Returns the enrollment row, or a Response to relay. */ - async function requireEnrollment( - c: Context, - spaceUri: string - ): Promise<{ authorityDid: string } | Response> { - const enrollment = await recordHost.getEnrollment(spaceUri); - if (!enrollment) { - return c.json( - { error: "NotFound", reason: "not-enrolled", message: "space is not enrolled on this record host" }, - 404 - ); - } - return enrollment; - } - - // ---- Enrollment endpoint ---- - // - // Authorize a space onto this record host. Caller must be either the space - // owner OR the declared authority — the host treats either as a sufficient - // signal of consent. Idempotent: re-enrolling the same space updates the - // authority binding. - const RECORD_HOST = `${config.namespace}.recordHost`; - - app.post(`/xrpc/${RECORD_HOST}.enroll`, auth, async (c) => { - const sa = getAuth(c); - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; authority?: string } - | null; - if (!body?.spaceUri || !body.authority) { - return c.json({ error: "InvalidRequest", message: "spaceUri and authority required" }, 400); - } - const parts = parseSpaceUri(body.spaceUri); - if (!parts) { - return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); - } - const callerIsOwner = sa.issuer === parts.ownerDid; - const callerIsAuthority = sa.issuer === body.authority; - if (!callerIsOwner && !callerIsAuthority) { - return c.json( - { error: "Forbidden", reason: "not-owner-or-authority" }, - 403 - ); - } - await recordHost.enroll({ - spaceUri: body.spaceUri, - authorityDid: body.authority, - enrolledAt: Date.now(), - enrolledBy: sa.issuer, - }); - return c.json({ ok: true }); - }); - - app.get(`/xrpc/${SPACE}.listRecords`, readAuth, async (c) => { - const spaceUri = c.req.query("spaceUri"); - const collection = c.req.query("collection"); - if (!spaceUri || !collection) { - return c.json({ error: "InvalidRequest", message: "spaceUri and collection required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const authz = await authorizeRead(c, authority, spaceUri); - if (authz instanceof Response) return authz; - - if (authz.via === "jwt") { - // JWT path consults the authority for member + app-policy checks. - // Split deployments where the host has no live authority will degrade - // here (getSpace returns null) and return 404, which is correct: a - // host that can't reach an authority can't validate JWT membership. - const sa = authz.sa; - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const result = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!result.allow) { - return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - } - // Credential and token paths are pre-authorized — credential's signature - // proves the authority granted access; token validation already happened. - - const list = await recordHost.listRecords(spaceUri, collection, { - byUser: c.req.query("byUser") ?? undefined, - cursor: c.req.query("cursor") ?? undefined, - limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, - }); - return c.json(list); - }); - - app.get(`/xrpc/${SPACE}.getRecord`, readAuth, async (c) => { - const spaceUri = c.req.query("spaceUri"); - const collection = c.req.query("collection"); - const author = c.req.query("author"); - const rkey = c.req.query("rkey"); - if (!spaceUri || !collection || !author || !rkey) { - return c.json({ error: "InvalidRequest", message: "spaceUri, collection, author, rkey required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const authz = await authorizeRead(c, authority, spaceUri); - if (authz instanceof Response) return authz; - - if (authz.via === "jwt") { - const sa = authz.sa; - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const result = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - targetAuthorDid: author, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - - const record = await recordHost.getRecord(spaceUri, collection, author, rkey); - if (!record) return c.json({ error: "NotFound" }, 404); - return c.json({ record }); - }); - - // Write endpoints - app.post(`/xrpc/${SPACE}.putRecord`, authWithCredential, async (c) => { - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; collection?: string; rkey?: string; record?: Record<string, unknown> } - | null; - if (!body?.spaceUri || !body.collection || !body.record) { - return c.json({ error: "InvalidRequest", message: "spaceUri, collection, record required" }, 400); - } - const enrollment = await requireEnrollment(c, body.spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, body.spaceUri, "rw"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - // JWT path needs authority access for member + app-policy checks. - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, caller.callerDid); - const result = checkAccess({ - op: "write", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - - // Validate that every blob referenced by this record has already been - // uploaded into this space. This mirrors how PDSes require uploadBlob - // before putRecord, and prevents forging refs to blobs the caller never - // actually claimed. - if (recordHostConfig.blobs) { - const cids = collectBlobCids(body.record); - for (const cid of cids) { - const meta = await recordHost.getBlobMeta(body.spaceUri, cid); - if (!meta) { - return c.json( - { - error: "InvalidRequest", - reason: "unknown-blob-ref", - message: `Record references blob ${cid} that has not been uploaded to this space.`, - }, - 400 - ); - } - } - } - - const rkey = body.rkey ?? nextTid(); - const now = Date.now(); - await recordHost.putRecord({ - spaceUri: body.spaceUri, - collection: body.collection, - authorDid: caller.callerDid, - rkey, - cid: null, - record: body.record, - createdAt: now, - }); - return c.json({ rkey, authorDid: caller.callerDid, createdAt: now }); - }); - - app.post(`/xrpc/${SPACE}.deleteRecord`, authWithCredential, async (c) => { - const body = (await c.req.json().catch(() => null)) as - | { spaceUri?: string; collection?: string; rkey?: string } - | null; - if (!body?.spaceUri || !body.collection || !body.rkey) { - return c.json({ error: "InvalidRequest", message: "spaceUri, collection, rkey required" }, 400); - } - const enrollment = await requireEnrollment(c, body.spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, body.spaceUri, "rw"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - const space = await authority.getSpace(body.spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(body.spaceUri, caller.callerDid); - const result = checkAccess({ - op: "delete", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - targetAuthorDid: caller.callerDid, - }); - if (!result.allow) return c.json({ error: "Forbidden", reason: result.reason }, 403); - } - // Credential path: scope=rw is checked in resolveCaller. Delete remains - // author-scoped — the credential's `sub` is the caller, and we only - // delete records authored by that DID. - - await recordHost.deleteRecord(body.spaceUri, body.collection, caller.callerDid, body.rkey); - return c.json({ ok: true }); - }); - - // Blobs (only registered when a blob adapter is configured) - if (recordHostConfig.blobs) { - const blobsCfg = recordHostConfig.blobs; - const blobAdapter = blobsCfg.adapter; - const maxSize = blobsCfg.maxSize ?? DEFAULT_BLOB_MAX_SIZE; - const accept = blobsCfg.accept; - - app.post(`/xrpc/${SPACE}.uploadBlob`, authWithCredential, async (c) => { - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, spaceUri, "rw"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, caller.callerDid); - const aclResult = checkAccess({ - op: "write", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); - } - } - - const mimeType = c.req.header("content-type") ?? "application/octet-stream"; - if (accept && !accept.includes(mimeType)) { - return c.json( - { error: "InvalidMimeType", message: `MIME type ${mimeType} is not accepted.` }, - 400 - ); - } - - const declaredLen = c.req.header("content-length"); - if (declaredLen && Number(declaredLen) > maxSize) { - return c.json( - { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, - 413 - ); - } - - const buf = await c.req.arrayBuffer(); - const bytes = new Uint8Array(buf); - if (bytes.byteLength > maxSize) { - return c.json( - { error: "BlobTooLarge", message: `Blob exceeds max size of ${maxSize} bytes.` }, - 413 - ); - } - - const cid = await createCid(0x55, bytes); - const cidString = cidToString(cid); - const key = await blobKey(spaceUri, cidString); - - await blobAdapter.put(key, bytes, { mimeType, size: bytes.byteLength }); - await recordHost.putBlobMeta({ - spaceUri, - cid: cidString, - mimeType, - size: bytes.byteLength, - authorDid: caller.callerDid, - createdAt: Date.now(), - }); - - return c.json({ - blob: { - $type: "blob", - ref: { $link: cidString }, - mimeType, - size: bytes.byteLength, - }, - }); - }); - - app.get(`/xrpc/${SPACE}.getBlob`, readAuth, async (c) => { - const spaceUri = c.req.query("spaceUri"); - const cid = c.req.query("cid"); - if (!spaceUri || !cid) { - return c.json({ error: "InvalidRequest", message: "spaceUri and cid required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const authz = await authorizeRead(c, authority, spaceUri); - if (authz instanceof Response) return authz; - - if (authz.via === "jwt") { - const sa = authz.sa; - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, sa.issuer); - const aclResult = checkAccess({ - op: "read", - space, - callerDid: sa.issuer, - member, - clientId: sa.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); - } - } - - const meta = await recordHost.getBlobMeta(spaceUri, cid); - if (!meta) return c.json({ error: "NotFound" }, 404); - const key = await blobKey(spaceUri, cid); - const bytes = await blobAdapter.get(key); - if (!bytes) return c.json({ error: "NotFound" }, 404); - - return new Response(bytes, { - headers: { - "content-type": meta.mimeType, - "content-length": String(meta.size), - }, - }); - }); - - app.get(`/xrpc/${SPACE}.listBlobs`, authWithCredential, async (c) => { - const spaceUri = c.req.query("spaceUri"); - if (!spaceUri) { - return c.json({ error: "InvalidRequest", message: "spaceUri required" }, 400); - } - const enrollment = await requireEnrollment(c, spaceUri); - if (enrollment instanceof Response) return enrollment; - - const caller = resolveCaller(c, spaceUri, "read"); - if (caller instanceof Response) return caller; - - if (!caller.viaCredential) { - const space = await authority.getSpace(spaceUri); - if (!space) return c.json({ error: "NotFound" }, 404); - const member = await authority.getMember(spaceUri, caller.callerDid); - const aclResult = checkAccess({ - op: "read", - space, - callerDid: caller.callerDid, - member, - clientId: caller.clientId, - }); - if (!aclResult.allow) { - return c.json({ error: "Forbidden", reason: aclResult.reason }, 403); - } - } - - const result = await recordHost.listBlobMeta(spaceUri, { - byUser: c.req.query("byUser") ?? undefined, - cursor: c.req.query("cursor") ?? undefined, - limit: c.req.query("limit") ? Number(c.req.query("limit")) : undefined, - }); - return c.json(result); - }); - } -} - -/** Authorize a read request — three valid paths: a verified space credential - * (set by the credential middleware), a read-grant invite token, or a - * service-auth JWT. - * - * Credential and token paths skip the membership check downstream — the - * credential or token IS the proof. The JWT path requires a member check - * in the route handler. */ -async function authorizeRead( - c: Context, - authority: SpaceAuthority, - spaceUri: string -): Promise< - | { via: "credential"; claims: CredentialClaims } - | { via: "token" } - | { via: "jwt"; sa: ServiceAuth } - | Response -> { - const cred = c.get("spaceCredential") as CredentialClaims | undefined; - if (cred) { - if (cred.space !== spaceUri) { - return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); - } - return { via: "credential", claims: cred }; - } - const rawToken = extractInviteToken(c.req.raw); - if (rawToken) { - const ok = await checkInviteReadGrant(authority, rawToken, spaceUri, hashInviteToken); - if (!ok) return c.json({ error: "Forbidden", reason: "invalid-invite-token" }, 403); - return { via: "token" }; - } - const sa = c.get("serviceAuth") as ServiceAuth | undefined; - if (sa) return { via: "jwt", sa }; - return c.json( - { error: "AuthRequired", message: "JWT, credential, or read-grant invite token required" }, - 401 - ); -} - -/** Unified caller resolution for write/manage paths on the record host. - * Either a verified credential (set by middleware) or a service-auth JWT. - * When a credential is present, also enforces space-match and the requested - * scope. Returns either a caller envelope or a Response to relay. */ -function resolveCaller( - c: Context, - requestSpace: string, - requiredScope: CredentialScope -): { callerDid: string; clientId: string | undefined; viaCredential: boolean } | Response { - const cred = c.get("spaceCredential") as CredentialClaims | undefined; - if (cred) { - if (cred.space !== requestSpace) { - return c.json({ error: "Forbidden", reason: "credential-wrong-space" }, 403); - } - if (requiredScope === "rw" && cred.scope !== "rw") { - return c.json({ error: "Forbidden", reason: "credential-wrong-scope" }, 403); - } - return { callerDid: cred.sub, clientId: undefined, viaCredential: true }; - } - const sa = c.get("serviceAuth") as ServiceAuth | undefined; - if (!sa) return c.json({ error: "AuthRequired", reason: "no-auth" }, 401); - return { callerDid: sa.issuer, clientId: sa.clientId, viaCredential: false }; -} - -function getAuth(c: Parameters<MiddlewareHandler>[0]): ServiceAuth { - const auth = c.get("serviceAuth") as ServiceAuth | undefined; - if (!auth) throw new Error("service auth not set"); - return auth; -} - -function publicSpaceView(space: SpaceRow, forOwner: boolean) { - return { - uri: space.uri, - ownerDid: space.ownerDid, - type: space.type, - key: space.key, - serviceDid: space.serviceDid, - appPolicyRef: space.appPolicyRef, - createdAt: space.createdAt, - ...(forOwner ? { appPolicy: space.appPolicy } : {}), - }; -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/schema.ts b/packages/contrail/src/core/spaces/schema.ts index ae33a7c..551c495 100644 --- a/packages/contrail/src/core/spaces/schema.ts +++ b/packages/contrail/src/core/spaces/schema.ts @@ -1,103 +1 @@ -import type { ContrailConfig, Database } from "../types"; -import type { SqlDialect } from "../dialect"; -import { getDialect } from "../dialect"; -import { - buildCollectionTables, - buildDynamicIndexes, - buildFtsTables, - buildCountColumns, -} from "../db/schema"; - -/** Spaces metadata tables — spaces, members, invites. No per-collection tables. */ -export function buildSpacesBaseSchema(dialect: SqlDialect): string[] { - return [ - `CREATE TABLE IF NOT EXISTS spaces ( - uri TEXT PRIMARY KEY, - owner_did TEXT NOT NULL, - type TEXT NOT NULL, - key TEXT NOT NULL, - service_did TEXT NOT NULL, - app_policy_ref TEXT, - app_policy ${dialect.recordColumnType}, - created_at ${dialect.bigintType} NOT NULL, - deleted_at ${dialect.bigintType} - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_owner ON spaces(owner_did)`, - `CREATE INDEX IF NOT EXISTS idx_spaces_type ON spaces(type)`, - - `CREATE TABLE IF NOT EXISTS spaces_members ( - space_uri TEXT NOT NULL, - did TEXT NOT NULL, - added_at ${dialect.bigintType} NOT NULL, - added_by TEXT, - PRIMARY KEY (space_uri, did) - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_members_did ON spaces_members(did)`, - - `CREATE TABLE IF NOT EXISTS spaces_blobs ( - space_uri TEXT NOT NULL, - cid TEXT NOT NULL, - mime_type TEXT NOT NULL, - size INTEGER NOT NULL, - author_did TEXT NOT NULL, - created_at ${dialect.bigintType} NOT NULL, - PRIMARY KEY (space_uri, cid) - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_author ON spaces_blobs(space_uri, author_did)`, - `CREATE INDEX IF NOT EXISTS idx_spaces_blobs_created ON spaces_blobs(space_uri, created_at)`, - - `CREATE TABLE IF NOT EXISTS spaces_invites ( - token_hash TEXT PRIMARY KEY, - space_uri TEXT NOT NULL, - kind TEXT NOT NULL DEFAULT 'join', - expires_at ${dialect.bigintType}, - max_uses INTEGER, - used_count INTEGER NOT NULL DEFAULT 0, - created_by TEXT NOT NULL, - created_at ${dialect.bigintType} NOT NULL, - revoked_at ${dialect.bigintType}, - note TEXT - )`, - `CREATE INDEX IF NOT EXISTS idx_spaces_invites_space ON spaces_invites(space_uri, created_at DESC)`, - - // Record-host enrollment table — the host's local cache of "spaces I - // accept records for, and which authority signs credentials for each." - // Filled by the recordHost.enroll endpoint, or auto-populated by the - // authority's createSpace when both roles run in the same process. - `CREATE TABLE IF NOT EXISTS record_host_enrollments ( - space_uri TEXT PRIMARY KEY, - authority_did TEXT NOT NULL, - enrolled_at ${dialect.bigintType} NOT NULL, - enrolled_by TEXT NOT NULL - )`, - `CREATE INDEX IF NOT EXISTS idx_enrollments_authority ON record_host_enrollments(authority_did)`, - ]; -} - -/** Full spaces schema (base + per-collection tables + indexes). For callers - * that need a single array of statements. Note: this does NOT include FTS - * virtual tables or ALTER TABLE count columns — those must be applied with - * try/catch fallbacks and are handled by `initSchema`. */ -export function buildSpacesSchema(db: Database, config?: ContrailConfig): string[] { - const dialect = getDialect(db); - const base = buildSpacesBaseSchema(dialect); - if (!config) return base; - return [ - ...base, - ...buildCollectionTables(config, dialect, { forSpaces: true }), - ...buildDynamicIndexes(config, dialect, { forSpaces: true }), - ]; -} - -export async function initSpacesSchema(db: Database, config?: ContrailConfig): Promise<void> { - const dialect = getDialect(db); - const stmts = buildSpacesSchema(db, config); - await db.batch(stmts.map((s) => db.prepare(s))); - if (!config) return; - for (const stmt of buildFtsTables(config, dialect, { forSpaces: true })) { - try { await db.prepare(stmt).run(); } catch { /* ignore */ } - } - for (const stmt of buildCountColumns(config, { forSpaces: true })) { - try { await db.prepare(stmt).run(); } catch { /* ignore */ } - } -} +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/tid.ts b/packages/contrail/src/core/spaces/tid.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/tid.ts +++ b/packages/contrail/src/core/spaces/tid.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/types.ts b/packages/contrail/src/core/spaces/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/types.ts +++ b/packages/contrail/src/core/spaces/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/spaces/uri.ts b/packages/contrail/src/core/spaces/uri.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/spaces/uri.ts +++ b/packages/contrail/src/core/spaces/uri.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/src/core/types.ts b/packages/contrail/src/core/types.ts index 1129419..551c495 100644 --- a/packages/contrail/src/core/types.ts +++ b/packages/contrail/src/core/types.ts @@ -1 +1 @@ -export * from "@atmo-dev/contrail-base"; +export * from "@atmo-dev/contrail-appview"; diff --git a/packages/contrail/tests/persistent.test.ts b/packages/contrail/tests/persistent.test.ts index f248cf0..a1811c0 100644 --- a/packages/contrail/tests/persistent.test.ts +++ b/packages/contrail/tests/persistent.test.ts @@ -5,10 +5,14 @@ import { runPersistent } from "../src/core/persistent"; import { getLastCursor, queryRecords } from "../src/core/db/records"; import { initSchema } from "../src/core/db/schema"; -// Mock identity resolution to avoid network calls in tests -vi.mock("../src/core/identity", () => ({ - refreshStaleIdentities: vi.fn().mockResolvedValue(undefined), -})); +// Identity helpers live in @atmo-dev/contrail-base post-split. Mock there. +vi.mock("@atmo-dev/contrail-base", async (importOriginal) => { + const actual = await importOriginal<typeof import("@atmo-dev/contrail-base")>(); + return { + ...actual, + refreshStaleIdentities: vi.fn().mockResolvedValue(undefined), + }; +}); let db: Database; diff --git a/packages/contrail/tests/refresh.test.ts b/packages/contrail/tests/refresh.test.ts index 2f9179c..67f8e6f 100644 --- a/packages/contrail/tests/refresh.test.ts +++ b/packages/contrail/tests/refresh.test.ts @@ -8,21 +8,27 @@ import type { Database } from "../src/core/types"; // the `pages` map below. const pages = new Map<string, Array<{ uri: string; cid: string; value: object }>>(); -vi.mock("../src/core/client", () => ({ - getClient: vi.fn(async (did: string) => ({ - get: async ( - _method: string, - opts: { params: { repo: string; collection: string; cursor?: string } } - ) => { - const key = `${opts.params.repo}|${opts.params.collection}`; - // Single page per (did, collection); cursor triggers empty page = done. - if (opts.params.cursor) return { ok: true, data: { records: [], cursor: undefined } }; - const records = pages.get(key) ?? []; - return { ok: true, data: { records, cursor: undefined } }; - }, - })), - getPDS: vi.fn(), -})); +// `getClient` lives in @atmo-dev/contrail-base after the package split. +// `refresh` (now in contrail-appview) imports it via its own shim that +// ultimately resolves to base — so we mock the base export directly. +vi.mock("@atmo-dev/contrail-base", async (importOriginal) => { + const actual = await importOriginal<typeof import("@atmo-dev/contrail-base")>(); + return { + ...actual, + getClient: vi.fn(async (_did: string) => ({ + get: async ( + _method: string, + opts: { params: { repo: string; collection: string; cursor?: string } } + ) => { + const key = `${opts.params.repo}|${opts.params.collection}`; + if (opts.params.cursor) return { ok: true, data: { records: [], cursor: undefined } }; + const records = pages.get(key) ?? []; + return { ok: true, data: { records, cursor: undefined } }; + }, + })), + getPDS: vi.fn(), + }; +}); const ALICE = "did:plc:alice"; const BOB = "did:plc:bob"; diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 9c32af0..996e91f 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -3,6 +3,7 @@ import path from "node:path"; const baseSrc = path.resolve(__dirname, "../contrail-base/src"); const authoritySrc = path.resolve(__dirname, "../contrail-authority/src"); +const recordHostSrc = path.resolve(__dirname, "../contrail-record-host/src"); export default defineConfig({ test: { @@ -18,6 +19,7 @@ export default defineConfig({ "@atmo-dev/contrail-base/postgres": path.join(baseSrc, "adapters/postgres.ts"), "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), "@atmo-dev/contrail-authority": path.join(authoritySrc, "index.ts"), + "@atmo-dev/contrail-record-host": path.join(recordHostSrc, "index.ts"), }, }, }); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6e26e5a..2d68ee6 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -376,12 +376,18 @@ importers: '@atcute/xrpc-server': specifier: ^0.1.12 version: 0.1.12 + '@atmo-dev/contrail-appview': + specifier: workspace:* + version: link:../contrail-appview '@atmo-dev/contrail-authority': specifier: workspace:* version: link:../contrail-authority '@atmo-dev/contrail-base': specifier: workspace:* version: link:../contrail-base + '@atmo-dev/contrail-record-host': + specifier: workspace:* + version: link:../contrail-record-host cac: specifier: ^7.0.0 version: 7.0.0 @@ -417,6 +423,58 @@ importers: specifier: ^4.63.0 version: 4.84.1(@cloudflare/workers-types@4.20260424.1) + packages/contrail-appview: + dependencies: + '@atcute/atproto': + specifier: ^3.1.10 + version: 3.1.11 + '@atcute/cbor': + specifier: ^2.3.2 + version: 2.3.2 + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atcute/client': + specifier: ^4.2.1 + version: 4.2.1 + '@atcute/identity': + specifier: ^1.1.4 + version: 1.1.4 + '@atcute/identity-resolver': + specifier: ^1.2.2 + version: 1.2.2(@atcute/identity@1.1.4) + '@atcute/jetstream': + specifier: ^1.0.2 + version: 1.1.2 + '@atcute/lexicons': + specifier: ^1.2.9 + version: 1.3.0 + '@atcute/xrpc-server': + specifier: ^0.1.12 + version: 0.1.12 + '@atmo-dev/contrail-authority': + specifier: workspace:* + version: link:../contrail-authority + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base + '@atmo-dev/contrail-record-host': + specifier: workspace:* + version: link:../contrail-record-host + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + '@types/node': + specifier: ^25.5.0 + version: 25.6.0 + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/contrail-authority: dependencies: '@atcute/cid': @@ -522,6 +580,25 @@ importers: specifier: ^4.1.0 version: 4.1.5(@types/node@25.6.0)(vite@8.0.10(@types/node@25.6.0)(esbuild@0.27.7)(jiti@2.6.1)(tsx@4.21.0)) + packages/contrail-record-host: + dependencies: + '@atcute/cid': + specifier: ^2.4.1 + version: 2.4.1 + '@atmo-dev/contrail-base': + specifier: workspace:* + version: link:../contrail-base + hono: + specifier: ^4.12.8 + version: 4.12.15 + devDependencies: + tsup: + specifier: ^8.5.0 + version: 8.5.1(jiti@2.6.1)(postcss@8.5.10)(tsx@4.21.0)(typescript@5.9.3) + typescript: + specifier: ^5.7.3 + version: 5.9.3 + packages/lexicons: dependencies: '@atcute/lex-cli': -- 2.51.2 From e7149dcd5ba61c2e8d44af77fe32c181b7e140a3 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 02:38:16 +0200 Subject: [PATCH 11/19] phase 7b --- packages/contrail-appview/src/index.ts | 11 + packages/contrail-appview/src/sync.ts | 229 ++++++++++++ packages/contrail-record-host/src/index.ts | 3 + packages/contrail-record-host/src/sync.ts | 331 ++++++++++++++++++ packages/contrail/tests/sync-e2e.test.ts | 304 ++++++++++++++++ packages/contrail/tests/sync-host.test.ts | 282 +++++++++++++++ .../lexicon-templates/recordHost/sync.json | 86 +++++ 7 files changed, 1246 insertions(+) create mode 100644 packages/contrail-appview/src/sync.ts create mode 100644 packages/contrail-record-host/src/sync.ts create mode 100644 packages/contrail/tests/sync-e2e.test.ts create mode 100644 packages/contrail/tests/sync-host.test.ts create mode 100644 packages/lexicons/lexicon-templates/recordHost/sync.json diff --git a/packages/contrail-appview/src/index.ts b/packages/contrail-appview/src/index.ts index 1aad674..dce992a 100644 --- a/packages/contrail-appview/src/index.ts +++ b/packages/contrail-appview/src/index.ts @@ -16,6 +16,17 @@ export * from "@atmo-dev/contrail-base"; export * from "@atmo-dev/contrail-authority"; export * from "@atmo-dev/contrail-record-host"; +// Record-sync ingestion (consumer side of recordHost.sync) +export { + runRecordHostSync, + buildRecordSyncSchema, + applyRecordSyncSchema, +} from "./sync"; +export type { + RecordHostSyncSource, + RecordHostSyncOptions, +} from "./sync"; + // Indexing pipeline (jetstream, persistent, backfill, refresh, ingest helpers) export * from "./core/jetstream"; export * from "./core/persistent"; diff --git a/packages/contrail-appview/src/sync.ts b/packages/contrail-appview/src/sync.ts new file mode 100644 index 0000000..e2526fc --- /dev/null +++ b/packages/contrail-appview/src/sync.ts @@ -0,0 +1,229 @@ +/** Appview-side ingestion loop for the recordHost.sync streaming endpoint. + * + * Opens an SSE connection to a remote host's `<ns>.recordHost.sync` endpoint + * for a (host, space) pair, parses the event stream, writes each + * `record.created` / `record.deleted` event into the local record host's + * tables, and persists the cursor after every checkpoint. + * + * Designed to be called per-subscription. Reconnect logic is the caller's — + * this function returns when the stream ends or an error throws. Wrap it in + * a retry-with-backoff loop in your worker / persistent-process. */ + +import type { + ContrailConfig, + Database, + RecordHost, + SqlDialect, +} from "@atmo-dev/contrail-base"; +import { getDialect } from "@atmo-dev/contrail-base"; + +export interface RecordHostSyncSource { + /** Remote host's base URL, e.g. "https://contrail-a.example.com". */ + hostUrl: string; + /** Space we want to sync. */ + spaceUri: string; + /** Authority DID — used for auto-enrolling locally on first connect. */ + authorityDid: string; + /** Credential the appview presents to read this space's stream. */ + credential: string; + /** Sync endpoint NSID; defaults to "<config.namespace>.recordHost.sync". */ + endpointNsid?: string; +} + +export interface RecordHostSyncOptions { + /** Local DB the records go into (same DB the appview's RecordHost adapter uses). */ + db: Database; + /** Resolved config. Used to derive the remote endpoint NSID and to find + * collection short names for table writes. */ + config: ContrailConfig; + /** Local record host — the destination for ingested events. The function + * calls `putRecord` / `deleteRecord` / `enroll` on this. */ + recordHost: RecordHost; + /** fetch implementation — pass the remote host app's fetch directly for + * in-process tests; defaults to globalThis.fetch. */ + fetch?: typeof fetch; + /** Aborts the stream when triggered. */ + signal?: AbortSignal; + /** Called on each cursor checkpoint, after persistence. */ + onCursor?: (cursor: string) => void; +} + +/** Run sync for a single (host, space) source until the stream ends or the + * signal aborts. Reads the prior cursor from `record_sync_subscriptions` if + * present; persists the new cursor as it advances. Auto-enrolls the space + * locally on first connect using the source's `authorityDid`. */ +export async function runRecordHostSync( + source: RecordHostSyncSource, + options: RecordHostSyncOptions +): Promise<void> { + const fetchImpl = options.fetch ?? globalThis.fetch; + await ensureSyncSchema(options.db); + + // Auto-enroll the space so the local record host accepts subsequent + // queries against it. + const existing = await options.recordHost.getEnrollment(source.spaceUri); + if (!existing) { + await options.recordHost.enroll({ + spaceUri: source.spaceUri, + authorityDid: source.authorityDid, + enrolledAt: Date.now(), + enrolledBy: source.authorityDid, + }); + } + + // Resume from the last persisted cursor for this subscription. + const since = await readCursor(options.db, source.hostUrl, source.spaceUri); + + const endpoint = + source.endpointNsid ?? `${options.config.namespace}.recordHost.sync`; + const url = new URL(`${source.hostUrl}/xrpc/${endpoint}`); + url.searchParams.set("spaceUri", source.spaceUri); + if (since) url.searchParams.set("since", since); + + const res = await fetchImpl(url.toString(), { + headers: { + "X-Space-Credential": source.credential, + accept: "text/event-stream", + }, + signal: options.signal, + }); + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error(`recordHost.sync ${res.status}: ${body}`); + } + if (!res.body) { + throw new Error("recordHost.sync response has no body"); + } + + const reader = res.body.getReader(); + const decoder = new TextDecoder(); + let buf = ""; + try { + while (true) { + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + while (true) { + const idx = buf.indexOf("\n\n"); + if (idx < 0) break; + const block = buf.slice(0, idx); + buf = buf.slice(idx + 2); + const dataLine = block.split("\n").find((l) => l.startsWith("data:")); + if (!dataLine) continue; + const json = dataLine.slice(5).trim(); + if (!json) continue; + let event: any; + try { + event = JSON.parse(json); + } catch { + continue; + } + await applyEvent(event, options); + if (event.kind === "cursor" && typeof event.value === "string") { + await persistCursor( + options.db, + source.hostUrl, + source.spaceUri, + event.value + ); + options.onCursor?.(event.value); + } + } + } + } finally { + await reader.cancel().catch(() => {}); + } +} + +async function applyEvent( + event: any, + options: RecordHostSyncOptions +): Promise<void> { + if (event.kind === "record.created") { + const p = event.payload; + await options.recordHost.putRecord({ + spaceUri: p.space, + collection: p.collection, + authorDid: p.did, + rkey: p.rkey, + cid: p.cid ?? null, + record: p.record ?? {}, + // The host's time_us is microseconds; createdAt on putRecord is ms-ish + // historically. Keep the host's ordering by passing it through; the + // host adapter writes time_us = createdAt * 1000 internally so this + // round-trips. We store the source's time_us directly to preserve + // ordering across hosts. + createdAt: p.time_us != null ? Math.floor(p.time_us / 1000) : Date.now(), + }); + } else if (event.kind === "record.deleted") { + const p = event.payload; + await options.recordHost.deleteRecord(p.space, p.collection, p.did, p.rkey); + } + // cursor events are handled by the caller for persistence +} + +// ---- Schema + cursor persistence ---- + +const SYNC_SCHEMA_APPLIED = new WeakSet<object>(); + +/** Idempotent: applies the `record_sync_subscriptions` table on first call + * for a given DB. Tracks per-DB (by reference) so repeated calls in tests + * don't re-issue DDL each time. */ +async function ensureSyncSchema(db: Database): Promise<void> { + if (SYNC_SCHEMA_APPLIED.has(db as unknown as object)) return; + const dialect = getDialect(db); + const stmts = buildRecordSyncSchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); + SYNC_SCHEMA_APPLIED.add(db as unknown as object); +} + +export function buildRecordSyncSchema(dialect: SqlDialect): string[] { + return [ + `CREATE TABLE IF NOT EXISTS record_sync_subscriptions ( + host_url TEXT NOT NULL, + space_uri TEXT NOT NULL, + cursor TEXT, + last_synced_at ${dialect.bigintType}, + PRIMARY KEY (host_url, space_uri) + )`, + ]; +} + +/** SchemaModule-shaped helper for `initSchema({ extraSchemas: [...] })`. */ +export async function applyRecordSyncSchema(db: Database): Promise<void> { + const dialect = getDialect(db); + const stmts = buildRecordSyncSchema(dialect); + await db.batch(stmts.map((s) => db.prepare(s))); +} + +async function readCursor( + db: Database, + hostUrl: string, + spaceUri: string +): Promise<string | null> { + const row = await db + .prepare( + `SELECT cursor FROM record_sync_subscriptions WHERE host_url = ? AND space_uri = ?` + ) + .bind(hostUrl, spaceUri) + .first<{ cursor: string | null } | null>(); + return row?.cursor ?? null; +} + +async function persistCursor( + db: Database, + hostUrl: string, + spaceUri: string, + cursor: string +): Promise<void> { + await db + .prepare( + `INSERT INTO record_sync_subscriptions (host_url, space_uri, cursor, last_synced_at) + VALUES (?, ?, ?, ?) + ON CONFLICT (host_url, space_uri) DO UPDATE SET + cursor = excluded.cursor, + last_synced_at = excluded.last_synced_at` + ) + .bind(hostUrl, spaceUri, cursor, Date.now()) + .run(); +} diff --git a/packages/contrail-record-host/src/index.ts b/packages/contrail-record-host/src/index.ts index ff4eacd..df57f36 100644 --- a/packages/contrail-record-host/src/index.ts +++ b/packages/contrail-record-host/src/index.ts @@ -25,3 +25,6 @@ export type { BlobGcOptions, BlobGcResult } from "./blob-gc"; export { collectBlobCids } from "./blob-refs"; export { registerRecordHostRoutes } from "./routes"; + +export { registerRecordHostSyncRoutes } from "./sync"; +export type { RecordHostSyncOptions, SyncEvent } from "./sync"; diff --git a/packages/contrail-record-host/src/sync.ts b/packages/contrail-record-host/src/sync.ts new file mode 100644 index 0000000..00af1e1 --- /dev/null +++ b/packages/contrail-record-host/src/sync.ts @@ -0,0 +1,331 @@ +/** Record-host sync endpoint — streams record events for a specific space. + * + * Two phases per connection: + * 1. **Catch-up**: scan the per-collection `spaces_records_<short>` tables + * for `time_us > since`, emit each row as `record.created`. After each + * batch, emit a `cursor` checkpoint so the client can persist progress. + * 2. **Live**: subscribe to the in-process pubsub for `space:<uri>` and + * forward record.created / record.deleted events. + * + * Phase 7b MVP limitations: + * - Catch-up only sees `record.created`; deletions in the past aren't + * replayed (the row is gone). Live deletions are emitted. + * - Brief race between catch-up end and live subscribe — if a write + * lands in that window, the next reconnect catches it via catch-up. + * - SSE only (no WS). Simpler. WS can be added if needed. + * + * Auth: requires a valid X-Space-Credential whose `space` claim matches + * the requested spaceUri and whose scope is `read` or `rw`. */ + +import type { Hono, MiddlewareHandler } from "hono"; +import type { + ContrailConfig, + CredentialClaims, + CredentialVerifier, + PubSub, + RealtimeEvent, + RecordHost, +} from "@atmo-dev/contrail-base"; +import { + DEFAULT_KEEPALIVE_MS, + extractSpaceCredential, + shortNameForNsid, + spacesRecordsTableName, + spaceTopic, +} from "@atmo-dev/contrail-base"; +import type { Database } from "@atmo-dev/contrail-base"; + +/** Events emitted on the wire. RealtimeEvent kinds (record.created / + * record.deleted) plus our own `cursor` checkpoint. */ +export type SyncEvent = + | RealtimeEvent + | { kind: "cursor"; value: string }; + +export interface RecordHostSyncOptions { + /** Database the host's record tables live on. */ + db: Database; + /** Optional pubsub for live mode. When omitted, sync is catch-up only — + * the stream ends after catch-up rather than tailing for new writes. */ + pubsub?: PubSub | null; + /** Required: verifier for the X-Space-Credential header. */ + credentialVerifier: CredentialVerifier; + /** Page size for catch-up scans. Default 100. */ + batchSize?: number; + /** SSE keepalive interval in ms. Default uses the realtime module's. */ + keepaliveMs?: number; +} + +export function registerRecordHostSyncRoutes( + app: Hono, + recordHost: RecordHost, + config: ContrailConfig, + options: RecordHostSyncOptions +): void { + const RECORD_HOST = `${config.namespace}.recordHost`; + const batchSize = options.batchSize ?? 100; + const keepaliveMs = options.keepaliveMs ?? DEFAULT_KEEPALIVE_MS; + + app.get(`/xrpc/${RECORD_HOST}.sync`, async (c) => { + // ---- Auth: credential required ---- + const credToken = extractSpaceCredential(c.req.raw); + if (!credToken) { + return c.json( + { error: "AuthRequired", reason: "credential-required" }, + 401 + ); + } + const verified = await options.credentialVerifier.verify(credToken); + if (!verified.ok) { + return c.json({ error: "AuthRequired", reason: verified.reason }, 401); + } + const claims = verified.claims; + + const spaceUri = c.req.query("spaceUri"); + if (!spaceUri) { + return c.json( + { error: "InvalidRequest", message: "spaceUri required" }, + 400 + ); + } + if (claims.space !== spaceUri) { + return c.json( + { error: "Forbidden", reason: "credential-wrong-space" }, + 403 + ); + } + + const enrollment = await recordHost.getEnrollment(spaceUri); + if (!enrollment) { + return c.json( + { error: "NotFound", reason: "not-enrolled" }, + 404 + ); + } + + const since = parseSince(c.req.query("since")); + + // Build the SSE response with a hand-rolled stream so we can interleave + // catch-up batches and live events under one cursor sequence. + const ac = new AbortController(); + c.req.raw.signal.addEventListener("abort", () => ac.abort(), { once: true }); + + const stream = new ReadableStream<Uint8Array>({ + async start(controller) { + const enc = new TextEncoder(); + let closed = false; + const close = () => { + if (closed) return; + closed = true; + try { controller.close(); } catch { /* already closed */ } + }; + ac.signal.addEventListener("abort", close, { once: true }); + + const keepalive = setInterval(() => { + if (closed) return; + try { + controller.enqueue(enc.encode(`: keepalive\n\n`)); + } catch { + close(); + } + }, keepaliveMs); + + const writeEvent = (e: SyncEvent) => { + if (closed) return false; + try { + controller.enqueue(enc.encode(frameEvent(e))); + return true; + } catch { + close(); + return false; + } + }; + + try { + controller.enqueue(enc.encode(`: open\n\n`)); + + // ---- Phase 1: catch-up ---- + const lastCursor = await streamCatchup({ + db: options.db, + config, + spaceUri, + since, + batchSize, + writeEvent, + isClosed: () => closed, + }); + if (closed) return; + + // ---- Phase 2: live ---- + if (options.pubsub) { + const liveCutoff = lastCursor ?? since; + for await (const event of options.pubsub.subscribe( + spaceTopic(spaceUri), + ac.signal + )) { + if (closed) break; + // Ignore events older than what catch-up already covered. + const ts = (event as any).payload?.time_us ?? null; + if ( + liveCutoff != null && + typeof ts === "number" && + ts <= liveCutoff + ) { + continue; + } + if ( + event.kind !== "record.created" && + event.kind !== "record.deleted" + ) { + continue; + } + const ok = writeEvent(event); + if (!ok) break; + // Emit a cursor checkpoint after each live event so consumers + // can resume from the latest seen point. + if (typeof ts === "number") { + writeEvent({ kind: "cursor", value: String(ts) }); + } + } + } + } catch (err) { + if (!closed) { + try { + controller.enqueue( + enc.encode( + `event: error\ndata: ${JSON.stringify({ + message: err instanceof Error ? err.message : String(err), + })}\n\n` + ) + ); + } catch { + /* already torn down */ + } + } + } finally { + clearInterval(keepalive); + close(); + } + }, + cancel() { + ac.abort(); + }, + }); + + return new Response(stream, { + status: 200, + headers: { + "content-type": "text/event-stream", + "cache-control": "no-cache, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + }, + }); + }); +} + +/** Catch-up phase: scan every configured per-collection spaces_records table + * for rows with `time_us > since`, emit them in time_us order. Returns the + * highest time_us emitted (or null if nothing emitted). */ +async function streamCatchup(args: { + db: Database; + config: ContrailConfig; + spaceUri: string; + since: number | null; + batchSize: number; + writeEvent: (e: SyncEvent) => boolean; + isClosed: () => boolean; +}): Promise<number | null> { + const { db, config, spaceUri, since, batchSize, writeEvent, isClosed } = args; + let highest = since; + + for (const [_short, colCfg] of Object.entries(config.collections)) { + if (colCfg.allowInSpaces === false) continue; + if (isClosed()) return highest; + + const collectionNsid = colCfg.collection; + const short = shortNameForNsid(config, collectionNsid); + if (!short) continue; + const table = spacesRecordsTableName(short); + + let cursor = since; + while (true) { + if (isClosed()) return highest; + + let rows: any[]; + try { + const sql = + cursor != null + ? `SELECT * FROM ${table} WHERE space_uri = ? AND time_us > ? ORDER BY time_us LIMIT ?` + : `SELECT * FROM ${table} WHERE space_uri = ? ORDER BY time_us LIMIT ?`; + const result = await (cursor != null + ? db.prepare(sql).bind(spaceUri, cursor, batchSize).all<any>() + : db.prepare(sql).bind(spaceUri, batchSize).all<any>()); + rows = result.results; + } catch { + // Table missing — skip this collection silently. + break; + } + + if (rows.length === 0) break; + + for (const row of rows) { + if (isClosed()) return highest; + const time_us = numericish(row.time_us); + const event: RealtimeEvent = { + topic: spaceTopic(spaceUri), + kind: "record.created", + payload: { + uri: row.uri, + did: row.did, + collection: collectionNsid, + rkey: row.rkey, + cid: row.cid ?? null, + record: parseRecordJson(row.record), + time_us, + space: spaceUri, + }, + ts: Date.now(), + }; + if (!writeEvent(event)) return highest; + if (highest == null || time_us > highest) highest = time_us; + cursor = time_us; + } + + // Cursor checkpoint after the batch. + if (highest != null) { + writeEvent({ kind: "cursor", value: String(highest) }); + } + + if (rows.length < batchSize) break; + } + } + + return highest; +} + +function frameEvent(event: SyncEvent): string { + return `event: ${event.kind}\ndata: ${JSON.stringify(event)}\n\n`; +} + +function parseSince(raw: string | undefined): number | null { + if (!raw) return null; + const n = Number(raw); + if (Number.isNaN(n) || !Number.isFinite(n)) return null; + return n; +} + +function numericish(v: unknown): number { + return typeof v === "string" ? Number(v) : (v as number); +} + +function parseRecordJson(value: unknown): Record<string, unknown> { + if (value == null) return {}; + if (typeof value === "string") { + try { + return JSON.parse(value) as Record<string, unknown>; + } catch { + return {}; + } + } + return value as Record<string, unknown>; +} diff --git a/packages/contrail/tests/sync-e2e.test.ts b/packages/contrail/tests/sync-e2e.test.ts new file mode 100644 index 0000000..154385f --- /dev/null +++ b/packages/contrail/tests/sync-e2e.test.ts @@ -0,0 +1,304 @@ +/** End-to-end multi-host sync test. + * + * Two Contrail instances (host A and appview B) running in one process, + * each with its own SQLite DB. Host A creates a space and writes records. + * Appview B opens a sync stream against host A and ingests the records + * into its own tables. We verify B can query the records locally. + * + * The "network" between them is host A's hono fetch, threaded through + * appview B's fetch parameter. Real deployments would use the actual + * fetch over HTTPS. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { HostedAdapter } from "../src/core/spaces/adapter"; +import { registerRecordHostSyncRoutes } from "@atmo-dev/contrail-record-host"; +import { runRecordHostSync, applyRecordSyncSchema } from "@atmo-dev/contrail-appview"; +import { + generateAuthoritySigningKey, + issueCredential, + createInProcessVerifier, + InMemoryPubSub, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; +import { wrapWithPublishing } from "../src/core/realtime/publishing-adapter"; + +const ALICE = "did:plc:alice"; +const SERVICE_DID = "did:web:test.example#svc"; +const SPACE_TYPE = "tools.atmo.event.space"; +const SPACE_KEY = "main"; +const SPACE_URI = `ats://${ALICE}/${SPACE_TYPE}/${SPACE_KEY}`; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +const HOST_CONFIG: ContrailConfig = { + namespace: "test.sync", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: SPACE_TYPE, serviceDid: SERVICE_DID }, + recordHost: {}, + }, +}; + +async function makeHost(): Promise<{ + app: Hono; + adapter: HostedAdapter; + db: any; +}> { + const db = createSqliteDatabase(":memory:"); + const cfg = { ...HOST_CONFIG }; + cfg.spaces!.authority!.signing = SIGNING; + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + + const baseAdapter = new HostedAdapter(db, resolved); + const pubsub = new InMemoryPubSub(); + const adapter = wrapWithPublishing(baseAdapter, pubsub) as HostedAdapter; + + // Provision a space + enroll on this host. + await adapter.createSpace({ + uri: SPACE_URI, + ownerDid: ALICE, + type: SPACE_TYPE, + key: SPACE_KEY, + serviceDid: SERVICE_DID, + appPolicyRef: null, + appPolicy: null, + }); + await adapter.addMember(SPACE_URI, ALICE, ALICE); + await adapter.enroll({ + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + enrolledAt: Date.now(), + enrolledBy: ALICE, + }); + + const app = new Hono(); + const verifier = createInProcessVerifier({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }); + registerRecordHostSyncRoutes(app, adapter, resolved, { + db, + pubsub, + credentialVerifier: verifier, + keepaliveMs: 60_000, + batchSize: 50, + }); + + return { app, adapter, db }; +} + +async function makeAppview(): Promise<{ db: any; adapter: HostedAdapter; resolved: any }> { + const db = createSqliteDatabase(":memory:"); + const cfg = { ...HOST_CONFIG }; + cfg.spaces!.authority!.signing = SIGNING; + const resolved = resolveConfig(cfg); + await initSchema(db, resolved, { + extraSchemas: [applyRecordSyncSchema], + }); + const adapter = new HostedAdapter(db, resolved); + return { db, adapter, resolved }; +} + +async function mintCredentialForAlice(): Promise<string> { + const { credential } = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: SPACE_URI, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + return credential; +} + +describe("recordHost.sync — end-to-end (host → appview)", () => { + it("appview ingests historical records into its own tables", async () => { + const host = await makeHost(); + const appview = await makeAppview(); + + // Plant 3 records on the host. + const t0 = Date.now(); + for (let i = 0; i < 3; i++) { + await host.adapter.putRecord({ + spaceUri: SPACE_URI, + collection: "app.event.message", + authorDid: ALICE, + rkey: `rk${i}`, + cid: null, + record: { $type: "app.event.message", text: `msg-${i}` }, + createdAt: t0 + i, + }); + } + + // Appview opens a sync stream with a 200ms abort — long enough for + // catch-up to drain, then we tear down before the live phase blocks. + const ac = new AbortController(); + const credential = await mintCredentialForAlice(); + const cursors: string[] = []; + + const syncPromise = runRecordHostSync( + { + hostUrl: "http://host", + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + credential, + }, + { + db: appview.db, + config: appview.resolved, + recordHost: appview.adapter, + // Route fetch to host A's hono app instead of the network. + fetch: ((input: any, init?: any) => { + const req = typeof input === "string" || input instanceof URL + ? new Request(input, init) + : (input as Request); + return host.app.fetch(req); + }) as typeof fetch, + signal: ac.signal, + onCursor: (c) => { + cursors.push(c); + // Once we've seen at least one cursor checkpoint, catch-up is + // making progress. Schedule abort to break out of live mode. + setTimeout(() => ac.abort(), 50); + }, + } + ).catch((err) => { + // AbortError is expected; rethrow other errors. + if (err.name !== "AbortError") throw err; + }); + + await syncPromise; + + // Verify the appview's local tables now have the host's records. + const ingested = await appview.adapter.listRecords( + SPACE_URI, + "app.event.message" + ); + expect(ingested.records).toHaveLength(3); + expect(ingested.records.map((r) => r.rkey).sort()).toEqual([ + "rk0", + "rk1", + "rk2", + ]); + expect(cursors.length).toBeGreaterThan(0); + + // Cursor was persisted in the subscriptions table. + const subRow = await appview.db + .prepare( + `SELECT cursor FROM record_sync_subscriptions WHERE host_url = ? AND space_uri = ?` + ) + .bind("http://host", SPACE_URI) + .first<{ cursor: string | null }>(); + expect(subRow?.cursor).toBeTruthy(); + + // Auto-enrolled on the appview side. + const enrollment = await appview.adapter.getEnrollment(SPACE_URI); + expect(enrollment).toBeTruthy(); + expect(enrollment?.authorityDid).toBe(SERVICE_DID); + }); + + it("appview resumes from persisted cursor — no re-emission of old records", async () => { + const host = await makeHost(); + const appview = await makeAppview(); + const credential = await mintCredentialForAlice(); + + // First batch. + const t0 = Date.now(); + await host.adapter.putRecord({ + spaceUri: SPACE_URI, + collection: "app.event.message", + authorDid: ALICE, + rkey: "first", + cid: null, + record: { text: "first" }, + createdAt: t0, + }); + + const networkFetch = ((input: any, init?: any) => { + const req = typeof input === "string" || input instanceof URL + ? new Request(input, init) + : (input as Request); + return host.app.fetch(req); + }) as typeof fetch; + + const ac1 = new AbortController(); + const seenCount: { count: number } = { count: 0 }; + await runRecordHostSync( + { + hostUrl: "http://host", + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + credential, + }, + { + db: appview.db, + config: appview.resolved, + recordHost: appview.adapter, + fetch: networkFetch, + signal: ac1.signal, + onCursor: () => { + seenCount.count++; + setTimeout(() => ac1.abort(), 30); + }, + } + ).catch((err) => { + if (err.name !== "AbortError") throw err; + }); + + // Insert second batch on the host AFTER the first sync completed. + await host.adapter.putRecord({ + spaceUri: SPACE_URI, + collection: "app.event.message", + authorDid: ALICE, + rkey: "second", + cid: null, + record: { text: "second" }, + createdAt: t0 + 100, + }); + + // Second sync run — should pick up only the new record, using the + // persisted cursor. + const ac2 = new AbortController(); + const ingestedRkeys: string[] = []; + // Wrap putRecord to observe what gets re-ingested. + const origPut = appview.adapter.putRecord.bind(appview.adapter); + appview.adapter.putRecord = async (record) => { + ingestedRkeys.push(record.rkey); + return origPut(record); + }; + + await runRecordHostSync( + { + hostUrl: "http://host", + spaceUri: SPACE_URI, + authorityDid: SERVICE_DID, + credential, + }, + { + db: appview.db, + config: appview.resolved, + recordHost: appview.adapter, + fetch: networkFetch, + signal: ac2.signal, + onCursor: () => setTimeout(() => ac2.abort(), 30), + } + ).catch((err) => { + if (err.name !== "AbortError") throw err; + }); + + // The second run should have ingested ONLY "second" (not re-ingested "first"). + expect(ingestedRkeys).toEqual(["second"]); + }); +}); diff --git a/packages/contrail/tests/sync-host.test.ts b/packages/contrail/tests/sync-host.test.ts new file mode 100644 index 0000000..e07e5a3 --- /dev/null +++ b/packages/contrail/tests/sync-host.test.ts @@ -0,0 +1,282 @@ +/** Tests for the recordHost.sync SSE endpoint. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { HostedAdapter } from "../src/core/spaces/adapter"; +import { registerRecordHostSyncRoutes } from "@atmo-dev/contrail-record-host"; +import { + generateAuthoritySigningKey, + issueCredential, + createInProcessVerifier, + InMemoryPubSub, + spaceTopic, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; +import { wrapWithPublishing } from "../src/core/realtime/publishing-adapter"; + +const ALICE = "did:plc:alice"; +const SERVICE_DID = "did:web:test.example#svc"; +const SPACE_TYPE = "tools.atmo.event.space"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +const CONFIG: ContrailConfig = { + namespace: "test.sync", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { + type: SPACE_TYPE, + serviceDid: SERVICE_DID, + signing: undefined as any, // filled in beforeAll + }, + recordHost: {}, + }, +}; + +async function makeHost(): Promise<{ + app: Hono; + adapter: HostedAdapter; + pubsub: InMemoryPubSub; + spaceUri: string; +}> { + const db = createSqliteDatabase(":memory:"); + const cfg = { ...CONFIG }; + cfg.spaces!.authority!.signing = SIGNING; + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + + const baseAdapter = new HostedAdapter(db, resolved); + const pubsub = new InMemoryPubSub(); + // Wrap so writes also publish onto pubsub topics — mirrors what the + // umbrella createApp does in realtime mode. + const adapter = wrapWithPublishing(baseAdapter, pubsub) as HostedAdapter; + + // Create a space + enroll + const spaceUri = `ats://${ALICE}/${SPACE_TYPE}/main`; + await adapter.createSpace({ + uri: spaceUri, + ownerDid: ALICE, + type: SPACE_TYPE, + key: "main", + serviceDid: SERVICE_DID, + appPolicyRef: null, + appPolicy: null, + }); + await adapter.addMember(spaceUri, ALICE, ALICE); + await adapter.enroll({ + spaceUri, + authorityDid: SERVICE_DID, + enrolledAt: Date.now(), + enrolledBy: ALICE, + }); + + // Build the SSE app. + const app = new Hono(); + const verifier = createInProcessVerifier({ + authorityDid: SERVICE_DID, + publicKey: SIGNING.publicKey, + }); + registerRecordHostSyncRoutes(app, adapter, resolved, { + db, + pubsub, + credentialVerifier: verifier, + keepaliveMs: 60_000, + }); + + return { app, adapter, pubsub, spaceUri }; +} + +async function mintCredential(spaceUri: string): Promise<string> { + const { credential } = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: spaceUri, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + return credential; +} + +/** Read the SSE response, return the first N parsed events. */ +async function readEvents( + res: Response, + count: number, + timeoutMs = 2000 +): Promise<Array<{ kind: string; payload?: any; value?: string }>> { + const reader = res.body!.getReader(); + const decoder = new TextDecoder(); + const events: Array<{ kind: string; payload?: any; value?: string }> = []; + let buf = ""; + const start = Date.now(); + while (events.length < count) { + if (Date.now() - start > timeoutMs) { + throw new Error(`timeout waiting for ${count} events; got ${events.length}`); + } + const { done, value } = await reader.read(); + if (done) break; + buf += decoder.decode(value, { stream: true }); + while (true) { + const idx = buf.indexOf("\n\n"); + if (idx < 0) break; + const block = buf.slice(0, idx); + buf = buf.slice(idx + 2); + // Skip comment-only blocks (`: open`, `: keepalive`). + const dataLine = block + .split("\n") + .find((l) => l.startsWith("data:")); + if (!dataLine) continue; + const json = dataLine.slice(5).trim(); + if (!json) continue; + events.push(JSON.parse(json)); + if (events.length >= count) break; + } + } + await reader.cancel().catch(() => {}); + return events; +} + +describe("recordHost.sync — catch-up phase", () => { + it("emits historical records as record.created in time_us order", async () => { + const { app, adapter, spaceUri } = await makeHost(); + + // Plant 3 records with deterministic timestamps. + const now = Date.now(); + for (let i = 0; i < 3; i++) { + await adapter.putRecord({ + spaceUri, + collection: "app.event.message", + authorDid: ALICE, + rkey: `rk${i}`, + cid: null, + record: { $type: "app.event.message", text: `msg-${i}` }, + createdAt: now + i, + }); + } + + const credential = await mintCredential(spaceUri); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}`, + { headers: { "X-Space-Credential": credential } } + ) + ); + expect(res.status).toBe(200); + + // Expect 3 record.created + at least one cursor checkpoint. + const events = await readEvents(res, 4); + const records = events.filter((e) => e.kind === "record.created"); + expect(records).toHaveLength(3); + expect(records.map((e) => e.payload.rkey)).toEqual(["rk0", "rk1", "rk2"]); + expect(records.every((e) => e.payload.space === spaceUri)).toBe(true); + const cursors = events.filter((e) => e.kind === "cursor"); + expect(cursors.length).toBeGreaterThan(0); + }); + + it("respects the since cursor — records at or before are skipped", async () => { + const { app, adapter, spaceUri } = await makeHost(); + + const t0 = 1_700_000_000_000; + await adapter.putRecord({ + spaceUri, + collection: "app.event.message", + authorDid: ALICE, + rkey: "old", + cid: null, + record: { text: "old" }, + createdAt: t0, + }); + await adapter.putRecord({ + spaceUri, + collection: "app.event.message", + authorDid: ALICE, + rkey: "new", + cid: null, + record: { text: "new" }, + createdAt: t0 + 100, + }); + + const credential = await mintCredential(spaceUri); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}&since=${t0}`, + { headers: { "X-Space-Credential": credential } } + ) + ); + expect(res.status).toBe(200); + + const events = await readEvents(res, 2); + const records = events.filter((e) => e.kind === "record.created"); + expect(records).toHaveLength(1); + expect(records[0]!.payload.rkey).toBe("new"); + }); +}); + +describe("recordHost.sync — auth + enrollment guards", () => { + it("rejects without a credential", async () => { + const { app, spaceUri } = await makeHost(); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}` + ) + ); + expect(res.status).toBe(401); + expect((await res.json() as any).reason).toBe("credential-required"); + }); + + it("rejects credential whose space doesn't match", async () => { + const { app, spaceUri } = await makeHost(); + const wrongSpaceCred = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: "ats://did:plc:alice/x/y", + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(spaceUri)}`, + { headers: { "X-Space-Credential": wrongSpaceCred.credential } } + ) + ); + expect(res.status).toBe(403); + expect((await res.json() as any).reason).toBe("credential-wrong-space"); + }); + + it("rejects un-enrolled spaces", async () => { + const { app } = await makeHost(); + const otherSpaceUri = `ats://${ALICE}/${SPACE_TYPE}/different`; + const cred = await issueCredential( + { + iss: SERVICE_DID, + sub: ALICE, + space: otherSpaceUri, + scope: "rw", + ttlMs: 60_000, + }, + SIGNING + ); + const res = await app.fetch( + new Request( + `http://localhost/xrpc/test.sync.recordHost.sync?spaceUri=${encodeURIComponent(otherSpaceUri)}`, + { headers: { "X-Space-Credential": cred.credential } } + ) + ); + expect(res.status).toBe(404); + expect((await res.json() as any).reason).toBe("not-enrolled"); + }); +}); diff --git a/packages/lexicons/lexicon-templates/recordHost/sync.json b/packages/lexicons/lexicon-templates/recordHost/sync.json new file mode 100644 index 0000000..795092f --- /dev/null +++ b/packages/lexicons/lexicon-templates/recordHost/sync.json @@ -0,0 +1,86 @@ +{ + "lexicon": 1, + "id": "tools.atmo.recordHost.sync", + "defs": { + "main": { + "type": "subscription", + "description": "SSE stream of record events for a specific space. Emits a catch-up phase (every record with `time_us > since`, in time_us order) followed by a live phase (record.created / record.deleted as they happen). Cursor checkpoints are emitted after each catch-up batch and after each live event so consumers can persist progress and resume on reconnect. Auth: requires X-Space-Credential whose `space` claim matches `spaceUri`.", + "parameters": { + "type": "params", + "required": ["spaceUri"], + "properties": { + "spaceUri": { + "type": "string", + "description": "Space to sync." + }, + "since": { + "type": "string", + "description": "Opaque cursor (last `value` from a `cursor` event). Server interprets as time_us; events with time_us > since are emitted." + } + } + }, + "message": { + "schema": { + "type": "union", + "refs": [ + "#recordCreated", + "#recordDeleted", + "#cursor" + ] + } + } + }, + "recordCreated": { + "type": "object", + "required": ["topic", "kind", "payload", "ts"], + "properties": { + "topic": { "type": "string" }, + "kind": { "type": "string", "const": "record.created" }, + "payload": { + "type": "object", + "required": ["uri", "did", "collection", "rkey", "record", "time_us", "space"], + "properties": { + "uri": { "type": "string" }, + "did": { "type": "string", "format": "did" }, + "collection": { "type": "string", "format": "nsid" }, + "rkey": { "type": "string" }, + "cid": { "type": "string", "format": "cid" }, + "record": { "type": "unknown" }, + "time_us": { "type": "integer" }, + "space": { "type": "string" } + } + }, + "ts": { "type": "integer" } + } + }, + "recordDeleted": { + "type": "object", + "required": ["topic", "kind", "payload", "ts"], + "properties": { + "topic": { "type": "string" }, + "kind": { "type": "string", "const": "record.deleted" }, + "payload": { + "type": "object", + "required": ["uri", "did", "collection", "rkey", "space"], + "properties": { + "uri": { "type": "string" }, + "did": { "type": "string", "format": "did" }, + "collection": { "type": "string", "format": "nsid" }, + "rkey": { "type": "string" }, + "space": { "type": "string" } + } + }, + "ts": { "type": "integer" } + } + }, + "cursor": { + "type": "object", + "required": ["kind", "value"], + "description": "Checkpoint: persist `value` and pass it as `since` on next reconnect to resume.", + "properties": { + "kind": { "type": "string", "const": "cursor" }, + "value": { "type": "string" } + } + } + } +} -- 2.51.2 From 00eb989887c3a9875c7ceedf7aef0c4bc1a7577e Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 02:50:16 +0200 Subject: [PATCH 12/19] phase 7c --- packages/contrail-authority/src/routes.ts | 53 ++++ packages/contrail-base/src/index.ts | 3 + packages/contrail-base/src/spaces/manifest.ts | 164 +++++++++++ packages/contrail-base/src/spaces/types.ts | 10 + .../contrail/tests/spaces-manifest.test.ts | 260 ++++++++++++++++++ .../spaces/getMembershipManifest.json | 41 +++ 6 files changed, 531 insertions(+) create mode 100644 packages/contrail-base/src/spaces/manifest.ts create mode 100644 packages/contrail/tests/spaces-manifest.test.ts create mode 100644 packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json diff --git a/packages/contrail-authority/src/routes.ts b/packages/contrail-authority/src/routes.ts index fe80e37..369ac31 100644 --- a/packages/contrail-authority/src/routes.ts +++ b/packages/contrail-authority/src/routes.ts @@ -21,9 +21,11 @@ import { checkInviteReadGrant, decodeUnverifiedClaims, DEFAULT_CREDENTIAL_TTL_MS, + DEFAULT_MANIFEST_TTL_MS, extractInviteToken, hashInviteToken, issueCredential, + issueMembershipManifest, nextTid, verifyCredential, } from "@atmo-dev/contrail-base"; @@ -324,6 +326,57 @@ export function registerAuthorityRoutes( ); return c.json({ credential, expiresAt }); }); + + // ---- Membership manifest ---- + // + // Issues a signed list of every space the caller is a member of (or owns) + // according to this authority. Appviews carry it on inbound requests so + // unioned listRecords queries can be filtered without syncing the full + // member list. Same key material as credentials, different payload. + + app.post(`/xrpc/${SPACE}.getMembershipManifest`, auth, async (c) => { + if (!authorityConfig.signing) { + return c.json( + { error: "NotImplemented", message: "authority is not configured to sign manifests" }, + 501 + ); + } + const sa = getAuth(c); + const cap = authorityConfig.manifestMaxSpaces ?? 500; + + // Page through listSpaces — owner-or-member union — up to the cap. + const seen = new Set<string>(); + const drain = async (scope: "owner" | "member"): Promise<void> => { + let cursor: string | undefined; + while (seen.size < cap) { + const result = await authority.listSpaces({ + ...(scope === "owner" ? { ownerDid: sa.issuer } : { memberDid: sa.issuer }), + cursor, + limit: Math.min(200, cap - seen.size), + }); + for (const s of result.spaces) { + if (s.deletedAt == null) seen.add(s.uri); + if (seen.size >= cap) break; + } + if (!result.cursor || seen.size >= cap) break; + cursor = result.cursor; + } + }; + await drain("owner"); + if (seen.size < cap) await drain("member"); + + const ttl = authorityConfig.manifestTtlMs ?? DEFAULT_MANIFEST_TTL_MS; + const { manifest, expiresAt } = await issueMembershipManifest( + { + iss: authorityConfig.serviceDid, + sub: sa.issuer, + spaces: [...seen], + ttlMs: ttl, + }, + authorityConfig.signing + ); + return c.json({ manifest, expiresAt, truncated: seen.size >= cap }); + }); } /** Verify a credential presented at refreshCredential. */ diff --git a/packages/contrail-base/src/index.ts b/packages/contrail-base/src/index.ts index e11f46c..1fde427 100644 --- a/packages/contrail-base/src/index.ts +++ b/packages/contrail-base/src/index.ts @@ -40,6 +40,9 @@ export * from "./spaces/acl"; // Credentials export * from "./spaces/credentials"; +// Membership manifests +export * from "./spaces/manifest"; + // Binding + key resolution export * from "./spaces/binding"; diff --git a/packages/contrail-base/src/spaces/manifest.ts b/packages/contrail-base/src/spaces/manifest.ts new file mode 100644 index 0000000..83eed8a --- /dev/null +++ b/packages/contrail-base/src/spaces/manifest.ts @@ -0,0 +1,164 @@ +/** Membership manifest: a short-lived signed list of spaces a caller is a + * member of, issued by an authority. Lets appviews filter unioned queries + * without syncing the full member list — each user carries their own + * bounded slice as they hit the appview. + * + * Same signing infrastructure as space credentials (ES256 JWT), different + * payload + endpoint. */ + +import { + signCredential, + verifyCredential, + decodeUnverifiedClaims, + type CredentialKeyMaterial, +} from "./credentials"; + +const ALG = "ES256"; +const TYP = "JWT"; +const DEFAULT_KEY_ID = "atproto_space_authority"; + +/** Default manifest TTL — same 2h as credentials. */ +export const DEFAULT_MANIFEST_TTL_MS = 2 * 60 * 60 * 1000; + +export interface MembershipManifestClaims { + /** Authority DID that issued (and signed) the manifest. */ + iss: string; + /** User DID this manifest is for. */ + sub: string; + /** Space URIs the user is a member of (according to this authority). */ + spaces: string[]; + /** Seconds since epoch. */ + iat: number; + exp: number; +} + +/** Sign a manifest using the authority's signing key. */ +export async function signMembershipManifest( + payload: MembershipManifestClaims, + key: CredentialKeyMaterial +): Promise<string> { + // signCredential internally builds the JWT given a CredentialClaims-like + // shape. The manifest payload has different fields (`spaces` instead of + // `space`/`scope`) so we hand-roll the JWT here using the same utilities. + const enc = new TextEncoder(); + const kid = `${payload.iss}#${key.keyId ?? DEFAULT_KEY_ID}`; + const header = { alg: ALG, typ: TYP, kid }; + const head = base64urlEncode(enc.encode(JSON.stringify(header))); + const body = base64urlEncode(enc.encode(JSON.stringify(payload))); + const signingInput = `${head}.${body}`; + const privateKey = await crypto.subtle.importKey( + "jwk", + key.privateKey, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["sign"] + ); + const sig = await crypto.subtle.sign( + { name: "ECDSA", hash: "SHA-256" }, + privateKey, + enc.encode(signingInput) + ); + return `${signingInput}.${base64urlEncode(new Uint8Array(sig))}`; +} + +/** Issue a manifest with current iat/exp. */ +export async function issueMembershipManifest( + args: Omit<MembershipManifestClaims, "iat" | "exp"> & { ttlMs: number }, + key: CredentialKeyMaterial +): Promise<{ manifest: string; expiresAt: number }> { + const now = Math.floor(Date.now() / 1000); + const expSec = now + Math.floor(args.ttlMs / 1000); + const payload: MembershipManifestClaims = { + iss: args.iss, + sub: args.sub, + spaces: args.spaces, + iat: now, + exp: expSec, + }; + const manifest = await signMembershipManifest(payload, key); + return { manifest, expiresAt: expSec * 1000 }; +} + +export type ManifestVerifyOk = { ok: true; claims: MembershipManifestClaims }; +export type ManifestVerifyErr = { + ok: false; + reason: "malformed" | "bad-alg" | "bad-signature" | "expired" | "not-yet-valid" | "unknown-issuer"; +}; + +export interface VerifyManifestOptions { + /** Resolve the issuer's verification key. */ + resolveKey: (iss: string, kid: string | undefined) => Promise<JsonWebKey | null>; + /** Time provider for tests. */ + now?: () => number; +} + +export async function verifyMembershipManifest( + jwt: string, + opts: VerifyManifestOptions +): Promise<ManifestVerifyOk | ManifestVerifyErr> { + const parts = jwt.split("."); + if (parts.length !== 3) return { ok: false, reason: "malformed" }; + const [headSeg, bodySeg, sigSeg] = parts as [string, string, string]; + + let header: { alg?: string; kid?: string }; + let claims: MembershipManifestClaims; + try { + header = JSON.parse(new TextDecoder().decode(base64urlDecode(headSeg))); + claims = JSON.parse(new TextDecoder().decode(base64urlDecode(bodySeg))); + } catch { + return { ok: false, reason: "malformed" }; + } + if (header.alg !== ALG) return { ok: false, reason: "bad-alg" }; + if (!Array.isArray(claims.spaces)) return { ok: false, reason: "malformed" }; + + const nowMs = (opts.now ?? Date.now)(); + const nowSec = Math.floor(nowMs / 1000); + if (claims.exp <= nowSec) return { ok: false, reason: "expired" }; + if (claims.iat > nowSec + 60) return { ok: false, reason: "not-yet-valid" }; + + const jwk = await opts.resolveKey(claims.iss, header.kid); + if (!jwk) return { ok: false, reason: "unknown-issuer" }; + + const publicKey = await crypto.subtle.importKey( + "jwk", + jwk, + { name: "ECDSA", namedCurve: "P-256" }, + false, + ["verify"] + ); + const sigBytes = base64urlDecode(sigSeg); + const enc = new TextEncoder(); + const valid = await crypto.subtle.verify( + { name: "ECDSA", hash: "SHA-256" }, + publicKey, + sigBytes as BufferSource, + enc.encode(`${headSeg}.${bodySeg}`) + ); + if (!valid) return { ok: false, reason: "bad-signature" }; + return { ok: true, claims }; +} + +/** Peek at claims without verifying — useful for routing decisions. */ +export function decodeUnverifiedManifest(jwt: string): MembershipManifestClaims | null { + const parts = jwt.split("."); + if (parts.length !== 3) return null; + try { + return JSON.parse(new TextDecoder().decode(base64urlDecode(parts[1]!))) as MembershipManifestClaims; + } catch { + return null; + } +} + +// Local base64url helpers — mirror the credentials module so we don't expose +// these as public utilities. +function base64urlEncode(bytes: Uint8Array): string { + const s = btoa(String.fromCharCode(...bytes)); + return s.replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, ""); +} +function base64urlDecode(s: string): Uint8Array { + const padded = s.replace(/-/g, "+").replace(/_/g, "/").padEnd(Math.ceil(s.length / 4) * 4, "="); + const bin = atob(padded); + const out = new Uint8Array(bin.length); + for (let i = 0; i < bin.length; i++) out[i] = bin.charCodeAt(i); + return out; +} diff --git a/packages/contrail-base/src/spaces/types.ts b/packages/contrail-base/src/spaces/types.ts index 6561773..0e62f6c 100644 --- a/packages/contrail-base/src/spaces/types.ts +++ b/packages/contrail-base/src/spaces/types.ts @@ -52,6 +52,16 @@ export interface AuthorityConfig { signing?: CredentialKeyMaterial; /** Credential lifetime in ms. Defaults to {@link DEFAULT_CREDENTIAL_TTL_MS}. */ credentialTtlMs?: number; + /** Membership-manifest lifetime in ms. Manifests carry a user's full + * member-of list and let appviews filter unioned queries without syncing + * the authority's full member tables. Same key material as credentials. + * Defaults to {@link DEFAULT_MANIFEST_TTL_MS}. */ + manifestTtlMs?: number; + /** Maximum number of spaces returned in a manifest. The endpoint paginates + * through `listSpaces` up to this cap; users with more spaces get a + * truncated manifest (the remainder won't be unioned in queries). Defaults + * to 500. */ + manifestMaxSpaces?: number; } /** Configuration for the **record host** role: stores per-space records and diff --git a/packages/contrail/tests/spaces-manifest.test.ts b/packages/contrail/tests/spaces-manifest.test.ts new file mode 100644 index 0000000..3b600f5 --- /dev/null +++ b/packages/contrail/tests/spaces-manifest.test.ts @@ -0,0 +1,260 @@ +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import type { MiddlewareHandler } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + generateAuthoritySigningKey, + signMembershipManifest, + verifyMembershipManifest, + decodeUnverifiedManifest, + issueMembershipManifest, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const CHARLIE = "did:plc:charlie"; + +const SERVICE_DID = "did:web:test.example#svc"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function makeConfig(overrides?: Partial<ContrailConfig["spaces"] extends { authority?: infer A } ? A : never>): ContrailConfig { + return { + namespace: "test.man", + collections: { + message: { collection: "app.event.message" }, + }, + spaces: { + authority: { + type: "tools.atmo.event.space", + serviceDid: SERVICE_DID, + signing: SIGNING, + manifestTtlMs: 60_000, + ...(overrides ?? {}), + }, + recordHost: {}, + }, + }; +} + +function fakeAuth(): MiddlewareHandler { + return async (c, next) => { + const did = c.req.header("X-Test-Did"); + if (!did) return c.json({ error: "AuthRequired" }, 401); + c.set("serviceAuth", { + issuer: did, + audience: SERVICE_DID, + lxm: undefined, + clientId: c.req.header("X-Test-App") ?? undefined, + }); + await next(); + }; +} + +async function makeApp(cfg: ContrailConfig = makeConfig()): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved, { spaces: { authMiddleware: fakeAuth() } }); +} + +function call(app: Hono, method: string, path: string, did: string | null, body?: any) { + const headers: Record<string, string> = {}; + if (did) headers["X-Test-Did"] = did; + if (body !== undefined) headers["Content-Type"] = "application/json"; + return app.fetch( + new Request(`http://localhost${path}`, { + method, + headers, + body: body !== undefined ? JSON.stringify(body) : undefined, + }) + ); +} + +async function createSpace(app: Hono, owner: string): Promise<string> { + const res = await call(app, "POST", "/xrpc/test.man.space.createSpace", owner, {}); + expect(res.status).toBe(200); + return ((await res.json()) as any).space.uri; +} + +describe("membership manifest — sign/verify primitives", () => { + it("round-trips via verifyMembershipManifest", async () => { + const { manifest } = await issueMembershipManifest( + { + iss: SERVICE_DID, + sub: ALICE, + spaces: ["ats://a/x/1", "ats://a/x/2"], + ttlMs: 60_000, + }, + SIGNING + ); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async (iss) => (iss === SERVICE_DID ? SIGNING.publicKey : null), + }); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.claims.iss).toBe(SERVICE_DID); + expect(result.claims.sub).toBe(ALICE); + expect(result.claims.spaces).toEqual(["ats://a/x/1", "ats://a/x/2"]); + } + }); + + it("rejects expired manifests", async () => { + const past = Math.floor(Date.now() / 1000) - 10; + const manifest = await signMembershipManifest( + { + iss: SERVICE_DID, + sub: ALICE, + spaces: ["ats://a/x/1"], + iat: past - 60, + exp: past, + }, + SIGNING + ); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("expired"); + }); + + it("rejects bad signatures", async () => { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: [], ttlMs: 60_000 }, + SIGNING + ); + const otherKey = await generateAuthoritySigningKey(); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async () => otherKey.publicKey, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("bad-signature"); + }); + + it("rejects unknown issuer", async () => { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: [], ttlMs: 60_000 }, + SIGNING + ); + const result = await verifyMembershipManifest(manifest, { + resolveKey: async () => null, + }); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reason).toBe("unknown-issuer"); + }); + + it("decodeUnverifiedManifest peeks without verifying", async () => { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: ["ats://a/x/1"], ttlMs: 60_000 }, + SIGNING + ); + const claims = decodeUnverifiedManifest(manifest); + expect(claims).not.toBeNull(); + expect(claims!.sub).toBe(ALICE); + expect(claims!.spaces).toEqual(["ats://a/x/1"]); + }); +}); + +describe("membership manifest — getMembershipManifest endpoint", () => { + it("returns a manifest covering owned + joined spaces", async () => { + const app = await makeApp(); + // Alice owns 2; Bob joins one of them. + const uriA1 = await createSpace(app, ALICE); + const uriA2 = await createSpace(app, ALICE); + await call(app, "POST", "/xrpc/test.man.space.addMember", ALICE, { + spaceUri: uriA1, + did: BOB, + }); + // Bob also owns one. + const uriB1 = await createSpace(app, BOB); + + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", BOB); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.manifest).toBeTypeOf("string"); + expect(body.expiresAt).toBeTypeOf("number"); + expect(body.truncated).toBe(false); + + // Verify signature + payload. + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async (iss) => (iss === SERVICE_DID ? SIGNING.publicKey : null), + }); + expect(result.ok).toBe(true); + if (result.ok) { + expect(result.claims.iss).toBe(SERVICE_DID); + expect(result.claims.sub).toBe(BOB); + expect(result.claims.spaces.sort()).toEqual([uriA1, uriB1].sort()); + } + + // Alice's membership shouldn't be visible to Bob's manifest. + if (result.ok) expect(result.claims.spaces).not.toContain(uriA2); + }); + + it("returns an empty spaces array for a user with no memberships", async () => { + const app = await makeApp(); + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", CHARLIE); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.claims.spaces).toEqual([]); + }); + + it("dedupes — owner who is also a member appears once", async () => { + const app = await makeApp(); + const uri = await createSpace(app, ALICE); + // createSpace internally addMembers the owner; both scopes will include it. + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", ALICE); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.claims.spaces).toEqual([uri]); + }); + + it("sets truncated:true when over manifestMaxSpaces", async () => { + const cfg = makeConfig(); + cfg.spaces!.authority!.manifestMaxSpaces = 2; + const app = await makeApp(cfg); + await createSpace(app, ALICE); + await createSpace(app, ALICE); + await createSpace(app, ALICE); + + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", ALICE); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + expect(body.truncated).toBe(true); + const result = await verifyMembershipManifest(body.manifest, { + resolveKey: async () => SIGNING.publicKey, + }); + expect(result.ok).toBe(true); + if (result.ok) expect(result.claims.spaces.length).toBe(2); + }); + + it("requires auth", async () => { + const app = await makeApp(); + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", null); + expect(res.status).toBe(401); + }); + + it("returns 501 when authority is not configured to sign", async () => { + const cfg = makeConfig(); + delete cfg.spaces!.authority!.signing; + const app = await makeApp(cfg); + const res = await call(app, "POST", "/xrpc/test.man.space.getMembershipManifest", ALICE); + expect(res.status).toBe(501); + }); +}); diff --git a/packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json b/packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json new file mode 100644 index 0000000..7061df7 --- /dev/null +++ b/packages/lexicons/lexicon-templates/spaces/getMembershipManifest.json @@ -0,0 +1,41 @@ +{ + "lexicon": 1, + "id": "tools.atmo.space.getMembershipManifest", + "defs": { + "main": { + "type": "procedure", + "description": "Mint a signed list of every space the caller is a member of (or owns) according to this authority. Appviews can carry the manifest on inbound requests so unioned listRecords queries are filtered against the caller's bounded slice without the appview having to sync the authority's full member list. Same key material as `getCredential`; different payload (an array of space URIs rather than a single space + scope). Truncated to `manifestMaxSpaces` (default 500) — the response sets `truncated: true` when the cap was hit.", + "input": { + "encoding": "application/json", + "schema": { + "type": "object", + "properties": {} + } + }, + "output": { + "encoding": "application/json", + "schema": { + "type": "object", + "required": ["manifest", "expiresAt", "truncated"], + "properties": { + "manifest": { + "type": "string", + "description": "Compact JWS (ES256) signed by the authority's key. Payload claims: `iss` (authority DID), `sub` (caller DID), `spaces` (array of space URIs), `iat`, `exp`." + }, + "expiresAt": { + "type": "integer", + "description": "Expiry as ms since epoch." + }, + "truncated": { + "type": "boolean", + "description": "True if the caller is in more spaces than fit in one manifest. The remainder won't be unioned in queries until the next refresh." + } + } + } + }, + "errors": [ + { "name": "NotImplemented", "description": "Authority is not configured to sign manifests." } + ] + } + } +} -- 2.51.2 From 30ce51aaa5fd18fc4204bc674b15cda7586e2988 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Sun, 3 May 2026 19:24:15 +0200 Subject: [PATCH 13/19] phase 7d? --- .../src/core/router/collection.ts | 38 +++- .../contrail-appview/src/core/router/index.ts | 16 ++ packages/contrail-base/src/spaces/manifest.ts | 37 +++ .../tests/spaces-manifest-appview.test.ts | 210 ++++++++++++++++++ packages/contrail/vitest.config.ts | 2 + 5 files changed, 298 insertions(+), 5 deletions(-) create mode 100644 packages/contrail/tests/spaces-manifest-appview.test.ts diff --git a/packages/contrail-appview/src/core/router/collection.ts b/packages/contrail-appview/src/core/router/collection.ts index 6d43d00..9482631 100644 --- a/packages/contrail-appview/src/core/router/collection.ts +++ b/packages/contrail-appview/src/core/router/collection.ts @@ -599,17 +599,45 @@ export function registerCollectionRoutes( // Union path: when the caller is authenticated, fold in records from // spaces they're a member of. Anonymous callers just get public results. + // + // The caller authenticates with service-auth (`Authorization: Bearer + // <jwt>`). Their member-of space list comes from one of: + // 1. `X-Membership-Manifest` header — signed list issued by some + // authority asserting `sub` is in these spaces. The manifest's + // `sub` MUST match the JWT's issuer (DID) — the manifest is not + // a bearer token. Preferred for multi-authority deployments. + // 2. Local listSpaces — appview asks its own authority adapter + // `listSpaces({ memberDid: jwt.issuer })`. Works when the appview + // operator IS the authority. let spaceUris: string[] | undefined; const hasAuthHeader = !!c.req.header("Authorization"); + const manifestHeader = c.req.header("X-Membership-Manifest"); if (spacesCtx) { const nsid = new URL(c.req.url).pathname.match(/\/xrpc\/([^?]+)/)?.[1] as Nsid | null; const auth = await verifyServiceAuthRequest(spacesCtx.verifier, c.req.raw, nsid); if (auth) { - const { spaces } = await spacesCtx.adapter.listSpaces({ - memberDid: auth.issuer, - limit: 200, - }); - spaceUris = spaces.map((s) => s.uri); + if (manifestHeader && spacesCtx.manifestVerifier) { + const verified = await spacesCtx.manifestVerifier.verify(manifestHeader); + if (!verified.ok) { + return c.json( + { error: "AuthRequired", reason: verified.reason, message: "invalid membership manifest" }, + 401 + ); + } + if (verified.claims.sub !== auth.issuer) { + return c.json( + { error: "Forbidden", reason: "manifest-sub-mismatch", message: "manifest sub does not match caller" }, + 403 + ); + } + spaceUris = verified.claims.spaces; + } else { + const { spaces } = await spacesCtx.adapter.listSpaces({ + memberDid: auth.issuer, + limit: 200, + }); + spaceUris = spaces.map((s) => s.uri); + } } else if (hasAuthHeader) { // Had an auth header but it was invalid — reject rather than // silently downgrading to public results. diff --git a/packages/contrail-appview/src/core/router/index.ts b/packages/contrail-appview/src/core/router/index.ts index 0c5e058..10e4990 100644 --- a/packages/contrail-appview/src/core/router/index.ts +++ b/packages/contrail-appview/src/core/router/index.ts @@ -12,6 +12,8 @@ import { buildVerifier, createServiceAuthMiddleware } from "../spaces/auth"; import { HostedAdapter } from "../spaces/adapter"; import type { StorageAdapter } from "../spaces/types"; import type { ServiceJwtVerifier } from "@atcute/xrpc-server/auth"; +import { createManifestVerifier } from "@atmo-dev/contrail-base"; +import type { ManifestVerifier } from "@atmo-dev/contrail-base"; import type { CommunityIntegration } from "../community-integration"; import { registerRealtimeRoutes } from "../realtime/router"; import type { RealtimeRoutesOptions } from "../realtime/router"; @@ -29,6 +31,12 @@ import type { MiddlewareHandler } from "hono"; export interface SpacesContext { adapter: StorageAdapter; verifier: ServiceJwtVerifier; + /** Verifies inbound `X-Membership-Manifest` headers. Built automatically + * when an authority is configured locally with signing keys; deployments + * that aggregate manifests from multiple authorities should construct one + * via {@link createManifestVerifier} with a custom key resolver and pass + * it through `options.spacesCtx`. */ + manifestVerifier?: ManifestVerifier; } export interface CreateAppOptions { @@ -131,6 +139,14 @@ export function createApp( ? { adapter: options.spaces?.adapter ?? new HostedAdapter(spacesDb, config), verifier: buildVerifier(config.spaces.authority), + manifestVerifier: config.spaces.authority.signing + ? createManifestVerifier({ + resolveKey: async (iss) => + iss === config.spaces!.authority!.serviceDid + ? config.spaces!.authority!.signing!.publicKey + : null, + }) + : undefined, } : null; diff --git a/packages/contrail-base/src/spaces/manifest.ts b/packages/contrail-base/src/spaces/manifest.ts index 83eed8a..fbc8d31 100644 --- a/packages/contrail-base/src/spaces/manifest.ts +++ b/packages/contrail-base/src/spaces/manifest.ts @@ -138,6 +138,43 @@ export async function verifyMembershipManifest( return { ok: true, claims }; } +/** Stateful manifest verifier with a TTL'd in-memory cache keyed by JWT. + * Returns the same shape as {@link verifyMembershipManifest}; cache hits skip + * the crypto round-trip but still respect `exp`. Reuse one verifier per + * process — never construct per-request. */ +export interface ManifestVerifier { + verify(jwt: string): Promise<ManifestVerifyOk | ManifestVerifyErr>; +} + +/** Build a manifest verifier with caching. The cache is unbounded for now — + * manifests are short-lived (default 2h) and clients typically refresh them, + * so cardinality is bounded by active-user count. */ +export function createManifestVerifier(opts: VerifyManifestOptions): ManifestVerifier { + const cache = new Map<string, ManifestVerifyOk | ManifestVerifyErr>(); + const now = opts.now ?? Date.now; + return { + async verify(jwt) { + const cached = cache.get(jwt); + if (cached) { + if (cached.ok) { + if (cached.claims.exp * 1000 > now()) return cached; + cache.delete(jwt); + } else { + // Negative cache only for permanent failures (signature, alg); + // expired / not-yet-valid will roll over with the clock. + if (cached.reason === "bad-signature" || cached.reason === "bad-alg" || cached.reason === "malformed") { + return cached; + } + cache.delete(jwt); + } + } + const result = await verifyMembershipManifest(jwt, opts); + cache.set(jwt, result); + return result; + }, + }; +} + /** Peek at claims without verifying — useful for routing decisions. */ export function decodeUnverifiedManifest(jwt: string): MembershipManifestClaims | null { const parts = jwt.split("."); diff --git a/packages/contrail/tests/spaces-manifest-appview.test.ts b/packages/contrail/tests/spaces-manifest-appview.test.ts new file mode 100644 index 0000000..4d50cf7 --- /dev/null +++ b/packages/contrail/tests/spaces-manifest-appview.test.ts @@ -0,0 +1,210 @@ +/** Appview-side manifest consumption: cross-space listRecords union path + * honoring `X-Membership-Manifest`. */ + +import { describe, it, expect, beforeAll } from "vitest"; +import { Hono } from "hono"; +import { createSqliteDatabase } from "../src/adapters/sqlite"; +import { initSchema } from "../src/core/db/schema"; +import { createApp } from "../src/core/router"; +import { resolveConfig } from "../src/core/types"; +import type { ContrailConfig } from "../src/core/types"; +import { + generateAuthoritySigningKey, + issueMembershipManifest, + markInProcess, +} from "@atmo-dev/contrail-base"; +import type { CredentialKeyMaterial } from "@atmo-dev/contrail-base"; + +const ALICE = "did:plc:alice"; +const BOB = "did:plc:bob"; +const CHARLIE = "did:plc:charlie"; + +const SERVICE_DID = "did:web:test.example#svc"; +const SPACE_TYPE = "tools.atmo.event.space"; + +let SIGNING: CredentialKeyMaterial; + +beforeAll(async () => { + SIGNING = await generateAuthoritySigningKey(); +}); + +function makeConfig(): ContrailConfig { + return { + namespace: "test.man2", + collections: { + message: { collection: "app.event.message" }, + }, + spaces: { + authority: { + type: SPACE_TYPE, + serviceDid: SERVICE_DID, + signing: SIGNING, + }, + recordHost: {}, + }, + }; +} + +async function makeApp(): Promise<Hono> { + const db = createSqliteDatabase(":memory:"); + const cfg = makeConfig(); + const resolved = resolveConfig(cfg); + await initSchema(db, resolved); + return createApp(db, resolved); +} + +/** Make a Request marked with an in-process principal so the union path's + * `verifyServiceAuthRequest` returns the right caller without minting a real + * JWT or wiring up a key resolver. */ +function inProc(url: string, did: string, headers: Record<string, string> = {}): Request { + const req = new Request(url, { headers }); + return markInProcess(req, did); +} + +async function createSpace(app: Hono, owner: string, key: string): Promise<string> { + const res = await app.fetch( + markInProcess( + new Request("http://localhost/xrpc/test.man2.space.createSpace", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ key }), + }), + owner + ) + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + return body.space.uri; +} + +async function plant( + app: Hono, + did: string, + spaceUri: string, + text: string +): Promise<void> { + const res = await app.fetch( + markInProcess( + new Request("http://localhost/xrpc/test.man2.space.putRecord", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + spaceUri, + collection: "app.event.message", + record: { $type: "app.event.message", text }, + }), + }), + did + ) + ); + expect(res.status).toBe(200); +} + +async function mintManifest(sub: string, spaces: string[]): Promise<string> { + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub, spaces, ttlMs: 60_000 }, + SIGNING + ); + return manifest; +} + +describe("appview union listRecords — manifest-driven", () => { + it("uses manifest space list when valid + sub matches caller", async () => { + const app = await makeApp(); + // Alice owns one space, Bob owns another. + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + const bobSpace = await createSpace(app, BOB, "bob-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + await plant(app, BOB, bobSpace, "from-bob"); + + // Alice presents a manifest covering ONLY bobSpace (doesn't matter that + // she's not actually a member — the manifest is the source of truth here). + // The authority would never sign such a manifest in practice, but the + // appview's contract is: trust the verified manifest's claims. + const manifest = await mintManifest(ALICE, [bobSpace]); + + const res = await app.fetch( + inProc( + "http://localhost/xrpc/test.man2.message.listRecords", + ALICE, + { "X-Membership-Manifest": manifest } + ) + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const texts = body.records.map((r: any) => r.value.text); + expect(texts).toContain("from-bob"); + expect(texts).not.toContain("from-alice"); + }); + + it("rejects manifest whose sub does not match the caller", async () => { + const app = await makeApp(); + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + + // Bob's manifest, presented by Alice → should 403. + const bobManifest = await mintManifest(BOB, [aliceSpace]); + + const res = await app.fetch( + inProc( + "http://localhost/xrpc/test.man2.message.listRecords", + ALICE, + { "X-Membership-Manifest": bobManifest } + ) + ); + expect(res.status).toBe(403); + expect((await res.json() as any).reason).toBe("manifest-sub-mismatch"); + }); + + it("rejects an unsigned/forged manifest", async () => { + const app = await makeApp(); + const otherKey = await generateAuthoritySigningKey(); + const { manifest } = await issueMembershipManifest( + { iss: SERVICE_DID, sub: ALICE, spaces: [], ttlMs: 60_000 }, + otherKey + ); + const res = await app.fetch( + inProc( + "http://localhost/xrpc/test.man2.message.listRecords", + ALICE, + { "X-Membership-Manifest": manifest } + ) + ); + expect(res.status).toBe(401); + expect((await res.json() as any).reason).toBe("bad-signature"); + }); + + it("falls back to local listSpaces when no manifest is present", async () => { + const app = await makeApp(); + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + const bobSpace = await createSpace(app, BOB, "bob-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + await plant(app, BOB, bobSpace, "from-bob"); + + // Alice queries with no manifest → local listSpaces returns aliceSpace only. + const res = await app.fetch( + inProc("http://localhost/xrpc/test.man2.message.listRecords", ALICE) + ); + expect(res.status).toBe(200); + const body = (await res.json()) as any; + const texts = body.records.map((r: any) => r.value.text); + expect(texts).toContain("from-alice"); + expect(texts).not.toContain("from-bob"); + }); + + it("anonymous (no auth, no manifest) gets public results — no 401", async () => { + const app = await makeApp(); + const aliceSpace = await createSpace(app, ALICE, "alice-only"); + await plant(app, ALICE, aliceSpace, "from-alice"); + + const res = await app.fetch( + new Request("http://localhost/xrpc/test.man2.message.listRecords") + ); + // No auth header, no manifest → drops through to anonymous public path. + expect(res.status).toBe(200); + const body = (await res.json()) as any; + // Private space records aren't visible publicly. + const texts = (body.records ?? []).map((r: any) => r.value?.text); + expect(texts).not.toContain("from-alice"); + }); +}); diff --git a/packages/contrail/vitest.config.ts b/packages/contrail/vitest.config.ts index 996e91f..7f32f75 100644 --- a/packages/contrail/vitest.config.ts +++ b/packages/contrail/vitest.config.ts @@ -4,6 +4,7 @@ import path from "node:path"; const baseSrc = path.resolve(__dirname, "../contrail-base/src"); const authoritySrc = path.resolve(__dirname, "../contrail-authority/src"); const recordHostSrc = path.resolve(__dirname, "../contrail-record-host/src"); +const appviewSrc = path.resolve(__dirname, "../contrail-appview/src"); export default defineConfig({ test: { @@ -20,6 +21,7 @@ export default defineConfig({ "@atmo-dev/contrail-base": path.join(baseSrc, "index.ts"), "@atmo-dev/contrail-authority": path.join(authoritySrc, "index.ts"), "@atmo-dev/contrail-record-host": path.join(recordHostSrc, "index.ts"), + "@atmo-dev/contrail-appview": path.join(appviewSrc, "index.ts"), }, }, }); -- 2.51.2 From 50d4771056a7bfe5f5aaadf53ab3e6f5e8723fe0 Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Mon, 4 May 2026 12:20:17 -0400 Subject: [PATCH 14/19] test(contrail-e2e): adopt post-PR30 spaces split + community integration Bring the e2e suite back to green under the package split: - makeSpacesConfig() wraps the old flat { type, serviceDid, resolver } shape under authority / recordHost and generates a fresh signing key per call. The config validator now requires spaces.authority whenever community is set, so this is the minimum viable shape for any test that uses spaces. - setupCommunityContrail() wires the community module via createCommunityIntegration({ db, config: resolveConfig(...) }) and passes the result as communityIntegration to the Contrail constructor. The community config field is now opaque to contrail core; routes are registered through the integration option. - 7 test files updated to use the helpers; 4 community tests pick up a workspace dep on @atmo-dev/contrail-community. 35/35 e2e tests pass against devnet. No source-package changes. --- apps/contrail-e2e/package.json | 1 + .../tests/community-delete.test.ts | 12 ++-- .../tests/community-invites.test.ts | 12 ++-- .../tests/community-lifecycle.test.ts | 12 ++-- .../tests/community-publishing.test.ts | 13 ++--- apps/contrail-e2e/tests/helpers.ts | 58 +++++++++++++++++++ apps/contrail-e2e/tests/spaces-auth.test.ts | 8 +-- .../spaces-firehose-invisibility.test.ts | 8 +-- .../tests/spaces-table-isolation.test.ts | 8 +-- pnpm-lock.yaml | 3 + 10 files changed, 85 insertions(+), 50 deletions(-) diff --git a/apps/contrail-e2e/package.json b/apps/contrail-e2e/package.json index ed43978..aa17c4f 100644 --- a/apps/contrail-e2e/package.json +++ b/apps/contrail-e2e/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@atmo-dev/contrail": "workspace:*", + "@atmo-dev/contrail-community": "workspace:*", "pg": "^8.20.0" }, "devDependencies": { diff --git a/apps/contrail-e2e/tests/community-delete.test.ts b/apps/contrail-e2e/tests/community-delete.test.ts index 4e2ffd3..0106c66 100644 --- a/apps/contrail-e2e/tests/community-delete.test.ts +++ b/apps/contrail-e2e/tests/community-delete.test.ts @@ -26,7 +26,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -34,6 +33,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, login, jsonOr, @@ -66,14 +66,10 @@ describe("community.delete e2e (soft-delete + cascade, real DB)", () => { cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/community-invites.test.ts b/apps/contrail-e2e/tests/community-invites.test.ts index e3a5fd0..58b0b31 100644 --- a/apps/contrail-e2e/tests/community-invites.test.ts +++ b/apps/contrail-e2e/tests/community-invites.test.ts @@ -27,7 +27,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -35,6 +34,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, login, jsonOr, @@ -80,14 +80,10 @@ describe("invite e2e (community + user-owned, real JWT)", () => { cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/community-lifecycle.test.ts b/apps/contrail-e2e/tests/community-lifecycle.test.ts index 137991c..7a40f60 100644 --- a/apps/contrail-e2e/tests/community-lifecycle.test.ts +++ b/apps/contrail-e2e/tests/community-lifecycle.test.ts @@ -25,7 +25,6 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -33,6 +32,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, login, jsonOr, @@ -78,14 +78,10 @@ describe("community lifecycle (mint → grant → list → revoke, + gap probes) cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/community-publishing.test.ts b/apps/contrail-e2e/tests/community-publishing.test.ts index 6087dd7..8a11f89 100644 --- a/apps/contrail-e2e/tests/community-publishing.test.ts +++ b/apps/contrail-e2e/tests/community-publishing.test.ts @@ -22,7 +22,7 @@ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import pg from "pg"; import type { Client } from "@atcute/client"; import "@atcute/atproto"; -import { Contrail, runPersistent } from "@atmo-dev/contrail"; +import { runPersistent } from "@atmo-dev/contrail"; import { createHandler } from "@atmo-dev/contrail/server"; import { createPostgresDatabase } from "@atmo-dev/contrail/postgres"; import { config as baseConfig } from "../config"; @@ -30,6 +30,7 @@ import { createTestAccount, createIsolatedSchema, createDevnetResolver, + setupCommunityContrail, createCaller, createAppPasswordFor, devnetRewriteFetch, @@ -85,14 +86,10 @@ describe("community publishing (proxy → PDS → Jetstream → index)", () => { cleanupSchema = iso.cleanup; const db = createPostgresDatabase(pool); - const contrail = new Contrail({ - ...baseConfig, + const contrail = await setupCommunityContrail({ db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + baseConfig, + spaceType: SPACE_TYPE, community: { serviceDid: CONTRAIL_SERVICE_DID, masterKey: TEST_MASTER_KEY, diff --git a/apps/contrail-e2e/tests/helpers.ts b/apps/contrail-e2e/tests/helpers.ts index 64f493f..264bf67 100644 --- a/apps/contrail-e2e/tests/helpers.ts +++ b/apps/contrail-e2e/tests/helpers.ts @@ -12,6 +12,9 @@ import { PlcDidDocumentResolver, } from "@atcute/identity-resolver"; import type { Did as AtDid, Nsid } from "@atcute/lexicons"; +import { Contrail, generateAuthoritySigningKey, resolveConfig } from "@atmo-dev/contrail"; +import type { ContrailConfig, Database, SpacesConfig } from "@atmo-dev/contrail"; +import { createCommunityIntegration } from "@atmo-dev/contrail-community"; export type Did = `did:${string}:${string}`; @@ -46,6 +49,61 @@ export function createDevnetResolver() { }); } +/** + * Build a `spaces` config block in the post-PR30 split shape: authority owns + * ACL + credential signing, recordHost owns storage. Tests that previously + * passed the flat `{ type, serviceDid, resolver }` shape now get this — the + * config validator enforces the split, and `community` requires `authority`. + * + * Pass the `type` NSID for the kind of space (e.g. "rsvp.atmo.event.space"). + * A fresh signing key is generated per call so credential issuance works in + * the auth tests without leaking key material across suites. + */ +export async function makeSpacesConfig(type: string): Promise<SpacesConfig> { + return { + authority: { + type, + serviceDid: CONTRAIL_SERVICE_DID, + signing: await generateAuthoritySigningKey(), + resolver: createDevnetResolver(), + }, + recordHost: {}, + }; +} + +/** + * Build a Contrail wired with a community integration. Post-PR30, community + * routes are not registered by passing a `community` config block alone — the + * caller must construct a `CommunityIntegration` from the resolved config and + * pass it as `communityIntegration` to the `Contrail` constructor (the same + * pattern `createApp({ community })` uses in the contrail-community unit + * tests). + * + * `community` is forwarded into the Contrail config so the integration can + * read `masterKey`, `fetch`, etc. through `config.community`. + */ +export async function setupCommunityContrail(opts: { + db: Database; + baseConfig: ContrailConfig; + spaceType: string; + community: Record<string, unknown>; +}): Promise<Contrail> { + const fullConfig: ContrailConfig = { + ...opts.baseConfig, + spaces: await makeSpacesConfig(opts.spaceType), + community: opts.community, + }; + const integration = createCommunityIntegration({ + db: opts.db, + config: resolveConfig(fullConfig), + }); + return new Contrail({ + ...fullConfig, + db: opts.db, + communityIntegration: integration, + }); +} + /** * Mint an atproto service-auth JWT via the PDS's getServiceAuth endpoint. * Requires the client to already be authed for a user. Returns the raw JWT diff --git a/apps/contrail-e2e/tests/spaces-auth.test.ts b/apps/contrail-e2e/tests/spaces-auth.test.ts index 7e5124a..a9981fc 100644 --- a/apps/contrail-e2e/tests/spaces-auth.test.ts +++ b/apps/contrail-e2e/tests/spaces-auth.test.ts @@ -21,7 +21,7 @@ import { config as baseConfig } from "../config"; import { createTestAccount, createIsolatedSchema, - createDevnetResolver, + makeSpacesConfig, mintServiceAuthJwt, CONTRAIL_SERVICE_DID, PDS_URL, @@ -51,11 +51,7 @@ describe("spaces auth (devnet PDS JWT → Contrail verifier)", () => { const contrail = new Contrail({ ...baseConfig, db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + spaces: await makeSpacesConfig(SPACE_TYPE), }); await contrail.init(); handle = createHandler(contrail); diff --git a/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts b/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts index 07f7827..4d81933 100644 --- a/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts +++ b/apps/contrail-e2e/tests/spaces-firehose-invisibility.test.ts @@ -24,7 +24,7 @@ import { config as baseConfig } from "../config"; import { createTestAccount, createIsolatedSchema, - createDevnetResolver, + makeSpacesConfig, mintServiceAuthJwt, CONTRAIL_SERVICE_DID, PDS_URL, @@ -57,11 +57,7 @@ describe("spaces firehose invisibility", () => { const contrail = new Contrail({ ...baseConfig, db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + spaces: await makeSpacesConfig(SPACE_TYPE), }); await contrail.init(); handle = createHandler(contrail); diff --git a/apps/contrail-e2e/tests/spaces-table-isolation.test.ts b/apps/contrail-e2e/tests/spaces-table-isolation.test.ts index 9a3c8c8..c61aed5 100644 --- a/apps/contrail-e2e/tests/spaces-table-isolation.test.ts +++ b/apps/contrail-e2e/tests/spaces-table-isolation.test.ts @@ -21,7 +21,7 @@ import { config as baseConfig } from "../config"; import { createTestAccount, createIsolatedSchema, - createDevnetResolver, + makeSpacesConfig, mintServiceAuthJwt, CONTRAIL_SERVICE_DID, PDS_URL, @@ -52,11 +52,7 @@ describe("spaces table isolation", () => { const contrail = new Contrail({ ...baseConfig, db, - spaces: { - type: SPACE_TYPE, - serviceDid: CONTRAIL_SERVICE_DID, - resolver: createDevnetResolver(), - }, + spaces: await makeSpacesConfig(SPACE_TYPE), }); await contrail.init(); handle = createHandler(contrail); diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 2d68ee6..adc192d 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -45,6 +45,9 @@ importers: '@atmo-dev/contrail': specifier: workspace:* version: link:../../packages/contrail + '@atmo-dev/contrail-community': + specifier: workspace:* + version: link:../../packages/contrail-community pg: specifier: ^8.20.0 version: 8.20.0 -- 2.51.2 From 00094ee251fdc3ea3402fecf9441eea2f4f76146 Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 09:05:59 -0400 Subject: [PATCH 15/19] fix(record-host): require owner-signed enrollment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The recordHost.enroll endpoint accepted either an owner-signed call OR a self-attested authority call. The latter let any DID claim to be the authority for any space's URI and rebind that space's authority. Drop the self-attesting-authority branch. Require the caller to be the space owner (sa.issuer === parts.ownerDid). The owner is still free to designate any authority via body.authority — only the legitimacy of the caller changes. Updates the existing test that codified the old behavior to assert the new rejection, and adds a positive test for the canonical owner-driven split-deployment enrollment flow. --- packages/contrail-record-host/src/routes.ts | 6 +-- .../contrail/tests/spaces-enrollment.test.ts | 46 ++++++++++++++++--- 2 files changed, 41 insertions(+), 11 deletions(-) diff --git a/packages/contrail-record-host/src/routes.ts b/packages/contrail-record-host/src/routes.ts index 63fc951..25cd2ab 100644 --- a/packages/contrail-record-host/src/routes.ts +++ b/packages/contrail-record-host/src/routes.ts @@ -107,11 +107,9 @@ export function registerRecordHostRoutes( if (!parts) { return c.json({ error: "InvalidRequest", reason: "malformed-uri" }, 400); } - const callerIsOwner = sa.issuer === parts.ownerDid; - const callerIsAuthority = sa.issuer === body.authority; - if (!callerIsOwner && !callerIsAuthority) { + if (sa.issuer !== parts.ownerDid) { return c.json( - { error: "Forbidden", reason: "not-owner-or-authority" }, + { error: "Forbidden", reason: "not-owner" }, 403 ); } diff --git a/packages/contrail/tests/spaces-enrollment.test.ts b/packages/contrail/tests/spaces-enrollment.test.ts index 3b7ef5b..78cc8b0 100644 --- a/packages/contrail/tests/spaces-enrollment.test.ts +++ b/packages/contrail/tests/spaces-enrollment.test.ts @@ -127,7 +127,7 @@ describe("auto-enrollment via createSpace", () => { expect(((await reenroll.json()) as any).ok).toBe(true); }); - it("non-owner / non-authority callers cannot enroll", async () => { + it("non-owner callers cannot enroll", async () => { const app = await makeApp(); const create = await call(app, "POST", "/xrpc/test.enroll.space.createSpace", ALICE, {}); const uri = ((await create.json()) as any).space.uri; @@ -137,7 +137,7 @@ describe("auto-enrollment via createSpace", () => { authority: SERVICE_DID, }); expect(res.status).toBe(403); - expect((await res.json()).reason).toBe("not-owner-or-authority"); + expect((await res.json()).reason).toBe("not-owner"); }); }); @@ -369,10 +369,10 @@ describe("split deployment — authority and record host on separate apps", () = expect((await list.json()).reason).toBe("not-enrolled"); }); - it("the declared authority can enroll a space without the owner's involvement", async () => { - // Scenario: the authority service ('Contrail' in our naming) is a - // separate DID and acts on behalf of an owner. The authority can enroll - // because phase 5 accepts either owner-or-authority. + it("a third party cannot enroll a space by claiming to be the authority", async () => { + // Regression: the enroll handler used to accept either owner-signed + // OR authority-self-attested calls. That let any DID claim "I am the + // authority for ats://<victim>/..." and rebind the space. const authorityDb = createSqliteDatabase(":memory:"); const hostDb = createSqliteDatabase(":memory:"); const cfg: ContrailConfig = { @@ -392,11 +392,43 @@ describe("split deployment — authority and record host on separate apps", () = const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); const uri = ((await create.json()) as any).space.uri; - // Authority service identifies itself via the JWT issuer matching its DID. + // SERVICE_DID self-attests as the authority for Alice's space. + // Must be rejected — only the owner can enroll. const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", SERVICE_DID, { spaceUri: uri, authority: SERVICE_DID, }); + expect(enroll.status).toBe(403); + expect((await enroll.json()).reason).toBe("not-owner"); + }); + + it("the owner can enroll their space designating a separate authority", async () => { + // Positive: the owner-signed path lets Alice point her space at the + // configured authority service (the canonical split-deployment flow). + const authorityDb = createSqliteDatabase(":memory:"); + const hostDb = createSqliteDatabase(":memory:"); + const cfg: ContrailConfig = { + namespace: "test.split", + collections: { message: { collection: "app.event.message" } }, + spaces: { + authority: { type: "tools.atmo.event.space", serviceDid: SERVICE_DID, signing: SIGNING }, + recordHost: {}, + }, + }; + const resolved = resolveConfig(cfg); + await initSchema(authorityDb, resolved); + await initSchema(hostDb, resolved); + + const authorityApp = buildAuthorityApp(authorityDb); + const hostApp = buildRecordHostApp(hostDb); + const create = await call(authorityApp, "POST", "/xrpc/test.split.space.createSpace", ALICE, {}); + const uri = ((await create.json()) as any).space.uri; + + const enroll = await call(hostApp, "POST", "/xrpc/test.split.recordHost.enroll", ALICE, { + spaceUri: uri, + authority: SERVICE_DID, + }); expect(enroll.status).toBe(200); + expect(((await enroll.json()) as any).ok).toBe(true); }); }); -- 2.51.2 From be62232403137eed344abf923b183c75bcd9b41b Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 09:44:26 -0400 Subject: [PATCH 16/19] fix(spaces): drop LocalBindingResolver from default composite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The default in-process credential verifier composed [Enrollment, Local], so a never-enrolled space still resolved its authority via LocalBindingResolver — collapsing the host-side consent layer the enrollment table is meant to provide. Default to enrollment-only. createLocalBindingResolver remains exported for explicit single-tenant wiring via options.credentialVerifier. --- packages/contrail-appview/src/core/spaces/router.ts | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/packages/contrail-appview/src/core/spaces/router.ts b/packages/contrail-appview/src/core/spaces/router.ts index 943f6a9..a0557f5 100644 --- a/packages/contrail-appview/src/core/spaces/router.ts +++ b/packages/contrail-appview/src/core/spaces/router.ts @@ -9,9 +9,7 @@ import { HostedAdapter } from "./adapter"; import { buildVerifier, createBindingCredentialVerifier, - createCompositeBindingResolver, createEnrollmentBindingResolver, - createLocalBindingResolver, createLocalKeyResolver, createServiceAuthMiddleware, } from "@atmo-dev/contrail-base"; @@ -71,18 +69,11 @@ export function registerSpacesRoutes( ); if (spacesConfig.recordHost) { - // Default in-process verifier: enrollment is the canonical binding - // source; Local-binding is a fallback for spaces created but not yet - // enrolled. Caller overrides via `options.credentialVerifier` to - // accept external authorities. const credentialVerifier = options.credentialVerifier ?? (authorityConfig.signing ? createBindingCredentialVerifier({ - bindings: createCompositeBindingResolver([ - createEnrollmentBindingResolver({ recordHost: adapter }), - createLocalBindingResolver({ authorityDid: authorityConfig.serviceDid }), - ]), + bindings: createEnrollmentBindingResolver({ recordHost: adapter }), keys: createLocalKeyResolver({ authorityDid: authorityConfig.serviceDid, publicKey: authorityConfig.signing.publicKey, -- 2.51.2 From 428feead5f637e03666e60f9092d123bfaa87263 Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 10:05:44 -0400 Subject: [PATCH 17/19] fix(spaces): schema-validate PDS-record declarations MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The PDS binding resolver trusted body.value.authority on any record returned for the space's URI, with only a startsWith('did:') sniff. A malformed or attacker-shaped record at the same NSID could rebind authority — record content alone, no signature verification. Validate before trusting: - $type must equal the URI's type (the space-type lexicon embeds the declaration fields per tools.atmo.space.declaration's contract) - createdAt must be a string - authority must match a strict did:plc | did:web regex Fail closed on any deviation. Adds three negative tests covering each deviation path. --- packages/contrail-base/src/spaces/binding.ts | 12 +++- .../contrail/tests/spaces-binding.test.ts | 66 +++++++++++++++++++ 2 files changed, 75 insertions(+), 3 deletions(-) diff --git a/packages/contrail-base/src/spaces/binding.ts b/packages/contrail-base/src/spaces/binding.ts index a81b95f..2457f4d 100644 --- a/packages/contrail-base/src/spaces/binding.ts +++ b/packages/contrail-base/src/spaces/binding.ts @@ -141,10 +141,16 @@ export function createPdsBindingResolver(args: { } if (!res.ok) return null; const body = (await res.json().catch(() => null)) as - | { value?: { authority?: unknown } } + | { value?: { $type?: unknown; authority?: unknown; createdAt?: unknown } } | null; - const authority = body?.value?.authority; - return typeof authority === "string" && authority.startsWith("did:") ? authority : null; + const value = body?.value; + if (!value) return null; + if (value.$type !== parts.type) return null; + if (typeof value.createdAt !== "string") return null; + const authority = value.authority; + if (typeof authority !== "string") return null; + if (!/^did:(plc|web):[a-zA-Z0-9._:%-]+(#[a-zA-Z0-9._-]+)?$/.test(authority)) return null; + return authority; }, }; } diff --git a/packages/contrail/tests/spaces-binding.test.ts b/packages/contrail/tests/spaces-binding.test.ts index 427b6f2..d121df4 100644 --- a/packages/contrail/tests/spaces-binding.test.ts +++ b/packages/contrail/tests/spaces-binding.test.ts @@ -158,6 +158,72 @@ describe("BindingResolver — PDS record", () => { }); expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); }); + + it("returns null when the record's $type doesn't match the URI's type", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + value: { + $type: "com.attacker.fake.type", + authority: "did:web:authority.example", + createdAt: "2026-04-30T00:00:00Z", + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when createdAt is missing or non-string", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + value: { + $type: "com.example.event.space", + authority: "did:web:authority.example", + // createdAt deliberately omitted + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); + + it("returns null when authority isn't a well-formed DID", async () => { + const fetch = mockFetch( + new Map([ + [ + "https://pds.test/xrpc/com.atproto.repo.getRecord", + { + value: { + $type: "com.example.event.space", + authority: "did:fake!!://garbage", + createdAt: "2026-04-30T00:00:00Z", + }, + }, + ], + ]) + ); + const r = createPdsBindingResolver({ + resolver: mockResolver({ pdsEndpoint: "https://pds.test" }), + fetch, + }); + expect(await r.resolveAuthority(SPACE_URI)).toBeNull(); + }); }); describe("BindingResolver — DID-doc service entry", () => { -- 2.51.2 From dffbd36cb8f8855d990cd14532cbefbe6b09c46f Mon Sep 17 00:00:00 2001 From: Tom Scanlan <tom@openmeet.net> Date: Fri, 8 May 2026 10:43:54 -0400 Subject: [PATCH 18/19] docs(auth): document binding-layer trust assumptions Two trust assumptions in the binding layer cannot be closed at the Contrail layer alone: - DID-doc service-entry rebind: deployments using createDidDocBindingResolver inherit PLC's rotation-key authorization model. Any rotation key on the owner's account can rewrite the #atproto_space_authority service entry. - PDS app password held for managed communities: community.provision and community.adopt hold unscoped app passwords that can write any record on the user-owned PDS, including the binding declaration read by createPdsBindingResolver. Both are constraints of their respective binding sources and onboarding flows, not bugs. Document each so operators picking a binding strategy can size them against their own threat model, with pointers to the canonical issues for the upstream/protocol-level work that would close them: #38 (DID-doc), #39 (PDS app password). --- docs/05-auth.md | 22 ++++++++++++++++++++++ docs/10-deployment-shapes.md | 9 +++++++++ 2 files changed, 31 insertions(+) diff --git a/docs/05-auth.md b/docs/05-auth.md index 7cb7578..6237fb8 100644 --- a/docs/05-auth.md +++ b/docs/05-auth.md @@ -190,6 +190,28 @@ No auth needed for: Public requests skip all verification middleware — no JWT parsing, no DID-doc fetch. Fast path. +## Trust assumptions + +Two trust assumptions in the binding layer cannot be closed at the Contrail layer alone. They are listed here so operators picking a binding strategy or onboarding flow can size them against their own threat model. + +### DID-doc binding path + +The record host can resolve a space's authority from three sources (see [Spaces § Discovery](./06-spaces.md#discovery--binding-resolution)): + +- **Local enrollment** — the host's own `record_host_enrollments` table, written via `recordHost.enroll`. Owner-signed; the host has full control over what's stored. +- **PDS record** — read from the owner's PDS at the space URI. +- **DID-doc service entry** — read from `service[id="#atproto_space_authority"]` on the owner's DID doc. + +The DID-doc path inherits PLC's authorization model: any rotation key on the owner's account can submit an update op rewriting the service entry. There is no per-entry signature or "this entry can only be edited by key X" constraint at the PLC layer. + +If the integrity of the space-authority binding needs to exceed what any one of the owner's rotation keys can already do, configure the host to resolve via local enrollment instead, or wait for an upstream signed-binding mechanism. Tracked upstream as [flo-bit/contrail#38](https://github.com/flo-bit/contrail/issues/38). + +### Contrail-held PDS app password + +Deployments running `community.provision` or `community.adopt` necessarily hold an ATProto app password for the user-owned PDS account so Contrail can write on the community's behalf. ATProto app passwords are unscoped at the PDS layer — they can write any record to the repo, including the `tools.atmo.space.declaration` record (or its embedded equivalent on a space-type record) that drives binding decisions in `createPdsBindingResolver`. + +This is an intentional consequence of the managed-community model, not a bug. Operators should treat stored app passwords with the same care as rotation keys, and reach for scoped app passwords (when ATProto adds them) or signed binding records to tighten the model further. Tracked upstream as [flo-bit/contrail#39](https://github.com/flo-bit/contrail/issues/39). + ## How the pieces fit A typical flow for a third-party app acting as a user in a space: diff --git a/docs/10-deployment-shapes.md b/docs/10-deployment-shapes.md index 25464ed..7b8675b 100644 --- a/docs/10-deployment-shapes.md +++ b/docs/10-deployment-shapes.md @@ -192,6 +192,15 @@ A deployment can act as the authority for spaces it owns *and* a record host for When in doubt, all-in-one. Splitting is for when you have a real operational reason to separate the two — different teams running them, different latency profiles, different scaling targets, different governance. +## Known trust assumptions + +Two assumptions in the binding layer cannot be closed at the Contrail layer alone, and which one applies depends on the shape you pick: + +- Deployments wiring `createDidDocBindingResolver` inherit PLC's rotation-key authorization model for the `#atproto_space_authority` service entry. Tracked as [flo-bit/contrail#38](https://github.com/flo-bit/contrail/issues/38). +- Deployments running `community.provision` or `community.adopt` hold an unscoped ATProto app password for each provisioned PDS account. Tracked as [flo-bit/contrail#39](https://github.com/flo-bit/contrail/issues/39). + +See [Auth § Trust assumptions](./05-auth.md#trust-assumptions) for the constraints in detail and what binding source to prefer when those assumptions don't fit your threat model. + ## What's not here - **Authority migration** — moving a space's authority from DID A to DID B. The architecture supports it (re-enroll on the host with the new authority binding) but no helper API yet. -- 2.51.2 From 5a238661f2c8e73ac875be094c716e311ad670c5 Mon Sep 17 00:00:00 2001 From: Florian <45694132+flo-bit@users.noreply.github.com> Date: Thu, 4 Jun 2026 14:27:57 +0200 Subject: [PATCH 19/19] bump packages --- .changeset/config.json | 12 ++- .changeset/spaces-host-authority-split.md | 101 ------------------ packages/contrail-appview/CHANGELOG.md | 9 ++ packages/contrail-appview/package.json | 2 +- packages/contrail-authority/CHANGELOG.md | 7 ++ packages/contrail-authority/package.json | 2 +- packages/contrail-base/CHANGELOG.md | 3 + packages/contrail-base/package.json | 2 +- packages/contrail-community/CHANGELOG.md | 109 +++++++++++++++++++ packages/contrail-community/package.json | 2 +- packages/contrail-record-host/CHANGELOG.md | 7 ++ packages/contrail-record-host/package.json | 2 +- packages/contrail/CHANGELOG.md | 115 +++++++++++++++++++-- packages/contrail/package.json | 2 +- packages/lexicons/CHANGELOG.md | 10 +- packages/lexicons/package.json | 2 +- packages/sync/CHANGELOG.md | 104 ++++++++++++++++++- packages/sync/package.json | 2 +- 18 files changed, 371 insertions(+), 122 deletions(-) delete mode 100644 .changeset/spaces-host-authority-split.md create mode 100644 packages/contrail-appview/CHANGELOG.md create mode 100644 packages/contrail-authority/CHANGELOG.md create mode 100644 packages/contrail-base/CHANGELOG.md create mode 100644 packages/contrail-community/CHANGELOG.md create mode 100644 packages/contrail-record-host/CHANGELOG.md diff --git a/.changeset/config.json b/.changeset/config.json index 745d87f..070c0d5 100644 --- a/.changeset/config.json +++ b/.changeset/config.json @@ -2,8 +2,16 @@ "$schema": "https://unpkg.com/@changesets/config@3.1.1/schema.json", "changelog": "@changesets/cli/changelog", "commit": false, - "fixed": [], - "linked": [["@atmo-dev/contrail", "@atmo-dev/contrail-sync", "@atmo-dev/contrail-community"]], + "fixed": [[ + "@atmo-dev/contrail", + "@atmo-dev/contrail-community", + "@atmo-dev/contrail-sync", + "@atmo-dev/contrail-base", + "@atmo-dev/contrail-authority", + "@atmo-dev/contrail-record-host", + "@atmo-dev/contrail-appview" + ]], + "linked": [], "access": "public", "baseBranch": "main", "updateInternalDependencies": "patch", diff --git a/.changeset/spaces-host-authority-split.md b/.changeset/spaces-host-authority-split.md deleted file mode 100644 index e14e46c..0000000 --- a/.changeset/spaces-host-authority-split.md +++ /dev/null @@ -1,101 +0,0 @@ ---- -"@atmo-dev/contrail": minor -"@atmo-dev/contrail-community": minor -"@atmo-dev/contrail-sync": minor ---- - -Spaces refactor: split authority + record host into independently runnable -roles, add space credentials, extract community into its own package. - -**Breaking — config shape** - -`spaces` is no longer flat — split into `authority` and `recordHost`: - -```ts -// before -spaces: { - type: "com.example.event.space", - serviceDid: "did:web:example.com", - blobs: { adapter, maxSize }, -} - -// after -spaces: { - authority: { - type: "com.example.event.space", - serviceDid: "did:web:example.com", - signing: await generateAuthoritySigningKey(), - }, - recordHost: { - blobs: { adapter, maxSize }, - }, -} -``` - -**Breaking — community moved to its own package** - -Community has been extracted to `@atmo-dev/contrail-community`. Wire it via -`createCommunityIntegration`: - -```ts -import { Contrail, resolveConfig } from "@atmo-dev/contrail"; -import { createCommunityIntegration } from "@atmo-dev/contrail-community"; - -const resolved = resolveConfig(config); -const communityIntegration = createCommunityIntegration({ db, config: resolved }); -const contrail = new Contrail({ ...config, communityIntegration }); -``` - -The community config (`config.community`) stays the same; only the wiring -moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, -`reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of -`@atmo-dev/contrail`. - -**New — space credentials (`X-Space-Credential`)** - -The space authority issues short-lived ES256 JWTs (default 2h TTL) via -`<ns>.space.getCredential` and `refreshCredential`. The record host accepts -them on read/write paths in lieu of per-request service-auth JWTs. Skips -DID-doc fetches and member checks; the credential's signature is the proof. - -Generate a signing key once at deploy time: - -```ts -import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; -const signing = await generateAuthoritySigningKey(); -// Store the JWK; pass to spaces.authority.signing. -``` - -**New — binding resolution** - -Verifiers can resolve "which authority signs for this space?" from three -sources, in order: local enrollment table, PDS records at -`at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service -entry, owner-self fallback. Lets user-owned DIDs authorize a third-party -authority via a normal PDS write — no DID-doc surgery. - -**New — independent deployments + enrollment** - -The authority and record host can run as separate processes/operators. -A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) -register a space onto a host. In-process deployments auto-enroll on -`createSpace`; nothing changes for single-instance setups. - -See `docs/10-deployment-shapes.md` for all-in-one / authority-only / -host-only configurations and when to choose each. - -**Migration** - -For most deployments running spaces today, the migration is: - -1. Update the config: split `spaces.{type, serviceDid, blobs}` into - `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. -2. Generate and store an authority signing key - (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. -3. If using community: install `@atmo-dev/contrail-community`, build - `createCommunityIntegration({ db, config })`, pass via - `new Contrail({ communityIntegration })` (or `createApp({ community })`). - -Existing service-auth JWT clients keep working as a fallback path. -Migrate to space credentials when convenient — exchange a JWT for a -credential once via `getCredential`, then reuse it. diff --git a/packages/contrail-appview/CHANGELOG.md b/packages/contrail-appview/CHANGELOG.md new file mode 100644 index 0000000..cd18649 --- /dev/null +++ b/packages/contrail-appview/CHANGELOG.md @@ -0,0 +1,9 @@ +# @atmo-dev/contrail-appview + +## 0.7.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 +- @atmo-dev/contrail-authority@0.7.0 +- @atmo-dev/contrail-record-host@0.7.0 diff --git a/packages/contrail-appview/package.json b/packages/contrail-appview/package.json index c76172b..f2627dd 100644 --- a/packages/contrail-appview/package.json +++ b/packages/contrail-appview/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-appview", - "version": "0.6.0", + "version": "0.7.0", "description": "Public-records appview for contrail — jetstream ingestion, backfill, query layer, feeds, labels, profiles, per-collection XRPC routes.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-authority/CHANGELOG.md b/packages/contrail-authority/CHANGELOG.md new file mode 100644 index 0000000..de82f64 --- /dev/null +++ b/packages/contrail-authority/CHANGELOG.md @@ -0,0 +1,7 @@ +# @atmo-dev/contrail-authority + +## 0.7.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 diff --git a/packages/contrail-authority/package.json b/packages/contrail-authority/package.json index 07c0477..a088146 100644 --- a/packages/contrail-authority/package.json +++ b/packages/contrail-authority/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-authority", - "version": "0.6.0", + "version": "0.7.0", "description": "Default space-authority implementation for contrail — member list, invites, app policy, credential issuance. Contrail's binary-membership ACL flavor; for ladder-style access levels see @atmo-dev/contrail-community.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-base/CHANGELOG.md b/packages/contrail-base/CHANGELOG.md new file mode 100644 index 0000000..e6ed047 --- /dev/null +++ b/packages/contrail-base/CHANGELOG.md @@ -0,0 +1,3 @@ +# @atmo-dev/contrail-base + +## 0.7.0 diff --git a/packages/contrail-base/package.json b/packages/contrail-base/package.json index e72201a..5535411 100644 --- a/packages/contrail-base/package.json +++ b/packages/contrail-base/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-base", - "version": "0.6.0", + "version": "0.7.0", "description": "Shared infrastructure for the contrail family of packages — interfaces (SpaceAuthority, RecordHost, CommunityIntegration), credential primitives, binding resolvers, realtime infra, schema scaffolding. No routes, no tables of its own.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-community/CHANGELOG.md b/packages/contrail-community/CHANGELOG.md new file mode 100644 index 0000000..163f0e5 --- /dev/null +++ b/packages/contrail-community/CHANGELOG.md @@ -0,0 +1,109 @@ +# @atmo-dev/contrail-community + +## 0.7.0 + +### Minor Changes + +- 7e3145b: Spaces refactor: split authority + record host into independently runnable + roles, add space credentials, extract community into its own package. + + **Breaking — config shape** + + `spaces` is no longer flat — split into `authority` and `recordHost`: + + ```ts + // before + spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, + } + + // after + spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, + } + ``` + + **Breaking — community moved to its own package** + + Community has been extracted to `@atmo-dev/contrail-community`. Wire it via + `createCommunityIntegration`: + + ```ts + import { Contrail, resolveConfig } from "@atmo-dev/contrail"; + import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ + db, + config: resolved, + }); + const contrail = new Contrail({ ...config, communityIntegration }); + ``` + + The community config (`config.community`) stays the same; only the wiring + moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, + `reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of + `@atmo-dev/contrail`. + + **New — space credentials (`X-Space-Credential`)** + + The space authority issues short-lived ES256 JWTs (default 2h TTL) via + `<ns>.space.getCredential` and `refreshCredential`. The record host accepts + them on read/write paths in lieu of per-request service-auth JWTs. Skips + DID-doc fetches and member checks; the credential's signature is the proof. + + Generate a signing key once at deploy time: + + ```ts + import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + const signing = await generateAuthoritySigningKey(); + // Store the JWK; pass to spaces.authority.signing. + ``` + + **New — binding resolution** + + Verifiers can resolve "which authority signs for this space?" from three + sources, in order: local enrollment table, PDS records at + `at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service + entry, owner-self fallback. Lets user-owned DIDs authorize a third-party + authority via a normal PDS write — no DID-doc surgery. + + **New — independent deployments + enrollment** + + The authority and record host can run as separate processes/operators. + A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) + register a space onto a host. In-process deployments auto-enroll on + `createSpace`; nothing changes for single-instance setups. + + See `docs/10-deployment-shapes.md` for all-in-one / authority-only / + host-only configurations and when to choose each. + + **Migration** + + For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. + 2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. + 3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + + Existing service-auth JWT clients keep working as a fallback path. + Migrate to space credentials when convenient — exchange a JWT for a + credential once via `getCredential`, then reuse it. + +### Patch Changes + +- Updated dependencies [7e3145b] + - @atmo-dev/contrail@0.7.0 + - @atmo-dev/contrail-base@0.7.0 diff --git a/packages/contrail-community/package.json b/packages/contrail-community/package.json index c6abcb2..ac7bc8a 100644 --- a/packages/contrail-community/package.json +++ b/packages/contrail-community/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-community", - "version": "0.4.2", + "version": "0.7.0", "description": "Community module for contrail — community-owned spaces with tiered access levels (member → moderator → admin), invite tokens, DID provisioning, and the access-level reconciler that keeps spaces_members in sync.", "type": "module", "sideEffects": false, diff --git a/packages/contrail-record-host/CHANGELOG.md b/packages/contrail-record-host/CHANGELOG.md new file mode 100644 index 0000000..bbc13ac --- /dev/null +++ b/packages/contrail-record-host/CHANGELOG.md @@ -0,0 +1,7 @@ +# @atmo-dev/contrail-record-host + +## 0.7.0 + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 diff --git a/packages/contrail-record-host/package.json b/packages/contrail-record-host/package.json index 2182f37..b0f50f9 100644 --- a/packages/contrail-record-host/package.json +++ b/packages/contrail-record-host/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-record-host", - "version": "0.6.0", + "version": "0.7.0", "description": "Default record-host implementation for contrail — stores records and blobs for permissioned spaces, enforces local enrollment as the host's consent layer.", "type": "module", "sideEffects": false, diff --git a/packages/contrail/CHANGELOG.md b/packages/contrail/CHANGELOG.md index 63e68f6..d404678 100644 --- a/packages/contrail/CHANGELOG.md +++ b/packages/contrail/CHANGELOG.md @@ -1,11 +1,119 @@ # @atmo-dev/contrail +## 0.7.0 + +### Minor Changes + +- 7e3145b: Spaces refactor: split authority + record host into independently runnable + roles, add space credentials, extract community into its own package. + + **Breaking — config shape** + + `spaces` is no longer flat — split into `authority` and `recordHost`: + + ```ts + // before + spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, + } + + // after + spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, + } + ``` + + **Breaking — community moved to its own package** + + Community has been extracted to `@atmo-dev/contrail-community`. Wire it via + `createCommunityIntegration`: + + ```ts + import { Contrail, resolveConfig } from "@atmo-dev/contrail"; + import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ + db, + config: resolved, + }); + const contrail = new Contrail({ ...config, communityIntegration }); + ``` + + The community config (`config.community`) stays the same; only the wiring + moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, + `reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of + `@atmo-dev/contrail`. + + **New — space credentials (`X-Space-Credential`)** + + The space authority issues short-lived ES256 JWTs (default 2h TTL) via + `<ns>.space.getCredential` and `refreshCredential`. The record host accepts + them on read/write paths in lieu of per-request service-auth JWTs. Skips + DID-doc fetches and member checks; the credential's signature is the proof. + + Generate a signing key once at deploy time: + + ```ts + import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + const signing = await generateAuthoritySigningKey(); + // Store the JWK; pass to spaces.authority.signing. + ``` + + **New — binding resolution** + + Verifiers can resolve "which authority signs for this space?" from three + sources, in order: local enrollment table, PDS records at + `at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service + entry, owner-self fallback. Lets user-owned DIDs authorize a third-party + authority via a normal PDS write — no DID-doc surgery. + + **New — independent deployments + enrollment** + + The authority and record host can run as separate processes/operators. + A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) + register a space onto a host. In-process deployments auto-enroll on + `createSpace`; nothing changes for single-instance setups. + + See `docs/10-deployment-shapes.md` for all-in-one / authority-only / + host-only configurations and when to choose each. + + **Migration** + + For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. + 2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. + 3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + + Existing service-auth JWT clients keep working as a fallback path. + Migrate to space credentials when convenient — exchange a JWT for a + credential once via `getCredential`, then reuse it. + +### Patch Changes + +- @atmo-dev/contrail-base@0.7.0 +- @atmo-dev/contrail-authority@0.7.0 +- @atmo-dev/contrail-record-host@0.7.0 +- @atmo-dev/contrail-appview@0.7.0 + ## 0.6.0 ### Minor Changes - af24714: Add per-collection `recordFilter` and apply Jetstream `#identity` handle changes during ingest. - - `CollectionConfig.recordFilter?: (record) => boolean` runs against each create/update during ingest; returning false drops the record before it reaches the DB. Useful for narrowing high-volume collections to just the records you care about (e.g. only `app.bsky.feed.post` records mentioning a particular URL). Deletes are not filtered, so they still tear down any record the filter previously let through. Throws are caught, logged, and treated as drops. - Jetstream `#identity` events (handle changes) now flow through to the `identities` table via a new `applyIdentityEvent` helper. UPDATE-only — unknown DIDs are no-ops so we don't materialize partial rows lacking PDS. @@ -71,7 +179,6 @@ ``` what changed: - - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -98,7 +205,6 @@ ``` changes: - - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. @@ -138,7 +244,6 @@ - ad3a61d: add `contrail dev` — local dev wrapper for cloudflare workers deployments. replaces `wrangler dev --test-scheduled` + a separate cron-trigger script with one command. on start it: - 1. connects to your local D1 via wrangler's `getPlatformProxy`, inspects state 2. prompts to run `backfillAll` if no completed backfills exist yet 3. prompts to run `refresh` if the ingest cursor is older than 60 minutes (configurable with `--stale-after`) @@ -166,7 +271,6 @@ options: `binding` (D1 binding name, default `"DB"`), `lexicons` (see below), `onInit` (one-shot app-specific setup). **`/xrpc/<ns>.lexicons` endpoint + `contrail-lex pull-service`** lets consumer apps typegen against a deployed contrail over HTTP, no PDS or DNS required: - - `contrail-lex generate` now emits a barrel `lexicons/generated/index.ts` that imports every lexicon the deployment speaks: generated + pulled + custom. The pulled lexicons are needed so consumer typegen can resolve `$ref`s out of the generated schemas. - Pass `{ lexicons }` to `createWorker` (or `createHandler(contrail, { lexicons })`) and the service exposes them at `GET /xrpc/<namespace>.lexicons`. - From a consumer app: @@ -184,7 +288,6 @@ unlike `backfillAll`, it ignores the `backfills` state table and sweeps fresh. useful after jetstream outages or after leaving a dev deployment idle for days. each record in each configured collection is classified as: - - **missing** — PDS has it, DB doesn't - **stale update** — DB has it with a different CID, _and_ the DB row was written before the ignore window (default 60s, configurable) - **in sync** — same CID, or DB row is within the ignore window diff --git a/packages/contrail/package.json b/packages/contrail/package.json index 0eec411..0219561 100644 --- a/packages/contrail/package.json +++ b/packages/contrail/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail", - "version": "0.6.0", + "version": "0.7.0", "description": "Index AT Protocol records with typed XRPC endpoints. Cloudflare Workers + D1, SvelteKit, Node.js.", "type": "module", "sideEffects": false, diff --git a/packages/lexicons/CHANGELOG.md b/packages/lexicons/CHANGELOG.md index d5f674f..ab5abd0 100644 --- a/packages/lexicons/CHANGELOG.md +++ b/packages/lexicons/CHANGELOG.md @@ -1,5 +1,12 @@ # @atmo-dev/contrail-lexicons +## 0.4.7 + +### Patch Changes + +- Updated dependencies [7e3145b] + - @atmo-dev/contrail@0.7.0 + ## 0.4.6 ### Patch Changes @@ -53,7 +60,6 @@ ``` what changed: - - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -86,7 +92,6 @@ ``` changes: - - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. @@ -110,7 +115,6 @@ options: `binding` (D1 binding name, default `"DB"`), `lexicons` (see below), `onInit` (one-shot app-specific setup). **`/xrpc/<ns>.lexicons` endpoint + `contrail-lex pull-service`** lets consumer apps typegen against a deployed contrail over HTTP, no PDS or DNS required: - - `contrail-lex generate` now emits a barrel `lexicons/generated/index.ts` that imports every lexicon the deployment speaks: generated + pulled + custom. The pulled lexicons are needed so consumer typegen can resolve `$ref`s out of the generated schemas. - Pass `{ lexicons }` to `createWorker` (or `createHandler(contrail, { lexicons })`) and the service exposes them at `GET /xrpc/<namespace>.lexicons`. - From a consumer app: diff --git a/packages/lexicons/package.json b/packages/lexicons/package.json index 9cc3ea4..066e06c 100644 --- a/packages/lexicons/package.json +++ b/packages/lexicons/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-lexicons", - "version": "0.4.6", + "version": "0.4.7", "description": "Generate atproto lexicon JSON (and optionally TypeScript types via @atcute/lex-cli) from a Contrail config.", "type": "module", "files": [ diff --git a/packages/sync/CHANGELOG.md b/packages/sync/CHANGELOG.md index f570219..aa4b041 100644 --- a/packages/sync/CHANGELOG.md +++ b/packages/sync/CHANGELOG.md @@ -1,5 +1,107 @@ # @atmo-dev/contrail-sync +## 0.7.0 + +### Minor Changes + +- 7e3145b: Spaces refactor: split authority + record host into independently runnable + roles, add space credentials, extract community into its own package. + + **Breaking — config shape** + + `spaces` is no longer flat — split into `authority` and `recordHost`: + + ```ts + // before + spaces: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + blobs: { adapter, maxSize }, + } + + // after + spaces: { + authority: { + type: "com.example.event.space", + serviceDid: "did:web:example.com", + signing: await generateAuthoritySigningKey(), + }, + recordHost: { + blobs: { adapter, maxSize }, + }, + } + ``` + + **Breaking — community moved to its own package** + + Community has been extracted to `@atmo-dev/contrail-community`. Wire it via + `createCommunityIntegration`: + + ```ts + import { Contrail, resolveConfig } from "@atmo-dev/contrail"; + import { createCommunityIntegration } from "@atmo-dev/contrail-community"; + + const resolved = resolveConfig(config); + const communityIntegration = createCommunityIntegration({ + db, + config: resolved, + }); + const contrail = new Contrail({ ...config, communityIntegration }); + ``` + + The community config (`config.community`) stays the same; only the wiring + moves. Imports of `CommunityAdapter`, `registerCommunityRoutes`, + `reconcile`, etc. now come from `@atmo-dev/contrail-community` instead of + `@atmo-dev/contrail`. + + **New — space credentials (`X-Space-Credential`)** + + The space authority issues short-lived ES256 JWTs (default 2h TTL) via + `<ns>.space.getCredential` and `refreshCredential`. The record host accepts + them on read/write paths in lieu of per-request service-auth JWTs. Skips + DID-doc fetches and member checks; the credential's signature is the proof. + + Generate a signing key once at deploy time: + + ```ts + import { generateAuthoritySigningKey } from "@atmo-dev/contrail"; + const signing = await generateAuthoritySigningKey(); + // Store the JWK; pass to spaces.authority.signing. + ``` + + **New — binding resolution** + + Verifiers can resolve "which authority signs for this space?" from three + sources, in order: local enrollment table, PDS records at + `at://<owner>/<type>/<key>`, DID-doc `#atproto_space_authority` service + entry, owner-self fallback. Lets user-owned DIDs authorize a third-party + authority via a normal PDS write — no DID-doc surgery. + + **New — independent deployments + enrollment** + + The authority and record host can run as separate processes/operators. + A new `<ns>.recordHost.enroll` endpoint lets owners (or authorities) + register a space onto a host. In-process deployments auto-enroll on + `createSpace`; nothing changes for single-instance setups. + + See `docs/10-deployment-shapes.md` for all-in-one / authority-only / + host-only configurations and when to choose each. + + **Migration** + + For most deployments running spaces today, the migration is: + 1. Update the config: split `spaces.{type, serviceDid, blobs}` into + `spaces.authority.{type, serviceDid}` and `spaces.recordHost.{blobs}`. + 2. Generate and store an authority signing key + (`generateAuthoritySigningKey()`); add to `spaces.authority.signing`. + 3. If using community: install `@atmo-dev/contrail-community`, build + `createCommunityIntegration({ db, config })`, pass via + `new Contrail({ communityIntegration })` (or `createApp({ community })`). + + Existing service-auth JWT clients keep working as a fallback path. + Migrate to space credentials when convenient — exchange a JWT for a + credential once via `getCredential`, then reuse it. + ## 0.4.0 ### Minor Changes @@ -26,7 +128,6 @@ ``` what changed: - - `buildSpaceUri` / `parseSpaceUri` (`@atmo-dev/contrail`) emit / accept `ats://`. anything else returns `null` from `parseSpaceUri`. - generated lexicons no longer claim `format: "at-uri"` on `spaceUri` params, on the `space` record-output field, or on `spaceView.uri` — they're plain `string`. (atproto's `at-uri` format would reject `ats://`.) regenerate committed `lexicons/generated/*` with `contrail-lex generate`; downstream `lex-cli generate` then emits `v.string()` instead of `v.resourceUriString()` for those fields. - realtime topics are unchanged in shape (`space:<uri>`), but `<uri>` is now an `ats://` URI. @@ -53,7 +154,6 @@ ``` changes: - - `#record` def now requires `["uri", "cid", "value"]` (matches atproto's standard `com.atproto.repo.listRecords#record`). `did`/`collection`/`rkey`/`time_us` remain in the response but are optional. - `getRecord` top-level output requires `["uri", "value"]` (matches atproto's `com.atproto.repo.getRecord`). - profile entries in `?profiles=true` responses use `value` instead of `record` for the profile record body. diff --git a/packages/sync/package.json b/packages/sync/package.json index ef9e485..9bd77e4 100644 --- a/packages/sync/package.json +++ b/packages/sync/package.json @@ -1,6 +1,6 @@ { "name": "@atmo-dev/contrail-sync", - "version": "0.4.0", + "version": "0.7.0", "description": "Client-side reactive watch-store over contrail's watchRecords endpoints. SSE + WebSocket transports, optimistic updates, optional IndexedDB cache.", "type": "module", "sideEffects": false,