diff --git a/packages/sources/check.ts b/packages/sources/check.ts new file mode 100644 index 0000000..3945921 --- /dev/null +++ b/packages/sources/check.ts @@ -0,0 +1,68 @@ +/** + * Resolve REAL sources against live infrastructure — a smoke test, not a unit test (network-bound). + * pnpm --filter @blento/sources exec tsx check.ts [did] + * + * Checks: + * 1. #atproto against the bsky appview (app.bsky.feed.getAuthorFeed, $self → did) + * 2. cursor pagination (walk page 1 → page 2 via nextCursor) + * 3. #atproto repo path via the actor PDS (com.atproto.repo.listRecords, did→PDS resolution) + * 4. cache dedupe (second resolve serves from the in-memory adapter) + */ +import { resolve, MemoryCacheAdapter, type Source, type SourceContext } from './src/index.js'; + +const did = process.argv[2] ?? 'did:plc:s42iw2fbfmgsgh7hdtvvoaao'; +const ctx: SourceContext = { self: did }; + +function fail(msg: string): never { + console.error('CHECK FAIL:', msg); + process.exit(1); +} + +// 1 + 2: author feed + pagination +const feedSource: Source = { + $type: 'app.blento.source#atproto', + method: 'app.bsky.feed.getAuthorFeed', + params: { actor: '$self', filter: 'posts_no_replies', limit: 2 } +}; +const page1 = await resolve(feedSource, ctx); +const feed1 = (page1.data as { feed?: unknown[] }).feed ?? []; +if (!Array.isArray(feed1)) fail('getAuthorFeed did not return a feed array'); + +let page2Len = 0; +if (page1.nextCursor) { + const page2 = await resolve(feedSource, ctx, { cursor: page1.nextCursor }); + page2Len = ((page2.data as { feed?: unknown[] }).feed ?? []).length; +} + +// 3: repo listRecords via the actor's PDS (repo.* → did→PDS resolution) +const listSource: Source = { + $type: 'app.blento.source#atproto', + method: 'com.atproto.repo.listRecords', + params: { repo: '$self', collection: 'app.bsky.feed.post', limit: 1 } +}; +const list = await resolve(listSource, ctx); +const records = (list.data as { records?: unknown[] }).records ?? []; +if (!Array.isArray(records)) fail('listRecords did not return a records array'); + +// 4: cache dedupe — second call must not change the result and must add exactly one entry +const cache = new MemoryCacheAdapter(); +const c1 = await resolve(feedSource, ctx, { cache }); +const c2 = await resolve(feedSource, ctx, { cache }); +if (JSON.stringify(c1) !== JSON.stringify(c2)) fail('cached result differs from fresh result'); +if (cache.size !== 1) fail(`expected 1 cache entry, got ${cache.size}`); + +console.log( + JSON.stringify( + { + did, + feedPage1: feed1.length, + nextCursor: page1.nextCursor ?? null, + feedPage2: page2Len, + listRecords: records.length, + cacheEntries: cache.size + }, + null, + 2 + ) +); +console.log('OK: all live source checks passed'); diff --git a/packages/sources/package.json b/packages/sources/package.json new file mode 100644 index 0000000..788dfce --- /dev/null +++ b/packages/sources/package.json @@ -0,0 +1,21 @@ +{ + "name": "@blento/sources", + "version": "0.0.0", + "private": true, + "type": "module", + "exports": { + ".": "./src/index.ts" + }, + "scripts": { + "check": "tsc --noEmit", + "test": "vitest run" + }, + "dependencies": { + "@blento/schema": "workspace:*" + }, + "devDependencies": { + "tsx": "^4.21.0", + "typescript": "^5.9.3", + "vitest": "^4.1.4" + } +} diff --git a/packages/sources/src/atproto.test.ts b/packages/sources/src/atproto.test.ts new file mode 100644 index 0000000..04ef329 --- /dev/null +++ b/packages/sources/src/atproto.test.ts @@ -0,0 +1,103 @@ +import { describe, it, expect } from 'vitest'; +import { resolve } from './resolve.js'; +import type { Source, SourceContext } from './types.js'; +import { fakeFetch } from './testutil.js'; + +const SELF = 'did:plc:s42iw2fbfmgsgh7hdtvvoaao'; + +describe('#atproto', () => { + it('substitutes $self and hits the default appview for a bsky method', async () => { + const ff = fakeFetch([ + { match: 'app.bsky.feed.getAuthorFeed', body: { feed: [], cursor: 'c1' } } + ]); + const source: Source = { + $type: 'app.blento.source#atproto', + method: 'app.bsky.feed.getAuthorFeed', + params: { actor: '$self', filter: 'posts_no_replies', limit: 2 } + }; + const ctx: SourceContext = { self: SELF, fetchImpl: ff.fetch }; + + const res = await resolve(source, ctx); + + expect(res).toEqual({ data: { feed: [], cursor: 'c1' }, nextCursor: 'c1' }); + const url = ff.calls[0].url; + expect(url).toContain('https://public.api.bsky.app/xrpc/app.bsky.feed.getAuthorFeed'); + expect(url).toContain(`actor=${encodeURIComponent(SELF)}`); + expect(url).toContain('filter=posts_no_replies'); + expect(url).toContain('limit=2'); + }); + + it('substitutes $self inside a longer string (at-uri)', async () => { + const ff = fakeFetch([{ match: 'com.atproto.repo.listRecords', body: { records: [] } }]); + // listRecords is repo.* → PDS path; needs did→PDS resolution first + const withPds = fakeFetch([ + { + match: 'plc.directory', + body: { service: [{ id: '#atproto_pds', serviceEndpoint: 'https://pds.example' }] } + }, + { match: 'com.atproto.repo.listRecords', body: { records: [], cursor: undefined } } + ]); + const source: Source = { + $type: 'app.blento.source#atproto', + method: 'com.atproto.repo.listRecords', + params: { repo: '$self', collection: 'app.blento.card', uri: 'at://$self/x' } + }; + await resolve(source, { self: SELF, fetchImpl: withPds.fetch }); + const call = withPds.calls.find((c) => c.url.includes('listRecords'))!; + expect(call.url).toContain('https://pds.example/xrpc/com.atproto.repo.listRecords'); + expect(call.url).toContain(`repo=${encodeURIComponent(SELF)}`); + expect(call.url).toContain(encodeURIComponent(`at://${SELF}/x`)); + expect(ff.calls.length).toBe(0); + }); + + it('routes repo.* / sync.* to the actor PDS resolved via plc.directory', async () => { + const ff = fakeFetch([ + { + match: 'plc.directory', + body: { service: [{ id: '#atproto_pds', serviceEndpoint: 'https://pds.host' }] } + }, + { match: 'com.atproto.sync.getRepo', body: { ok: true } } + ]); + const source: Source = { + $type: 'app.blento.source#atproto', + method: 'com.atproto.sync.getRepo', + params: { did: '$self' } + }; + await resolve(source, { self: SELF, fetchImpl: ff.fetch }); + expect(ff.calls[0].url).toBe(`https://plc.directory/${SELF}`); + expect(ff.calls[1].url).toContain('https://pds.host/xrpc/com.atproto.sync.getRepo'); + }); + + it('uses an explicit service only when allow-listed', async () => { + const ff = fakeFetch([{ match: 'my.appview', body: { data: 1 } }]); + const source: Source = { + $type: 'app.blento.source#atproto', + method: 'app.example.getThing', + service: 'https://my.appview', + params: {} + }; + // not allow-listed → refuse + await expect(resolve(source, { self: SELF, fetchImpl: ff.fetch })).rejects.toThrow( + /not allow-listed/ + ); + // allow-listed via ctx.appviews → ok + const res = await resolve(source, { + self: SELF, + fetchImpl: ff.fetch, + appviews: ['https://my.appview'] + }); + expect(res.data).toEqual({ data: 1 }); + }); + + it('threads cursor in and reports nextCursor out', async () => { + const ff = fakeFetch([{ match: 'getAuthorFeed', body: { feed: [1], cursor: 'next' } }]); + const source: Source = { + $type: 'app.blento.source#atproto', + method: 'app.bsky.feed.getAuthorFeed', + params: { actor: '$self' } + }; + const res = await resolve(source, { self: SELF, fetchImpl: ff.fetch }, { cursor: 'page2' }); + expect(ff.calls[0].url).toContain('cursor=page2'); + expect(res.nextCursor).toBe('next'); + }); +}); diff --git a/packages/sources/src/atproto.ts b/packages/sources/src/atproto.ts new file mode 100644 index 0000000..83ed5f7 --- /dev/null +++ b/packages/sources/src/atproto.ts @@ -0,0 +1,92 @@ +/** + * The `#atproto` kind: any XRPC query. `method` (NSID) + `params` + optional `service`. + * + * Server resolution policy: + * - `com.atproto.repo.*` / `com.atproto.sync.*` → the actor's PDS (did→PDS via plc.directory). + * - everything else → the named `service` (must be allow-listed) else a known-appview registry + * (default: bsky public appview). An unlisted / unknown `service` is refused (gated). + */ +import type { ResolveResult, Source, SourceContext } from './types.js'; +import { resolvePds } from './identity.js'; + +/** Built-in default appview. Doubles as the default allow-listed service. */ +export const DEFAULT_APPVIEW = 'https://public.api.bsky.app'; + +type AtprotoSource = Extract; + +/** Substitute context vars (`$self` today) into a param value, recursing through arrays/objects. */ +function substitute(value: unknown, ctx: SourceContext): unknown { + if (typeof value === 'string') { + // exact match is the common case (`actor: '$self'`); also interpolate inside longer strings + if (value === '$self') return ctx.self; + return value.includes('$self') ? value.split('$self').join(ctx.self) : value; + } + if (Array.isArray(value)) return value.map((v) => substitute(v, ctx)); + if (value && typeof value === 'object') { + const out: Record = {}; + for (const [k, v] of Object.entries(value as Record)) + out[k] = substitute(v, ctx); + return out; + } + return value; +} + +/** + * Return the source with all context vars substituted into its params. Done ONCE up front so the + * cache key and the actual fetch see the identical resolved spec. + */ +export function substituteAtproto(source: AtprotoSource, ctx: SourceContext): AtprotoSource { + return { ...source, params: substitute(source.params ?? {}, ctx) as Record }; +} + +const isRepoOrSync = (method: string) => + method.startsWith('com.atproto.repo.') || method.startsWith('com.atproto.sync.'); + +/** Pick the base URL for an appview query, enforcing the service allowlist. */ +function resolveAppview(service: string | undefined, ctx: SourceContext): string { + const def = ctx.defaultAppview ?? DEFAULT_APPVIEW; + if (!service) return def; + const allowed = new Set([def, DEFAULT_APPVIEW, ...(ctx.appviews ?? [])]); + if (!allowed.has(service)) { + throw new Error(`atproto source refused: service "${service}" is not allow-listed`); + } + return service; +} + +/** + * Resolve an ALREADY-substituted `#atproto` source into `{ data, nextCursor }`. + * (Call `substituteAtproto` first — `resolve()` does this so the cache key matches the fetch.) + */ +export async function resolveAtproto( + source: AtprotoSource, + ctx: SourceContext, + cursor: string | undefined +): Promise { + const fetchImpl = ctx.fetchImpl ?? fetch; + + const params: Record = { ...(source.params ?? {}) }; + if (cursor !== undefined) params.cursor = cursor; + + const base = isRepoOrSync(source.method) + ? await resolvePds(ctx.self, resolveAppview(source.service, ctx), fetchImpl) + : resolveAppview(source.service, ctx); + + const url = new URL(`/xrpc/${source.method}`, base); + for (const [k, v] of Object.entries(params)) { + if (v === undefined || v === null) continue; + if (Array.isArray(v)) for (const item of v) url.searchParams.append(k, String(item)); + else url.searchParams.set(k, String(v)); + } + + const res = await fetchImpl(url); + if (!res.ok) { + throw new Error(`atproto ${source.method} failed: ${res.status} ${res.statusText}`); + } + const data = (await res.json()) as unknown; + const nextCursor = + data && typeof data === 'object' && typeof (data as { cursor?: unknown }).cursor === 'string' + ? (data as { cursor: string }).cursor + : undefined; + + return { data, nextCursor }; +} diff --git a/packages/sources/src/cache.test.ts b/packages/sources/src/cache.test.ts new file mode 100644 index 0000000..84238cc --- /dev/null +++ b/packages/sources/src/cache.test.ts @@ -0,0 +1,62 @@ +import { describe, it, expect } from 'vitest'; +import { cacheKey, MemoryCacheAdapter } from './cache.js'; +import { resolve } from './resolve.js'; +import type { Source, SourceContext } from './types.js'; +import { fakeFetch } from './testutil.js'; + +describe('cache key', () => { + it('is stable regardless of param key order', () => { + const a: Source = { + $type: 'app.blento.source#atproto', + method: 'm', + params: { a: 1, b: 2 } + }; + const b: Source = { + $type: 'app.blento.source#atproto', + method: 'm', + params: { b: 2, a: 1 } + }; + expect(cacheKey(a)).toBe(cacheKey(b)); + }); + + it('differs by cursor and by spec', () => { + const s: Source = { $type: 'app.blento.source#atproto', method: 'm', params: { x: 1 } }; + const s2: Source = { $type: 'app.blento.source#atproto', method: 'm', params: { x: 2 } }; + expect(cacheKey(s, 'c1')).not.toBe(cacheKey(s, 'c2')); + expect(cacheKey(s)).not.toBe(cacheKey(s2)); + }); +}); + +describe('resolve caching', () => { + it('serves the second call from cache (no second fetch)', async () => { + const ff = fakeFetch([{ match: 'getAuthorFeed', body: { feed: [1] } }]); + const source: Source = { + $type: 'app.blento.source#atproto', + method: 'app.bsky.feed.getAuthorFeed', + params: { actor: '$self' } + }; + const ctx: SourceContext = { self: 'did:plc:aaa', fetchImpl: ff.fetch }; + const cache = new MemoryCacheAdapter(); + + const r1 = await resolve(source, ctx, { cache }); + const r2 = await resolve(source, ctx, { cache }); + + expect(r1).toEqual(r2); + expect(ff.calls.length).toBe(1); + expect(cache.size).toBe(1); + }); + + it('keys by resolved spec: different $self → different entries', async () => { + const ff = fakeFetch([{ match: 'getAuthorFeed', body: { feed: [] } }]); + const source: Source = { + $type: 'app.blento.source#atproto', + method: 'app.bsky.feed.getAuthorFeed', + params: { actor: '$self' } + }; + const cache = new MemoryCacheAdapter(); + await resolve(source, { self: 'did:plc:aaa', fetchImpl: ff.fetch }, { cache }); + await resolve(source, { self: 'did:plc:bbb', fetchImpl: ff.fetch }, { cache }); + expect(cache.size).toBe(2); + expect(ff.calls.length).toBe(2); + }); +}); diff --git a/packages/sources/src/cache.ts b/packages/sources/src/cache.ts new file mode 100644 index 0000000..2fe8aa5 --- /dev/null +++ b/packages/sources/src/cache.ts @@ -0,0 +1,46 @@ +/** + * Cache adapter + stable cache-key derivation. + * + * The key is computed from the RESOLVED source spec (after `$self` substitution) plus the cursor, so + * two actors — or two cursors — never collide, and two byte-identical fetches always share a key. + */ +import type { CacheAdapter, Source } from './types.js'; + +/** Deterministic JSON: object keys sorted recursively so key order can't perturb the hash. */ +function canonical(value: unknown): string { + if (value === null || typeof value !== 'object') return JSON.stringify(value) ?? 'null'; + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + const obj = value as Record; + const parts = Object.keys(obj) + .sort() + .filter((k) => obj[k] !== undefined) + .map((k) => `${JSON.stringify(k)}:${canonical(obj[k])}`); + return `{${parts.join(',')}}`; +} + +/** + * A stable, inspectable cache key for a resolved source (+ cursor). `#ref` has no fetch of its own, + * so it never produces a key — callers alias it to the owner's result instead. + */ +export function cacheKey(source: Source, cursor?: string): string { + const spec = canonical(source); + return cursor ? `sources:${spec}:@${cursor}` : `sources:${spec}`; +} + +/** Trivial in-memory cache for tests and single-process consumers. TTL is ignored (no eviction). */ +export class MemoryCacheAdapter implements CacheAdapter { + private store = new Map(); + + get(key: string): string | null { + return this.store.get(key) ?? null; + } + + set(key: string, value: string): void { + this.store.set(key, value); + } + + /** Test/debug helper: number of entries currently held. */ + get size(): number { + return this.store.size; + } +} diff --git a/packages/sources/src/http.test.ts b/packages/sources/src/http.test.ts new file mode 100644 index 0000000..b7270f2 --- /dev/null +++ b/packages/sources/src/http.test.ts @@ -0,0 +1,33 @@ +import { describe, it, expect } from 'vitest'; +import { resolve } from './resolve.js'; +import type { Source, SourceContext } from './types.js'; +import { fakeFetch } from './testutil.js'; + +const httpSource = (url: string): Source => ({ $type: 'app.blento.source#http', url }); + +describe('#http', () => { + it('refuses when the host is not allow-listed (default: empty)', async () => { + const ff = fakeFetch([{ match: 'example.com', body: {} }]); + const ctx: SourceContext = { self: 'x', fetchImpl: ff.fetch }; + await expect(resolve(httpSource('https://example.com/data.json'), ctx)).rejects.toThrow( + /not allow-listed/ + ); + expect(ff.calls.length).toBe(0); + }); + + it('fetches when the host is allow-listed and parses JSON', async () => { + const ff = fakeFetch([{ match: 'api.example.com', body: { hello: 'world' } }]); + const ctx: SourceContext = { + self: 'x', + fetchImpl: ff.fetch, + httpAllowlist: ['api.example.com'] + }; + const res = await resolve(httpSource('https://api.example.com/data.json'), ctx); + expect(res.data).toEqual({ hello: 'world' }); + }); + + it('refuses non-http(s) protocols', async () => { + const ctx: SourceContext = { self: 'x', httpAllowlist: ['x'] }; + await expect(resolve(httpSource('file:///etc/passwd'), ctx)).rejects.toThrow(/protocol/); + }); +}); diff --git a/packages/sources/src/http.ts b/packages/sources/src/http.ts new file mode 100644 index 0000000..19225ed --- /dev/null +++ b/packages/sources/src/http.ts @@ -0,0 +1,38 @@ +/** + * The `#http` kind: a server-side GET of a fixed URL. Guarded by a host allowlist (empty ⇒ refuse), + * because arbitrary server-side fetch is the primary risk surface. + */ +import type { ResolveResult, Source, SourceContext } from './types.js'; + +type HttpSource = Extract; + +function assertHostAllowed(url: string, allowlist: string[] | undefined): URL { + let parsed: URL; + try { + parsed = new URL(url); + } catch { + throw new Error(`http source refused: invalid url "${url}"`); + } + if (parsed.protocol !== 'http:' && parsed.protocol !== 'https:') { + throw new Error(`http source refused: unsupported protocol "${parsed.protocol}"`); + } + const host = parsed.hostname.toLowerCase(); + const allowed = (allowlist ?? []).some((h) => h.toLowerCase() === host); + if (!allowed) { + throw new Error(`http source refused: host "${host}" is not allow-listed`); + } + return parsed; +} + +/** Resolve an `#http` source. Returns parsed JSON when the response is JSON, else the raw text. */ +export async function resolveHttp(source: HttpSource, ctx: SourceContext): Promise { + const url = assertHostAllowed(source.url, ctx.httpAllowlist); + const fetchImpl = ctx.fetchImpl ?? fetch; + + const res = await fetchImpl(url); + if (!res.ok) throw new Error(`http source failed: ${res.status} ${res.statusText}`); + + const contentType = res.headers.get('content-type') ?? ''; + const data = contentType.includes('json') ? await res.json() : await res.text(); + return { data }; +} diff --git a/packages/sources/src/identity.ts b/packages/sources/src/identity.ts new file mode 100644 index 0000000..fae0138 --- /dev/null +++ b/packages/sources/src/identity.ts @@ -0,0 +1,58 @@ +/** + * Identity resolution for the `#atproto` PDS path: turn a did/handle into the actor's PDS endpoint. + * Only `com.atproto.repo.*` / `com.atproto.sync.*` queries need this; appview queries take the + * handle/did as a plain param. + */ + +const PLC_DIRECTORY = 'https://plc.directory'; + +function pdsFromDidDoc(doc: unknown): string | undefined { + const services = (doc as { service?: unknown }).service; + if (!Array.isArray(services)) return undefined; + const svc = services.find((s) => (s as { id?: string }).id === '#atproto_pds') as + | { serviceEndpoint?: unknown } + | undefined; + return typeof svc?.serviceEndpoint === 'string' ? svc.serviceEndpoint : undefined; +} + +/** Resolve a DID → PDS endpoint. Supports `did:plc` (plc.directory) and `did:web` (well-known). */ +async function resolveDidToPds(did: string, fetchImpl: typeof fetch): Promise { + if (did.startsWith('did:plc:')) { + const r = await fetchImpl(`${PLC_DIRECTORY}/${did}`); + if (!r.ok) throw new Error(`plc.directory returned ${r.status} for ${did}`); + const pds = pdsFromDidDoc(await r.json()); + if (!pds) throw new Error(`no #atproto_pds service in DID doc for ${did}`); + return pds; + } + if (did.startsWith('did:web:')) { + const host = did.slice('did:web:'.length).replace(/:/g, '/'); + const r = await fetchImpl(`https://${host}/.well-known/did.json`); + if (!r.ok) throw new Error(`did:web doc returned ${r.status} for ${did}`); + const pds = pdsFromDidDoc(await r.json()); + if (!pds) throw new Error(`no #atproto_pds service in DID doc for ${did}`); + return pds; + } + throw new Error(`unsupported DID method: ${did}`); +} + +/** + * Resolve a did OR handle to the actor's PDS endpoint. A handle is first resolved to a did via the + * given appview's `com.atproto.identity.resolveHandle`. + */ +export async function resolvePds( + identifier: string, + appview: string, + fetchImpl: typeof fetch +): Promise { + let did = identifier; + if (!identifier.startsWith('did:')) { + const url = new URL('/xrpc/com.atproto.identity.resolveHandle', appview); + url.searchParams.set('handle', identifier); + const r = await fetchImpl(url); + if (!r.ok) throw new Error(`could not resolve handle ${identifier} (${r.status})`); + const body = (await r.json()) as { did?: string }; + if (!body.did) throw new Error(`no did for handle ${identifier}`); + did = body.did; + } + return resolveDidToPds(did, fetchImpl); +} diff --git a/packages/sources/src/index.ts b/packages/sources/src/index.ts new file mode 100644 index 0000000..7802aed --- /dev/null +++ b/packages/sources/src/index.ts @@ -0,0 +1,16 @@ +/** + * `@blento/sources` — resolve a node's declarative `Source` into typed data. + * + * Pure, dependency-light: no KV, no SvelteKit, no atproto client. Inject `fetch` and a `CacheAdapter` + * via the context/options. See ../../../blento-schema-design.md § Source. + */ +export type { + Source, + SourceContext, + CacheAdapter, + ResolveOptions, + ResolveResult +} from './types.js'; +export { resolve, resolveGraph } from './resolve.js'; +export { cacheKey, MemoryCacheAdapter } from './cache.js'; +export { DEFAULT_APPVIEW } from './atproto.js'; diff --git a/packages/sources/src/resolve.test.ts b/packages/sources/src/resolve.test.ts new file mode 100644 index 0000000..faf41a9 --- /dev/null +++ b/packages/sources/src/resolve.test.ts @@ -0,0 +1,79 @@ +import { describe, it, expect } from 'vitest'; +import { resolve, resolveGraph } from './resolve.js'; +import { MemoryCacheAdapter } from './cache.js'; +import type { Source, SourceContext } from './types.js'; +import { fakeFetch } from './testutil.js'; + +const atproto = (params: Record): Source => ({ + $type: 'app.blento.source#atproto', + method: 'app.bsky.feed.getAuthorFeed', + params +}); + +describe('#custom (stub)', () => { + it('throws not-implemented', async () => { + const s: Source = { $type: 'app.blento.source#custom', loader: 'x' }; + await expect(resolve(s, { self: 'x' })).rejects.toThrow(/not implemented/); + }); +}); + +describe('lone #ref', () => { + it('cannot be resolved directly', async () => { + const s: Source = { $type: 'app.blento.source#ref', node: 'owner' }; + await expect(resolve(s, { self: 'x' })).rejects.toThrow(/resolveGraph/); + }); +}); + +describe('resolveGraph', () => { + const ctx = (fetchImpl: typeof fetch): SourceContext => ({ self: 'did:plc:aaa', fetchImpl }); + + it('aliases #ref nodes to the owner result and dedupes identical inline sources', async () => { + const ff = fakeFetch([{ match: 'getAuthorFeed', body: { feed: ['post'], cursor: 'z' } }]); + const nodes = [ + { id: 'owner', source: atproto({ actor: '$self' }) }, + // identical spec to owner → should dedupe to a single fetch + { id: 'twin', source: atproto({ actor: '$self' }) }, + { id: 'ref', source: { $type: 'app.blento.source#ref', node: 'owner' } as Source }, + { id: 'plain', source: undefined } + ]; + + const cache = new MemoryCacheAdapter(); + const out = await resolveGraph(nodes, ctx(ff.fetch), { cache }); + + expect(out.owner).toEqual({ data: { feed: ['post'], cursor: 'z' }, nextCursor: 'z' }); + expect(out.ref).toEqual(out.owner); + expect(out.twin).toEqual(out.owner); + expect(out.plain).toBeUndefined(); + // owner + twin share a cache key → exactly one network fetch + expect(ff.calls.length).toBe(1); + }); + + it('yields null for a #ref to a non-source / missing owner', async () => { + const ff = fakeFetch([{ match: 'getAuthorFeed', body: { feed: [] } }]); + const nodes = [ + { id: 'plain', source: undefined }, + { id: 'refToPlain', source: { $type: 'app.blento.source#ref', node: 'plain' } as Source }, + { id: 'refToMissing', source: { $type: 'app.blento.source#ref', node: 'nope' } as Source } + ]; + const out = await resolveGraph(nodes, ctx(ff.fetch)); + expect(out.refToPlain).toBeNull(); + expect(out.refToMissing).toBeNull(); + }); + + it('disallows ref→ref chains', async () => { + const ff = fakeFetch([{ match: 'getAuthorFeed', body: { feed: [] } }]); + const nodes = [ + { id: 'owner', source: atproto({ actor: '$self' }) }, + { id: 'r1', source: { $type: 'app.blento.source#ref', node: 'owner' } as Source }, + { id: 'r2', source: { $type: 'app.blento.source#ref', node: 'r1' } as Source } + ]; + await expect(resolveGraph(nodes, ctx(ff.fetch))).rejects.toThrow(/#ref chain/); + }); + + it('isolates a failing source to null without failing the whole graph', async () => { + const ff = fakeFetch([{ match: 'getAuthorFeed', body: { error: 'boom' }, status: 500 }]); + const nodes = [{ id: 'bad', source: atproto({ actor: '$self' }) }]; + const out = await resolveGraph(nodes, ctx(ff.fetch)); + expect(out.bad).toBeNull(); + }); +}); diff --git a/packages/sources/src/resolve.ts b/packages/sources/src/resolve.ts new file mode 100644 index 0000000..477bc25 --- /dev/null +++ b/packages/sources/src/resolve.ts @@ -0,0 +1,140 @@ +/** + * `resolve()` — the core entry point — and `resolveGraph()` for `#ref` sharing. + * + * A source fetches ONE page. The caller decides how far to walk (via `nextCursor`). Results are + * read/written through an optional injectable cache, keyed by the resolved spec (+ cursor). + */ +import type { Node } from '@blento/schema'; +import type { + CacheAdapter, + ResolveOptions, + ResolveResult, + Source, + SourceContext +} from './types.js'; +import { resolveAtproto, substituteAtproto } from './atproto.js'; +import { resolveHttp } from './http.js'; +import { cacheKey } from './cache.js'; + +/** + * Return the source with context vars substituted — the canonical spec used for BOTH cache key and + * fetch. Only `#atproto` params reference context vars today; other kinds pass through unchanged. + */ +function resolveSpec(source: Source, ctx: SourceContext): Source { + return source.$type === 'app.blento.source#atproto' ? substituteAtproto(source, ctx) : source; +} + +async function readCache(cache: CacheAdapter, key: string): Promise { + const raw = await cache.get(key); + if (raw == null) return undefined; + try { + return JSON.parse(raw) as ResolveResult; + } catch { + return undefined; + } +} + +/** Dispatch a resolved (substituted) source to its kind handler. */ +async function dispatch( + source: Source, + ctx: SourceContext, + cursor: string | undefined +): Promise { + switch (source.$type) { + case 'app.blento.source#atproto': + return resolveAtproto(source, ctx, cursor); + case 'app.blento.source#http': + return resolveHttp(source, ctx); + case 'app.blento.source#custom': + // Later gated tier — sandboxed loaders. Not built yet. + throw new Error('custom source loaders are not implemented'); + case 'app.blento.source#ref': + // `#ref` is resolution-layer only: it means "reuse another node's loaded data". A lone ref + // has nothing to fetch — resolve it through resolveGraph(), which has the sibling results. + throw new Error('#ref sources can only be resolved via resolveGraph()'); + default: { + const exhaustive: never = source; + throw new Error(`unknown source kind: ${JSON.stringify(exhaustive)}`); + } + } +} + +/** + * Resolve one declarative source into `{ data, nextCursor }`. With a cache adapter, a hit returns + * instantly; a miss fetches then writes back under the spec-derived key. + */ +export async function resolve( + source: Source, + ctx: SourceContext, + opts: ResolveOptions = {} +): Promise { + const { cursor, cache, ttl } = opts; + const spec = resolveSpec(source, ctx); + + if (cache) { + const key = cacheKey(spec, cursor); + const hit = await readCache(cache, key); + if (hit) return hit; + const result = await dispatch(spec, ctx, cursor); + await cache.set(key, JSON.stringify(result), ttl); + return result; + } + + return dispatch(spec, ctx, cursor); +} + +/** A node with an (optional) inline source — the subset of `@blento/schema`'s `Node` we need. */ +type SourcedNode = Pick; + +/** + * Resolve every node's inline source, then alias `#ref` nodes to their owner's result. + * + * Identical inline fetches are deduped by cache key so each distinct source runs once. `#ref` → an + * owner that is itself a `#ref` is disallowed (no ref→ref chains); a dangling / non-source owner + * yields `null`. Returns a map of nodeId → result (or `null` when the node has no resolvable data). + */ +export async function resolveGraph( + nodes: SourcedNode[], + ctx: SourceContext, + opts: ResolveOptions = {} +): Promise> { + const REF = 'app.blento.source#ref'; + const out: Record = {}; + + // 1. Resolve each distinct inline (non-ref) source once, deduped by cache key. + const inline = nodes.filter((n) => n.source && n.source.$type !== REF); + const byKey = new Map>(); + const nodeKey = new Map(); + + for (const node of inline) { + const spec = resolveSpec(node.source!, ctx); + const key = cacheKey(spec, opts.cursor); + nodeKey.set(node.id, key); + if (!byKey.has(key)) byKey.set(key, resolve(node.source!, ctx, opts)); + } + + const settled = new Map(); + await Promise.all( + [...byKey.entries()].map(async ([key, p]) => { + try { + settled.set(key, await p); + } catch { + settled.set(key, null); + } + }) + ); + for (const node of inline) out[node.id] = settled.get(nodeKey.get(node.id)!) ?? null; + + // 2. Alias `#ref` nodes to their owner's already-resolved result. + for (const node of nodes) { + if (node.source?.$type !== REF) continue; + const ownerId = node.source.node; + const owner = nodes.find((n) => n.id === ownerId); + if (owner?.source?.$type === REF) { + throw new Error(`#ref chain not allowed: ${node.id} → ${ownerId} (also a #ref)`); + } + out[node.id] = ownerId in out ? out[ownerId] : null; + } + + return out; +} diff --git a/packages/sources/src/testutil.ts b/packages/sources/src/testutil.ts new file mode 100644 index 0000000..ad63480 --- /dev/null +++ b/packages/sources/src/testutil.ts @@ -0,0 +1,32 @@ +/** + * Test helpers: a recording fake `fetch` that maps URL substrings to JSON responses. + */ +export interface FakeCall { + url: string; + method: string; +} + +export interface FakeFetch { + fetch: typeof fetch; + calls: FakeCall[]; +} + +type Route = { match: string; body: unknown; status?: number; json?: boolean }; + +/** Build a fake fetch that returns the first route whose `match` is a substring of the URL. */ +export function fakeFetch(routes: Route[]): FakeFetch { + const calls: FakeCall[] = []; + const impl = (async (input: Parameters[0], init?: Parameters[1]) => { + const url = typeof input === 'string' ? input : input.toString(); + calls.push({ url, method: init?.method ?? 'GET' }); + const route = routes.find((r) => url.includes(r.match)); + if (!route) throw new Error(`fakeFetch: no route for ${url}`); + const json = route.json ?? true; + const headers = new Headers({ 'content-type': json ? 'application/json' : 'text/plain' }); + return new Response(json ? JSON.stringify(route.body) : String(route.body), { + status: route.status ?? 200, + headers + }); + }) as unknown as typeof fetch; + return { fetch: impl, calls }; +} diff --git a/packages/sources/src/types.ts b/packages/sources/src/types.ts new file mode 100644 index 0000000..2afb194 --- /dev/null +++ b/packages/sources/src/types.ts @@ -0,0 +1,54 @@ +/** + * Public types for `@blento/sources`. + * + * A `Source` (imported from `@blento/schema`) is a declarative read. `resolve()` turns it into typed + * data. Renderers never see a `Source` — only the already-fetched JSON this package produces. + */ +import type { Source } from '@blento/schema'; + +export type { Source }; + +/** + * Ambient inputs a source needs to resolve that are NOT part of the stored spec: the page owner + * (for `$self`), the fetch implementation, and the two security allowlists (data, not code). + */ +export interface SourceContext { + /** Page owner did or handle. Substituted for the `$self` param var. */ + self: string; + /** Fetch implementation (defaults to the global `fetch`). Injectable for tests / Workers. */ + fetchImpl?: typeof fetch; + /** + * Host allowlist for `#http` sources (bare hostnames, e.g. `api.example.com`). Empty / undefined + * ⇒ every `#http` fetch is refused. Arbitrary server-side fetch is the risk surface. + */ + httpAllowlist?: string[]; + /** + * Allowed appview base URLs for non-repo/sync `#atproto` queries, IN ADDITION to the built-in + * default (bsky public appview). A `service` not in this set is refused (gated). + */ + appviews?: string[]; + /** Override the default appview used when a `#atproto` source names no `service`. */ + defaultAppview?: string; +} + +/** A minimal, injectable cache. The app supplies KV; tests use the in-memory adapter. */ +export interface CacheAdapter { + get(key: string): Promise | string | null; + set(key: string, value: string, ttl?: number): Promise | void; +} + +/** Options for a single `resolve()` call. */ +export interface ResolveOptions { + /** Pagination cursor to fetch the next page (threaded into the underlying query). */ + cursor?: string; + /** Optional cache. When present, results are read/written under a spec-derived key. */ + cache?: CacheAdapter; + /** TTL (seconds) for cache writes. */ + ttl?: number; +} + +/** The result of resolving one source: the fetched data plus an optional next-page cursor. */ +export interface ResolveResult { + data: unknown; + nextCursor?: string; +} diff --git a/packages/sources/tsconfig.json b/packages/sources/tsconfig.json new file mode 100644 index 0000000..e17014e --- /dev/null +++ b/packages/sources/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../tsconfig.base.json", + "compilerOptions": { + "rootDir": ".", + "outDir": "dist" + }, + "include": ["src/**/*.ts"] +}