Automerge backed Markdown editor for the Atmosphere
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853/* oxlint-disable sonarjs/cognitive-complexity */import { type BasesEvaluatedRow, type BasesPrimitive } from './batch-evaluator'import { compileBasesQuery } from './bases-compile'import type { BasesQueryPlan } from './bases-contract'import type { BasesRowInput } from './bases-contract'import { createBasesRunner, type BasesRunner } from './bases-run'import type { BasesYamlDiagnostic } from './bases-parser'import type { TypedNodeCatalogRow, TypedNodeCatalogSnapshot, TypedNodeCatalogEvent,} from '../types-records/typed-node-catalog'import { BASES_LIMITS } from './bases-contract'import { brandTrustedBasesSnapshot, createBasesPrimitiveSnapshotWalker, type BasesSnapshot, type PrimitiveSnapshotCounters, type PrimitiveSnapshotWalker } from './bases-snapshot'import { baseSource } from './base-source-config'import { typedNodeSubjectKey } from '../types-records/typed-node-catalog'import type { BaseNoteCandidate, BaseNoteSnapshot } from './base-note-candidates'import type { AutomergeUrl } from '../storage/document-runtime-contract'import type { Heads } from '../sync/heads'import { headsEqual } from '../sync/heads'import { parseFrontmatterTags } from '../services/metadata/extract-metadata'import { DEFAULT_TEMPORAL_FORMATS, type TemporalFormats } from '../utils/temporal-value'import type { MetadataCache } from '../services/metadata/metadata-cache'
const CONTROLLER_SLICE_MS = 8const MAX_STEPS_PER_SLICE = 64
function hasSliceBudget(startedAt: number, steps: number): boolean { return steps < MAX_STEPS_PER_SLICE && performance.now() - startedAt < CONTROLLER_SLICE_MS}
export interface TypedNodeCatalogSource { getSnapshot(): TypedNodeCatalogSnapshot subscribe(callback: (event?: TypedNodeCatalogEvent | { kind: 'notes' | 'metadata' }) => void): () => void getNotes?(): BaseNoteSnapshot tags?: { readHeads(url: AutomergeUrl): Promise<Heads | undefined> indexedHeads(url: AutomergeUrl): Heads | undefined frontmatter(url: AutomergeUrl): Record<string, unknown> | undefined | null hasCache(url: AutomergeUrl): boolean catalogHeads?(url: AutomergeUrl): Heads | undefined }}
function needsTags(plan: BasesQueryPlan): boolean { return plan.programs.some(program => program.properties.some(property => property === 'tags' || property === 'row.tags' || property === 'note.tags' || property === 'node.recordTags'))}
function needsStatus(plan: BasesQueryPlan): boolean { return plan.programs.some(program => program.properties.some(property => property === 'status' || property === 'note.status' || property === 'row.status'))}
function needsRecordStatus(plan: BasesQueryPlan): boolean { return plan.programs.some(program => program.properties.some(property => property === 'node.recordStatus' || property === 'row.status'))}
function needsBroadStatus(plan: BasesQueryPlan): boolean { return plan.programs.some(program => program.properties.includes('row.status'))}
interface FactProjection { readonly tags: boolean; readonly status: boolean; readonly recordStatus: boolean; readonly broadStatus: boolean; readonly recordStatusTypes?: ReadonlySet<string> }
function recordStatusTypes(plan: BasesQueryPlan): ReadonlySet<string> | undefined { const names = new Set<string>() for (const program of plan.programs) for (let index = 0; index < program.instructions.length; index++) { const instruction = program.instructions[index]! if (instruction.op !== 'property' || program.properties[instruction.argument as number] !== 'node.recordStatus') continue const constant = program.instructions[index + 1] const indexed = program.instructions[index + 2] if (constant?.op !== 'const' || indexed?.op !== 'index') return undefined const name = program.constants[constant.argument as number] if (typeof name !== 'string') return undefined names.add(name) } return names}
function factProjection(plan: BasesQueryPlan): FactProjection { const broadStatus = needsBroadStatus(plan) return { tags: needsTags(plan), status: needsStatus(plan), recordStatus: needsRecordStatus(plan), broadStatus, ...(!broadStatus ? { recordStatusTypes: recordStatusTypes(plan) } : {}) }}
type FrontmatterProjection = Readonly<{ tags?: readonly string[]; status?: string | null }>
function frontmatterStatus(value: Record<string, unknown> | undefined | null): string | null { const raw = value?.status if (raw === undefined || raw === null) return null if (typeof raw !== 'string') throw new IncompatibleFrontmatterStatusError('Frontmatter status must be a string') if (raw.length > BASES_LIMITS.stringLength) throw new ProjectionLimitError('Frontmatter status exceeds the Base query value limit') return raw}
class IncompatibleFrontmatterStatusError extends Error { readonly code = 'frontmatter.incompatible'}
class IncompatibleRecordStatusError extends Error { readonly code = 'record.incompatible'}
function frontmatterTags(value: Record<string, unknown> | undefined | null): readonly string[] { const raw = value?.tags if (Array.isArray(raw) && raw.length > BASES_LIMITS.listItems) throw new ProjectionLimitError('Frontmatter tag list exceeds the Base query limit') if (typeof raw === 'string' && raw.length > BASES_LIMITS.stringLength) throw new ProjectionLimitError('Frontmatter tag exceeds the Base query value limit') if (Array.isArray(raw) && raw.some(tag => typeof tag === 'string' && tag.length > BASES_LIMITS.stringLength)) throw new ProjectionLimitError('Frontmatter tag exceeds the Base query value limit') return [...parseFrontmatterTags(value)]}
function recordTags(row: TypedNodeCatalogRow): string[] { const tags = new Set<string>() for (const field of row.fields) { if (field.key !== 'tags') continue if (field.kind !== 'text' && field.kind !== 'multi-select') continue const value = field.value if (field.kind === 'text' && typeof value !== 'string' || field.kind === 'multi-select' && !Array.isArray(value)) continue for (const tag of parseFrontmatterTags({ tags: value })) tags.add(tag) } return [...tags]}
function tagsForGroup(group: ProjectionGroup, frontmatter: ReadonlyMap<AutomergeUrl, FrontmatterProjection>): Record<string, unknown> { const noteTags = frontmatter.get(group.first.documentId)?.tags ?? [] const tags = new Set(noteTags) const byType: Record<string, string[]> = Object.create(null) as Record<string, string[]> for (const row of group.rows) { byType[row.typeName] = recordTags(row) for (const tag of byType[row.typeName]!) tags.add(tag) } return { 'note.tags': noteTags, 'row.tags': [...tags], 'node.recordTags': byType }}
function statusForGroup(group: ProjectionGroup, frontmatter: ReadonlyMap<AutomergeUrl, FrontmatterProjection> | undefined, includeRecords: boolean, broad: boolean, recordTypes?: ReadonlySet<string>): Record<string, unknown> { const note = frontmatter?.get(group.first.documentId)?.status const values = new Set<string>() if (note !== undefined && note !== null) values.add(note) const byType: Record<string, string | null> = Object.create(null) as Record<string, string | null> if (includeRecords) for (const row of group.rows) { if (recordTypes && !recordTypes.has(row.typeName)) continue let status: string | null = null for (const field of row.fields) { if (field.key !== 'status') continue if (field.state === 'conflicted' || field.value !== undefined && ((field.kind !== 'text' && field.kind !== 'select') || typeof field.value !== 'string')) { throw new IncompatibleRecordStatusError('Record status has an incompatible or conflicted value') } if (typeof field.value === 'string') status = field.value } byType[row.typeName] = status if (status !== null) values.add(status) } return { ...(note !== undefined ? { 'note.status': note } : {}), ...(broad ? { 'row.status': values.size ? [...values] : null } : {}), ...(includeRecords ? { 'node.recordStatus': byType } : {}), }}
export type TypeQueryState = | Readonly<{ status: 'disabled' }> | Readonly<{ status: 'indexing' }> | Readonly<{ status: 'unavailable'; message: string }> | Readonly<{ status: 'invalid' | 'error'; query: string; selectedView?: string; diagnostics: readonly BasesYamlDiagnostic[]; results: readonly BasesEvaluatedRow[] }> | Readonly<{ status: 'ready' query: string selectedView?: string columns: readonly string[] orderedRowIds?: readonly string[] results: readonly TypeQueryResultRow[] }>
export interface TypeQueryResultRow { readonly evaluated: BasesEvaluatedRow readonly navigation: TypeQueryNavigation readonly recordsByType: Readonly<Record<string, TypedNodeCatalogRow>>}
export type TypeQueryNavigation = Readonly<Pick<TypedNodeCatalogRow, 'subjectKey' | 'subject' | 'documentId' | 'path' | 'name' | 'node'> & { readonly fileTitle?: string }>
interface TypeQuerySubjectSource { readonly navigation: TypeQueryNavigation readonly recordsByType: Record<string, TypedNodeCatalogRow>}
function captureSubjectSource(sources: Map<string, TypeQuerySubjectSource>, row: TypedNodeCatalogRow, fileTitle?: string): void { let source = sources.get(row.subjectKey) if (!source) { source = { navigation: Object.freeze({ subjectKey: row.subjectKey, subject: row.subject, documentId: row.documentId, path: row.path, name: row.name, node: row.node, ...(fileTitle !== undefined ? { fileTitle } : {}) }), recordsByType: Object.create(null) as Record<string, TypedNodeCatalogRow>, } sources.set(row.subjectKey, source) } source.recordsByType[row.typeName] = row}function resultRow(evaluated: BasesEvaluatedRow, source: TypeQuerySubjectSource): TypeQueryResultRow { const recordsByType = Object.assign(Object.create(null) as Record<string, TypedNodeCatalogRow>, source.recordsByType) return Object.freeze({ evaluated, navigation: source.navigation, recordsByType: Object.freeze(recordsByType), })}
class ProjectionLimitError extends Error { readonly code = 'limit.projection'}
function pathParts(path: string): { name: string; folder: string; ext: string } { const slash = path.lastIndexOf('/') const fullname = slash < 0 ? path : path.slice(slash + 1) const dot = fullname.lastIndexOf('.') return { name: dot > 0 ? fullname.slice(0, dot) : fullname, folder: slash < 0 ? '' : path.slice(0, slash), ext: dot > 0 ? fullname.slice(dot + 1) : '', }}
interface PendingNote { readonly id: string; readonly note: BaseNoteCandidate; readonly title: string; readonly walker: PrimitiveSnapshotWalker }
function prepareNote(rows: readonly BasesRowInput[], sources: ReadonlyMap<string, TypeQuerySubjectSource>, note: BaseNoteCandidate, counters: PrimitiveSnapshotCounters, frontmatter?: ReadonlyMap<AutomergeUrl, FrontmatterProjection>, broadStatus = false, fileTitle?: (documentId: AutomergeUrl) => string | null): PendingNote | undefined { const id = typedNodeSubjectKey(note.documentId, { kind: 'root' }) if (sources.has(id)) return undefined if (id.length > BASES_LIMITS.idLength) throw new ProjectionLimitError('Base note identity exceeds the query limit') if (rows.length >= BASES_LIMITS.inputRows) throw new ProjectionLimitError('Base candidate count exceeds the query limit') const parts = pathParts(note.path) const title = fileTitle?.(note.documentId) ?? parts.name const values = { 'file.name': parts.name, 'file.title': title, 'file.fullname': note.name, 'file.path': note.path, 'file.folder': parts.folder, 'file.ext': parts.ext, 'node.records': Object.create(null) as Record<string, BasesPrimitive>, ...(frontmatter?.get(note.documentId)?.tags ? { 'note.tags': frontmatter.get(note.documentId)!.tags!, 'row.tags': frontmatter.get(note.documentId)!.tags! } : {}), ...(frontmatter?.get(note.documentId)?.status !== undefined ? { 'note.status': frontmatter.get(note.documentId)!.status! } : {}), ...(broadStatus ? { 'row.status': frontmatter?.get(note.documentId)?.status === null || frontmatter?.get(note.documentId)?.status === undefined ? null : [frontmatter.get(note.documentId)!.status!] } : {}), } return { id, note, title, walker: createBasesPrimitiveSnapshotWalker(values, counters) }}
function appendNote(rows: BasesRowInput[], sources: Map<string, TypeQuerySubjectSource>, pending: PendingNote): void { const { id, note, title, walker } = pending if (!walker.done || !walker.value || typeof walker.value !== 'object' || Array.isArray(walker.value)) throw new ProjectionLimitError('Base note projection is incomplete') rows.push(Object.freeze({ id, values: walker.value as Readonly<Record<string, BasesPrimitive>> })) sources.set(id, { navigation: Object.freeze({ subjectKey: id, subject: { kind: 'root' as const, documentId: note.documentId }, documentId: note.documentId, path: note.path, name: note.name, fileTitle: title, node: { kind: 'document', id: 'root', from: 0, to: 0, text: '' }, }), recordsByType: Object.create(null) as Record<string, TypedNodeCatalogRow> })}
/** Pure, bounded projection from one immutable catalog generation to Bases rows. */interface ProjectionFootprint { readonly nodes: number; readonly bytes: number }interface ProjectionMetrics { nodes: number; bytes: number; footprints: Map<string, ProjectionFootprint> }
export function projectTypedNodeCatalogRows(rows: readonly TypedNodeCatalogRow[], metrics?: ProjectionMetrics, fileTitle?: (documentId: AutomergeUrl) => string | null): BasesSnapshot { return projectTypedRows(rows, metrics, undefined, undefined, fileTitle)}
function projectTypedRows(rows: readonly TypedNodeCatalogRow[], metrics?: ProjectionMetrics, frontmatter?: ReadonlyMap<AutomergeUrl, FrontmatterProjection>, demand?: FactProjection, fileTitle?: (documentId: AutomergeUrl) => string | null): BasesSnapshot { // Synchronous projection is deliberately the same state machine as the // scheduled path; only the draining policy differs. const runner = createProjectionRunner(rows, undefined, frontmatter, demand, fileTitle) const projected: BasesRowInput[] = [] while (true) { const result = runner.step() projected.push(...result.rows) if (result.done) { if (metrics) { metrics.nodes = runner.counters.nodes metrics.bytes = runner.counters.bytes metrics.footprints = new Map(runner.footprints) } return brandTrustedBasesSnapshot(projected) } }}
function emptyResults(): readonly BasesEvaluatedRow[] { return Object.freeze([]) }
function isSingletonEquivalentPlan(plan: BasesQueryPlan): boolean { if (plan.selectedView.sort.length > 0 || plan.selectedView.groupBy !== undefined || plan.selectedView.limit !== undefined || Object.keys(plan.selectedView.summaries).length > 0 || Object.keys(plan.summaryPrograms).length > 0) return false // Without callbacks, control-flow jumps, regexes, or constructed containers, // replacing one primitive checkbox value consumes the same per-row VM/value // shape as the successful baseline evaluation. return plan.programs.every(program => program.constants.every(value => !(value !== null && typeof value === 'object' && 'kind' in value && value.kind === 'regex')) && program.instructions.every(instruction => !['call-global', 'call-receiver', 'jump-if-false', 'jump-if-true', 'jump', 'if', 'make-list', 'make-map', 'unary'].includes(instruction.op)) && program.instructions.every((instruction, index) => instruction.op !== 'binary' || ( (instruction.argument === '==' || instruction.argument === '!=') && program.instructions.length === 5 && index === 4 && program.instructions[0]?.op === 'property' && program.instructions[1]?.op === 'const' && program.instructions[2]?.op === 'index' && program.instructions[3]?.op === 'const' && program.constants[program.instructions[3]!.argument as number] === null )))}
function isDirectScalarPlan(plan: BasesQueryPlan): boolean { // TypeQueryState exposes public rows, not pipeline groups. Without a limit or // summaries, one existing member can therefore be re-evaluated independently: // sort/group work only controls pipeline/group products, while public rows // retain catalog order in the Bases runtime. if (plan.selectedView.sort.length > 0 || plan.selectedView.limit !== undefined || Object.keys(plan.selectedView.summaries).length > 0 || Object.keys(plan.summaryPrograms).length > 0) return false return plan.programs.every(program => program.constants.every(value => !(value !== null && typeof value === 'object')) && program.instructions.every(instruction => instruction.op === 'property' || instruction.op === 'const' || instruction.op === 'index' || instruction.op === 'binary'))}
interface ProjectionGroup { readonly id: string; readonly first: TypedNodeCatalogRow; readonly rows: TypedNodeCatalogRow[]; readonly records: Record<string, Record<string, BasesPrimitive>>; readonly fieldCounts: Map<string, number> }interface ProjectionStep { readonly rows: ReadonlyArray<BasesRowInput>; readonly done: boolean }interface ProjectionRunner { readonly counters: PrimitiveSnapshotCounters; readonly footprints: ReadonlyMap<string, ProjectionFootprint>; step(): ProjectionStep; cancel(): void }
function createProjectionRunner(rows: readonly TypedNodeCatalogRow[], onSourceRow?: (row: TypedNodeCatalogRow) => void, frontmatter?: ReadonlyMap<AutomergeUrl, FrontmatterProjection>, projection?: FactProjection, fileTitle?: (documentId: AutomergeUrl) => string | null): ProjectionRunner { if (rows.length > BASES_LIMITS.typeCatalogRows) throw new ProjectionLimitError(`Type catalog rows exceed ${BASES_LIMITS.typeCatalogRows}`) let rowIndex = 0; let fieldIndex = 0; let current: { row: TypedNodeCatalogRow; fields: Record<string, BasesPrimitive>; count: number; valueWalker?: PrimitiveSnapshotWalker; valueStartNodes?: number; valueStartBytes?: number } | undefined let cancelled = false let totalFields = 0 const snapshotCounters: PrimitiveSnapshotCounters = { nodes: 0, bytes: 0 } const footprints = new Map<string, { nodes: number; bytes: number }>() const groups = new Map<string, ProjectionGroup>() let finalizeIndex = 0 let finalGroups: readonly ProjectionGroup[] | undefined let tagWalker: PrimitiveSnapshotWalker | undefined let finalTitle: { group: ProjectionGroup; parts: ReturnType<typeof pathParts>; records: Record<string, BasesPrimitive>; walker: PrimitiveSnapshotWalker; bytesBefore: number } | undefined let operations = 0 return { counters: snapshotCounters, footprints, cancel: () => { cancelled = true }, step: () => { if (cancelled) return { rows: Object.freeze([]), done: true } operations = 0 const emitted: BasesRowInput[] = [] while (operations < BASES_LIMITS.batchRows) { if (!current) { if (rowIndex >= rows.length) { finalGroups ??= [...groups.values()] if (finalizeIndex >= finalGroups.length) return { rows: Object.freeze(emitted), done: true } const group = finalGroups[finalizeIndex]! if (projection?.tags || projection?.status || projection?.recordStatus) { tagWalker ??= createBasesPrimitiveSnapshotWalker({ ...(projection.tags && frontmatter ? tagsForGroup(group, frontmatter) : {}), ...(projection.status || projection.recordStatus ? statusForGroup(group, frontmatter, projection.recordStatus, projection.broadStatus, projection.recordStatusTypes) : {}), }, snapshotCounters) operations += tagWalker.step(Math.min(BASES_LIMITS.sliceValueNodes, BASES_LIMITS.batchRows - operations)) if (!tagWalker.done) continue } if (!finalTitle) { const parts = pathParts(group.first.path) const records: Record<string, BasesPrimitive> = Object.create(null) as Record<string, BasesPrimitive> for (const typeName of Object.keys(group.records)) records[typeName] = Object.freeze(group.records[typeName]!) finalTitle = { group, parts, records, walker: createBasesPrimitiveSnapshotWalker(fileTitle?.(group.first.documentId) ?? parts.name, snapshotCounters), bytesBefore: snapshotCounters.bytes } } const { parts, records, walker, bytesBefore } = finalTitle operations += walker.step(Math.min(BASES_LIMITS.sliceValueNodes, BASES_LIMITS.batchRows - operations)) if (!walker.done) continue const footprint = footprints.get(group.id) ?? { nodes: 0, bytes: 0 } footprint.bytes += snapshotCounters.bytes - bytesBefore footprints.set(group.id, footprint) const values: Record<string, unknown> = { 'file.name': parts.name, 'file.title': walker.value, 'file.fullname': group.first.name, 'file.path': group.first.path, 'file.folder': parts.folder, 'file.ext': parts.ext, 'node.records': Object.freeze(records), ...(tagWalker?.value as object ?? {}) } tagWalker = undefined finalTitle = undefined finalizeIndex++ emitted.push(Object.freeze({ id: group.id, values: Object.freeze(values) })) operations += 1 continue } const row = rows[rowIndex++]! onSourceRow?.(row) let group = groups.get(row.subjectKey) if (!group) { if (groups.size >= BASES_LIMITS.projectedSubjects) throw new ProjectionLimitError(`Type catalog subjects exceed ${BASES_LIMITS.projectedSubjects}`) group = { id: row.subjectKey, first: row, rows: [], records: Object.create(null) as Record<string, Record<string, BasesPrimitive>>, fieldCounts: new Map() } groups.set(row.subjectKey, group) } group.rows.push(row) if (Object.keys(group.records).length >= BASES_LIMITS.typesPerSubject && !group.records[row.typeName]) throw new ProjectionLimitError(`Type catalog types per subject exceed ${BASES_LIMITS.typesPerSubject}`) current = { row, fields: Object.create(null) as Record<string, BasesPrimitive>, count: 0 } fieldIndex = 0 } const row = current.row if (fieldIndex < row.fields.length) { const field = row.fields[fieldIndex]! if (!current.valueWalker) { current.valueStartNodes = snapshotCounters.nodes current.valueStartBytes = snapshotCounters.bytes current.valueWalker = createBasesPrimitiveSnapshotWalker(field.value ?? field.rawValue ?? null, snapshotCounters) } const valueWork = Math.min(BASES_LIMITS.sliceValueNodes, BASES_LIMITS.batchRows - operations) operations += current.valueWalker.step(valueWork) if (!current.valueWalker.done) continue const footprint = footprints.get(row.subjectKey) ?? { nodes: 0, bytes: 0 } footprint.nodes += snapshotCounters.nodes - current.valueStartNodes! footprint.bytes += snapshotCounters.bytes - current.valueStartBytes! footprints.set(row.subjectKey, footprint) current.fields[field.key] = current.valueWalker.value ?? null current.valueWalker = undefined current.valueStartNodes = undefined current.valueStartBytes = undefined fieldIndex += 1 current.count += 1 totalFields += 1 if (totalFields > BASES_LIMITS.projectedFields) throw new ProjectionLimitError(`Type catalog fields exceed ${BASES_LIMITS.projectedFields}`) const group = groups.get(row.subjectKey)! const total = (group.fieldCounts.get(row.typeName) ?? 0) + 1 group.fieldCounts.set(row.typeName, total) if (total > BASES_LIMITS.fieldsPerSubject) throw new ProjectionLimitError(`Type catalog fields per subject exceed ${BASES_LIMITS.fieldsPerSubject}`) continue } groups.get(row.subjectKey)!.records[row.typeName] = Object.freeze(current.fields) current = undefined; operations += 1 } return { rows: Object.freeze(emitted), done: false } }, }}
/** Evaluates raw .base text against exactly one captured catalog snapshot. */export class TypeQueryController { private temporalFormats: TemporalFormats = DEFAULT_TEMPORAL_FORMATS private readonly listeners = new Set<() => void>() private readonly source: TypedNodeCatalogSource private unsubscribeFileTitles: (() => void) | undefined private query: string private selectedView: string | undefined private state: TypeQueryState private unsubscribeSource: (() => void) | undefined private runner: BasesRunner | undefined private generation = 0 private timer: ReturnType<typeof setTimeout> | undefined private evaluatedRevision: number | undefined private evaluatedQuery: string | undefined private evaluatedSelectedView: string | undefined private evaluatedPlan: BasesQueryPlan | undefined private readonly work = { fullProjectedRows: 0, incrementalProjectedSubjects: 0, incrementalEvaluatedSubjects: 0 } private resultIndexById = new Map<string, number>() private projectionMetrics: ProjectionMetrics | undefined private evaluatedGeneratedBytes = 0 private tagFrontmatter: ReadonlyMap<AutomergeUrl, FrontmatterProjection> | undefined
constructor(source: TypedNodeCatalogSource, query: string, selectedView?: string, private readonly fileTitles?: Pick<MetadataCache, 'getDisplayTitle' | 'subscribeDisplayTitles'>) { this.source = source this.query = query this.selectedView = selectedView this.state = Object.freeze({ status: 'disabled' }) this.connect() }
/** React StrictMode replays effect cleanup without replacing memoized state. */ connect(): void { if (this.unsubscribeSource) return this.unsubscribeSource = this.source.subscribe(event => this.onSourceChange(event)) this.scheduleEvaluation() this.unsubscribeFileTitles = this.fileTitles?.subscribeDisplayTitles(() => { // A later single-subject edit cannot replace the full re-evaluation: // file.title may have changed for any number of other subjects. this.evaluatedRevision = undefined this.evaluatedPlan = undefined this.scheduleEvaluation() }) }
getState(): TypeQueryState { return this.state } getWorkDiagnostics(): Readonly<typeof this.work> { return Object.freeze({ ...this.work }) } setQuery(query: string): void { this.query = query; this.scheduleEvaluation() } setTemporalFormats(formats: TemporalFormats): void { if (formats.dateFormat === this.temporalFormats.dateFormat && formats.timeFormat === this.temporalFormats.timeFormat) return this.temporalFormats = Object.freeze({ ...formats }) this.evaluatedPlan = undefined this.scheduleEvaluation() } setSelectedView(selectedView: string | undefined): void { if (selectedView === this.selectedView) return this.selectedView = selectedView this.scheduleEvaluation() } private selectionState(): { selectedView?: string } { return this.selectedView === undefined ? {} : { selectedView: this.selectedView } } subscribe(listener: () => void): () => void { this.listeners.add(listener); return () => { this.listeners.delete(listener) } } destroy(): void { this.unsubscribeSource?.() this.unsubscribeSource = undefined this.unsubscribeFileTitles?.() this.unsubscribeFileTitles = undefined this.runner?.cancel() this.runner = undefined if (this.timer !== undefined) clearTimeout(this.timer) this.listeners.clear() }
private readonly resolveFileTitle = (documentId: TypedNodeCatalogRow['documentId']): string | null => this.fileTitles?.getDisplayTitle(documentId) ?? null private readonly navigationTitle = (row: TypedNodeCatalogRow): string | undefined => this.fileTitles ? this.resolveFileTitle(row.documentId) ?? pathParts(row.path).name : undefined
/** Synchronous compatibility facade used by unit and corpus tests. */ evaluateNow(): void { if (this.timer !== undefined) clearTimeout(this.timer) this.timer = undefined this.runner?.cancel() this.runner = undefined this.evaluate() }
private scheduleEvaluation(): void { const generation = ++this.generation this.runner?.cancel() if (this.timer !== undefined) clearTimeout(this.timer) const snapshot = this.source.getSnapshot() const mode = baseSource(this.query, this.selectedView) const compiled = compileBasesQuery(this.query, this.selectedView) if ('__brand' in compiled && (needsTags(compiled) || needsStatus(compiled))) this.state = Object.freeze({ status: 'indexing' }) const notes = mode === 'notes' || mode === 'both' ? this.source.getNotes?.() : undefined if (mode !== 'notes' && snapshot.status === 'disabled') this.state = Object.freeze({ status: 'disabled' }) else if (notes?.status === 'indexing') this.state = Object.freeze({ status: 'indexing' }) else if (snapshot.status === 'indexing' && mode !== 'notes') { if (this.state.status === 'disabled' || this.state.status === 'indexing') this.state = Object.freeze({ status: 'indexing' }) } else if (this.state.status === 'disabled' || this.state.status === 'indexing') this.state = Object.freeze({ status: 'indexing' }) this.publish() this.timer = setTimeout(() => { this.timer = undefined this.startEvaluation(generation) }, 0) }
private onSourceChange(event?: TypedNodeCatalogEvent | { kind: 'notes' | 'metadata' }): void { const plan = compileBasesQuery(this.query, this.selectedView) const tagQuery = '__brand' in plan && needsTags(plan) const statusQuery = '__brand' in plan && needsStatus(plan) if (event?.kind === 'metadata' && !tagQuery && !statusQuery) return if (this.tagFrontmatter && event?.kind === 'subjects') { this.scheduleEvaluation(); return } if (event?.kind === 'subjects' && tagQuery) { this.scheduleEvaluation(); return } if (event?.kind === 'notes' && baseSource(this.query, this.selectedView) === 'records') return if (event?.kind === 'subjects' && this.tryIncrementalSubjects(event)) return this.scheduleEvaluation() }
private tryIncrementalSubjects(event: Extract<TypedNodeCatalogEvent, { kind: 'subjects' }>): boolean { const plan = this.evaluatedPlan if (plan?.selectedView.source !== 'records') return false const eligiblePlan = event.incrementalClass === 'fixed-boolean' || event.incrementalClass === 'scalar' ? plan !== undefined && (isSingletonEquivalentPlan(plan) || isDirectScalarPlan(plan)) : false if (this.state.status !== 'ready' || !plan || !eligiblePlan || this.evaluatedRevision !== event.previousRevision || this.evaluatedQuery !== this.query || this.evaluatedSelectedView !== this.selectedView || event.changedSubjectKeys.length !== 1 || !this.projectionMetrics) return false const subjectKey = event.changedSubjectKeys[0]! const existingIndex = this.resultIndexById.get(subjectKey) ?? -1 // Membership changes retain the full runner as their correctness oracle. if (existingIndex < 0 || event.rows.length === 0) return false try { const sourceById = new Map<string, TypeQuerySubjectSource>() const metrics: ProjectionMetrics = { nodes: 0, bytes: 0, footprints: new Map() } const input = projectTypedNodeCatalogRows(event.rows, metrics, this.resolveFileTitle) for (const row of event.rows) captureSubjectSource(sourceById, row, this.navigationTitle(row)) this.work.incrementalProjectedSubjects += input.length if (input.length !== 1 || input[0]!.id !== subjectKey) return false const incrementalRunner = createBasesRunner(input, plan, this.temporalFormats) const result = incrementalRunner.drain() this.work.incrementalEvaluatedSubjects += 1 if (result.status !== 'ready' || result.rows.length !== 1 || result.rows[0]!.id !== subjectKey || JSON.stringify(result.columns) !== JSON.stringify(this.state.columns)) return false const source = sourceById.get(subjectKey) if (!source) return false const previousFootprint = this.projectionMetrics?.footprints.get(subjectKey) const nextFootprint = metrics.footprints.get(subjectKey) if (!previousFootprint || !nextFootprint || this.projectionMetrics.nodes - previousFootprint.nodes + nextFootprint.nodes > BASES_LIMITS.inputNodes || this.projectionMetrics.bytes - previousFootprint.bytes + nextFootprint.bytes > BASES_LIMITS.inputBytes) return false const nextTotalBytes = this.projectionMetrics.bytes - previousFootprint.bytes + nextFootprint.bytes if (nextTotalBytes * Math.max(1, plan.programs.length) > BASES_LIMITS.generatedBytes) return false // The full run owns the exact aggregate generated-string budget. A // singleton re-evaluation may replace some of that work, but without a // per-row runtime ledger we conservatively add it. This can trigger a // safe full fallback after repeated edits; it cannot admit work that the // full query budget would reject. const incrementalGeneratedBytes = incrementalRunner.getGeneratedBytes() if (this.evaluatedGeneratedBytes + incrementalGeneratedBytes > BASES_LIMITS.generatedBytes) return false const results = [...this.state.results] results[existingIndex] = resultRow(result.rows[0]!, source) this.generation += 1 this.runner?.cancel() this.runner = undefined if (this.timer !== undefined) clearTimeout(this.timer) this.timer = undefined this.state = Object.freeze({ ...this.state, results: Object.freeze(results) }) this.evaluatedRevision = event.revision this.projectionMetrics = { nodes: this.projectionMetrics.nodes - previousFootprint.nodes + nextFootprint.nodes, bytes: this.projectionMetrics.bytes - previousFootprint.bytes + nextFootprint.bytes, footprints: new Map(this.projectionMetrics.footprints).set(subjectKey, nextFootprint), } this.evaluatedGeneratedBytes += incrementalGeneratedBytes this.publish() if ((globalThis as { __reamsPerfDiagnostics__?: unknown }).__reamsPerfDiagnostics__) globalThis.performance?.mark('reams:record-edit-base') globalThis.performance?.mark('reams:record-checkbox-base') return true } catch { return false } }
private evaluate(): void { this.startEvaluation(++this.generation, false) }
private startEvaluation(generation: number, asynchronous = true, tagFrontmatter?: ReadonlyMap<AutomergeUrl, FrontmatterProjection>): void { const snapshot = this.source.getSnapshot() const mode = baseSource(this.query, this.selectedView) if (!mode) { const result = compileBasesQuery(this.query, this.selectedView) this.setDiagnosticState('status' in result && result.status === 'invalid' ? 'invalid' : 'error', 'diagnostics' in result ? result.diagnostics : []) return } const noteState = mode === 'records' ? undefined : this.source.getNotes?.() const compiledTags = compileBasesQuery(this.query, this.selectedView) const projection = '__brand' in compiledTags ? factProjection(compiledTags) : undefined const candidatesReady = (mode === 'notes' || snapshot.status === 'ready') && (mode === 'records' || noteState?.status === 'ready') const needsMetadata = '__brand' in compiledTags && (needsTags(compiledTags) || needsStatus(compiledTags)) if (needsMetadata && !tagFrontmatter && candidatesReady) { if (!this.source.tags || !asynchronous) { this.state = Object.freeze({ status: 'unavailable', message: 'Tag metadata freshness is unavailable' }); this.publish(); return } const notes = noteState?.status === 'ready' ? noteState.notes : [] const active = mode === 'both' ? new Set(notes.map(note => note.documentId)) : undefined let typed: readonly TypedNodeCatalogRow[] = snapshot.rows if (mode === 'notes') typed = [] else if (active) typed = snapshot.rows.filter(row => active.has(row.documentId)) const urls = [...new Set([...typed.map(row => row.documentId), ...notes.map(note => note.documentId)])] this.state = Object.freeze({ status: 'indexing' }); this.publish() const tags = this.source.tags const frontmatter = new Map<AutomergeUrl, FrontmatterProjection>() let cursor = 0 const step = async (): Promise<void> => { if (generation !== this.generation) return try { const started = performance.now() while (cursor < urls.length && performance.now() - started < CONTROLLER_SLICE_MS) { const url = urls[cursor++]! const heads = await tags.readHeads(url) if (generation !== this.generation) return const indexed = tags.indexedHeads(url) const catalog = mode === 'notes' ? undefined : tags.catalogHeads?.(url) if (!heads?.length || !indexed?.length || !tags.hasCache(url) || !headsEqual(heads, indexed) || (catalog && !headsEqual(heads, catalog))) { this.state = Object.freeze({ status: 'indexing' }); this.publish(); return } const metadata = tags.frontmatter(url) frontmatter.set(url, { ...(needsTags(compiledTags) ? { tags: frontmatterTags(metadata) } : {}), ...(needsStatus(compiledTags) ? { status: frontmatterStatus(metadata) } : {}), }) } if (cursor < urls.length) { this.timer = setTimeout(() => { void step() }, 0); return } this.startEvaluation(generation, asynchronous, frontmatter) } catch (error) { this.publishFault(generation, error) } } void step() return } this.tagFrontmatter = tagFrontmatter if (mode !== 'records' && !noteState) this.state = Object.freeze({ status: 'unavailable', message: 'Note candidate source is unavailable' }) else if (noteState?.status === 'error') this.state = Object.freeze({ status: 'unavailable', message: noteState.message }) else if (noteState?.status === 'indexing') this.state = Object.freeze({ status: 'indexing' }) else if (snapshot.status === 'disabled' && mode !== 'notes') this.state = Object.freeze({ status: 'disabled' }) else if (snapshot.status === 'indexing' && mode !== 'notes') { this.state = Object.freeze({ status: 'indexing' }) } else if (snapshot.status === 'error' && mode !== 'notes') this.state = Object.freeze({ status: 'unavailable', message: snapshot.error?.message ?? 'Type index is unavailable' }) else { const notes = noteState?.status === 'ready' ? noteState.notes : [] const noteById = new Map(notes.map(note => [note.documentId, note])) let sourceRows: readonly TypedNodeCatalogRow[] = [] if (mode === 'records') sourceRows = snapshot.rows else if (mode === 'both') sourceRows = snapshot.rows .filter(row => noteById.has(row.documentId)) .map(row => { const note = noteById.get(row.documentId)! return row.path === note.path && row.name === note.name ? row : { ...row, path: note.path, name: note.name } }) this.work.fullProjectedRows += sourceRows.length try { if (asynchronous) { this.startAsyncEvaluation(generation, sourceRows, notes, snapshot.revision, tagFrontmatter, projection); return } const sourceById = new Map<string, TypeQuerySubjectSource>() const metrics: ProjectionMetrics = { nodes: 0, bytes: 0, footprints: new Map() } const typedRows = projectTypedRows(sourceRows, metrics, tagFrontmatter, projection, this.resolveFileTitle) for (const row of sourceRows) captureSubjectSource(sourceById, row, this.navigationTitle(row)) const inputRows = mode === 'records' ? typedRows : [...typedRows] if (mode !== 'records') for (const note of notes) { const pending = prepareNote(inputRows, sourceById, note, metrics, tagFrontmatter, projection?.broadStatus, this.resolveFileTitle) if (!pending) continue while (!pending.walker.done) pending.walker.step(BASES_LIMITS.sliceValueNodes) appendNote(inputRows as BasesRowInput[], sourceById, pending) } const compiled = compileBasesQuery(this.query, this.selectedView) if (!('__brand' in compiled)) { this.state = Object.freeze({ status: compiled.status === 'ready' ? 'error' : compiled.status, query: this.query, ...this.selectionState(), diagnostics: 'diagnostics' in compiled ? compiled.diagnostics : [], results: emptyResults() }) this.publish() return } const runner = createBasesRunner(mode === 'records' ? typedRows : brandTrustedBasesSnapshot(inputRows), compiled, this.temporalFormats) this.runner = runner if (asynchronous) { const runSlice = (): void => { if (generation !== this.generation) { runner.cancel(); return } const runState = runner.step() if (runState.status === 'running') { this.timer = setTimeout(runSlice, 0); return } if (runState.status !== 'complete' || !runState.result || runState.result.status !== 'ready' || generation !== this.generation) { return } this.publishResult(sourceById, runState.result, true, generation, snapshot.revision, compiled, metrics, runner.getGeneratedBytes()) } runSlice() return } const result = runner.drain() if (result.status === 'ready') this.publishResult(sourceById, result, false, this.generation, snapshot.revision, compiled, metrics, runner.getGeneratedBytes()) else { this.state = Object.freeze({ status: result.status, query: this.query, ...this.selectionState(), diagnostics: result.diagnostics, results: emptyResults() }); this.publish() } return } catch (error) { const message = error instanceof Error ? error.message : 'Type catalog projection exceeded its limits' this.state = Object.freeze({ status: 'error', query: this.query, ...this.selectionState(), diagnostics: Object.freeze([Object.freeze({ severity: 'error' as const, code: error instanceof ProjectionLimitError ? error.code : 'limit.projection', message, path: Object.freeze([]) })]), results: emptyResults() }) this.publish() return } return } this.publish() }
private startAsyncEvaluation(generation: number, sourceRows: readonly TypedNodeCatalogRow[], notes: readonly BaseNoteCandidate[], snapshotRevision: number, frontmatter?: ReadonlyMap<AutomergeUrl, FrontmatterProjection>, demand?: FactProjection): void { const sourceById = new Map<string, TypeQuerySubjectSource>() const projectedRows: BasesRowInput[] = [] let projection: ProjectionRunner try { projection = createProjectionRunner(sourceRows, row => captureSubjectSource(sourceById, row, this.navigationTitle(row)), frontmatter, demand, this.resolveFileTitle) } catch (error) { this.publishFault(generation, error) return } let noteCursor = 0 let pendingNote: PendingNote | undefined const project = (): void => { if (generation !== this.generation) { projection.cancel(); return } try { const startedAt = performance.now() let steps = 0 let projectionStep: ProjectionStep do { projectionStep = projection.step() projectedRows.push(...projectionStep.rows) steps += 1 } while (!projectionStep.done && hasSliceBudget(startedAt, steps)) if (!projectionStep.done) { this.timer = setTimeout(project, 0); return } while (noteCursor < notes.length && hasSliceBudget(startedAt, steps)) { pendingNote ??= prepareNote(projectedRows, sourceById, notes[noteCursor]!, projection.counters, frontmatter, demand?.broadStatus, this.resolveFileTitle) if (!pendingNote) { noteCursor++; steps++; continue } steps += pendingNote.walker.step(BASES_LIMITS.sliceValueNodes) if (!pendingNote.walker.done) break appendNote(projectedRows, sourceById, pendingNote) noteCursor++ pendingNote = undefined } if (noteCursor < notes.length) { this.timer = setTimeout(project, 0); return } const compiled = compileBasesQuery(this.query, this.selectedView) if (!('__brand' in compiled)) { this.setDiagnosticState(compiled.status === 'invalid' ? 'invalid' : 'error', 'diagnostics' in compiled ? compiled.diagnostics : []); return } const runner = createBasesRunner(brandTrustedBasesSnapshot(projectedRows), compiled, this.temporalFormats); this.runner = runner const run = (): void => { if (generation !== this.generation) { runner.cancel(); return } try { const startedAt = performance.now() let steps = 0 let runState = runner.step() while (runState.status === 'running' && hasSliceBudget(startedAt, ++steps)) runState = runner.step() if (runState.status === 'running') { this.timer = setTimeout(run, 0); return } if (runState.result?.status === 'ready') this.publishTagResult(sourceById, runState.result, generation, snapshotRevision, compiled, { nodes: projection.counters.nodes, bytes: projection.counters.bytes, footprints: new Map(projection.footprints), }, runner.getGeneratedBytes()) else if (runState.result && 'diagnostics' in runState.result) this.setDiagnosticState(runState.result.status === 'invalid' ? 'invalid' : 'error', runState.result.diagnostics) } catch (error) { this.publishFault(generation, error) } } run() } catch (error) { this.publishFault(generation, error) } } project() }
private publishTagResult(sourceById: ReadonlyMap<string, TypeQuerySubjectSource>, result: Extract<import('./bases-contract').BasesEvaluationResult, { status: 'ready' }>, generation: number, revision: number, plan: BasesQueryPlan, metrics: ProjectionMetrics, bytes: number): void { const frontmatter = this.tagFrontmatter const tags = this.source.tags if (!frontmatter || !tags) { this.publishResult(sourceById, result, true, generation, revision, plan, metrics, bytes); return } // Recheck the authoritative frontier after projection/evaluation; a write // between the initial gate and publication cannot leak a stale exclusion. const check = async () => { try { for (const url of frontmatter.keys()) { const heads = await tags.readHeads(url) if (generation !== this.generation) return if (!heads?.length || !tags.hasCache(url) || !headsEqual(heads, tags.indexedHeads(url) ?? []) || (tags.catalogHeads?.(url) && !headsEqual(heads, tags.catalogHeads(url)!))) { this.scheduleEvaluation(); return } } if (generation === this.generation) this.publishResult(sourceById, result, true, generation, revision, plan, metrics, bytes) } catch (error) { this.publishFault(generation, error) } } void check() }
private setDiagnosticState(status: 'invalid' | 'error', diagnostics: readonly BasesYamlDiagnostic[]): void { this.state = Object.freeze({ status, query: this.query, ...this.selectionState(), diagnostics, results: emptyResults() }); this.publish() }
private publishFault(generation: number, error: unknown): void { if (generation !== this.generation) return const message = error instanceof Error ? error.message : 'Type catalog evaluation failed' this.state = Object.freeze({ status: 'error', query: this.query, ...this.selectionState(), diagnostics: Object.freeze([Object.freeze({ severity: 'error' as const, code: error instanceof ProjectionLimitError || error instanceof IncompatibleFrontmatterStatusError || error instanceof IncompatibleRecordStatusError ? error.code : 'execution.fault', message, path: Object.freeze([]) })]), results: emptyResults() }); this.publish() }
private publishResult(sourceById: ReadonlyMap<string, TypeQuerySubjectSource>, result: Extract<import('./bases-contract').BasesEvaluationResult, { status: 'ready' }>, asynchronous: boolean, generation: number, snapshotRevision: number, plan: BasesQueryPlan, metrics: ProjectionMetrics, generatedBytes: number): void { const results: TypeQueryResultRow[] = [] let cursor = 0 const publishSlice = (): void => { if (generation !== this.generation) return const startedAt = performance.now() let steps = 0 do { const end = Math.min(cursor + BASES_LIMITS.batchRows, result.rows.length) while (cursor < end) { const evaluated = result.rows[cursor++]! const source = sourceById.get(evaluated.id) if (source) results.push(resultRow(evaluated, source)) } steps += 1 } while (cursor < result.rows.length && (!asynchronous || hasSliceBudget(startedAt, steps))) if (cursor < result.rows.length) { if (asynchronous) this.timer = setTimeout(publishSlice, 0) else publishSlice() return } this.resultIndexById = new Map(results.map((row, index) => [row.evaluated.id, index])) this.state = Object.freeze({ status: 'ready', query: this.query, ...this.selectionState(), columns: result.columns, ...(result.orderedRowIds ? { orderedRowIds: result.orderedRowIds } : {}), results: Object.freeze(results) }) this.evaluatedRevision = snapshotRevision this.evaluatedQuery = this.query this.evaluatedSelectedView = this.selectedView this.evaluatedPlan = plan this.projectionMetrics = metrics this.evaluatedGeneratedBytes = generatedBytes this.publish() if ((globalThis as { __reamsPerfDiagnostics__?: unknown }).__reamsPerfDiagnostics__) globalThis.performance?.mark('reams:record-edit-base') } publishSlice() }
private publish(): void { for (const listener of [...this.listeners]) { try { listener() } catch { /* isolate observers */ } } }}