diff --git a/experiments/event-shape-study/README.md b/experiments/event-shape-study/README.md index 257ffb9..49a2063 100644 --- a/experiments/event-shape-study/README.md +++ b/experiments/event-shape-study/README.md @@ -11,14 +11,15 @@ Current status: - `current-baseline` is the control condition and now has its own approach-local code snapshot. - `shared-props` improves retained memory, but it does not clear the timing bar. +- `lazy-position` isolates lazy `position` materialization and regresses timing even more decisively than the baseline study bar allows. - `lazy-position-shared-props` regresses timing badly and is rejected. - `planned-flat-eager-event-shape` has now been tested and rejected. Every approach directory now also carries an approach-local `code/` snapshot and a checked-in -`artifacts/stress-mixed-16MiB.json` file for comparable large-input smoke coverage. The branch now -also has the full drafted experiment toolchain again: one-shot large-input stress, session-oriented -large-input stress, a budgeted `1 GiB` mitata lane, and a preflight inventory that confirms the -study surface is complete before new collection starts. +`artifacts/stress-mixed-16MiB.json` file for comparable large-input smoke coverage. This branch's +documented toolchain now covers the standard comparison flow, the deterministic round-robin check, +the one-shot large-input stress lane, and a preflight inventory that confirms those study surfaces +exist before new collection starts. The newest artifact paths both reject the candidate: @@ -30,10 +31,9 @@ important point: this candidate still does not clear the `+5%` acceptance bar an regresses retained memory overall. Important scope note: the current checked-in comparisons are statistically grounded for the -standard study sizes, not for `1 GiB`-class stress inputs. Large-input work now has three separate -study-local collection lanes documented in [methods.md](methods.md): one-shot stress artifacts, -session-oriented stress artifacts, and a budgeted `1 GiB` mitata lane. The current branch only keeps -the lighter `16 MiB` mixed-article smoke artifact per approach checked in so far. +standard study sizes, not for `1 GiB`-class stress inputs. The current branch documents and ships +only the one-shot large-input stress lane from the broader draft. It still keeps just the lighter +`16 MiB` mixed-article smoke artifact per approach checked in by default. The study uses this decision rule: diff --git a/experiments/event-shape-study/methods.md b/experiments/event-shape-study/methods.md index 3c0c6fb..910641d 100644 --- a/experiments/event-shape-study/methods.md +++ b/experiments/event-shape-study/methods.md @@ -104,8 +104,8 @@ budget or the acceptance gate. The checked-in archive currently keeps one cheaper large-input smoke artifact per approach at `16 MiB` under `artifacts/stress-mixed-16MiB.json`. That proves the archive path end to end. -The broader scenario matrix is now driven by tooling rather than being implied by the current -checked-in artifact count. +The broader one-shot scenario matrix is now driven by tooling rather than being implied by the +current checked-in artifact count. Use the batch runner when you want to fill a one-shot scenario matrix instead of one-off files: @@ -180,169 +180,6 @@ The one-shot stress collector records per-case failures inside the JSON artifact the whole file when one runtime exception hits a pathological input. That matters because a recorded failure tells us more than a missing artifact would. -## Large-input session stress runs - -One-shot large-input stress answers whether an approach survives large text and how much whole-run -work it does. Session stress answers a different question: whether the same event shape helps when -callers keep a session alive and ask for outline, events, parse, and diagnostics results more than -once. - -The session collector mirrors the case semantics in `session_bench.ts` for the first study-local -workload set: - -- `session.outline() cold` -- `session.outline() warm` -- `session.events() cold` -- `session.events() warm` -- `session.parse() cold` -- `session.parse() warm` -- `session.parseWithDiagnostics() cold` -- `session.parseWithDiagnostics() warm` -- `consumer workflow: session outline -> events -> parse cold` -- `consumer workflow: session outline -> events -> parse warm` - -The session lane now uses the same ten scenario families as the one-shot matrix. -That keeps `16 MiB` and `1 GiB` session coverage aligned with the broader large-input archive, -even though the checked-in session artifacts are still much sparser than the implemented surface. - -Collect one session-stress artifact for one approach: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-sys \ - --v8-flags=--expose-gc \ - experiments/event-shape-study/tools/collect_large_input_session_stress_report.ts \ - --approach-dir=current-baseline \ - --variant=current-baseline \ - --scenario=mixed-article \ - --size-mib=16 \ - --repeats=1 \ - --out=experiments/event-shape-study/current-baseline/artifacts/session-stress-mixed-16MiB.json -``` - -Use the session batch runner when you want the same session scenario set across several approaches: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-run --allow-sys \ - experiments/event-shape-study/tools/run_large_input_session_stress_matrix.ts \ - --approaches=current-baseline,shared-props \ - --scenarios=plain-paragraphs,mixed-article,pathological-recovery,table-heavy,template-heavy,inline-heavy,uri-heavy,unicode-heavy,synthetic-article,outline-heavy \ - --sizes-mib=16 \ - --repeats=1 -``` - -The session matrix runner now gives child collectors a larger V8 heap budget by default with -`--max-old-space-size-mib=8192`. Override that flag if you need a different ceiling for a -particular machine or scenario lane. - -The first full `16 MiB` matrix attempt on this branch exposed a practical budget issue: -`current-baseline` on `pathological-recovery` kept one CPU core busy for more than eleven -minutes before the run was stopped. Treat that scenario as a separately budgeted lane rather -than assuming it belongs in the same quick follow-up batch as `mixed-article` and `synthetic-article`. - -The current checked-in session archive covers only the first non-pathological `16 MiB` slice for all four -approaches: - -- `mixed-article` -- `synthetic-article` - -`pathological-recovery` remains intentionally incomplete on this branch until it is rerun with a -separate time budget. - -Warm session cases intentionally include the cache-priming step before the measured second access. -That matches the benchmark helper semantics in `session_bench.ts`, so the checked-in study artifacts -and the benchmark file describe the same workload shape. - -## Budgeted large-input mitata runs - -The one-shot stress collector is useful for survivability and scale diagnostics, but it does not use -mitata. For the `1 GiB` tier, the study now also has a budgeted mitata lane for streaming parser work. -Its purpose is narrower than the standard event-shape benchmark suite: - -- keep the command under about two minutes on the study machine -- stress real parser traversal on a `1 GiB` input -- stay on streaming parser paths instead of requiring full tree materialization - -The current mitata large-input collector measures two streaming cases: - -- `outlineEvents() streamed count` -- `events() streamed count` - -The collector now lowers mitata's minimum sample count to `1` by default for this lane and records -the checksum from the first measured execution instead of doing a separate full pre-pass. That keeps -the `1 GiB` collector much closer to its intended command budget on expensive scenarios. - -The mitata lane now uses the same ten scenario families as the one-shot and session lanes. The -implementation surface is no longer narrower than the rest of the study, and the checked-in `1 GiB` -archive now covers every non-pathological scenario across all four approaches. - -For limit-study sweeps, the matrix runner also supports `--continue-on-error` so one failing size or -approach does not hide the rest of the boundary. That mode still exits non-zero at the end, but it -keeps going and prints a structured failure summary for every failed job. - -Even with those mitigations, `pathological-recovery` at `1 GiB` currently exceeds the practical -survivability budget for `current-baseline` on this machine: the direct collector run was hard-killed -before it could write an artifact. Treat that scenario as a separate limit study rather than assuming -it belongs in the same normal archive pass as `plain-paragraphs`, `mixed-article`, and `synthetic-article`. - -The follow-up limit-study sweeps showed that the failure is not just a `1 GiB` budget miss. -On this branch and machine: - -- `256 MiB` baseline `pathological-recovery` overran a nominal `60s` budget for more than thirty minutes before it was stopped -- `64 MiB` baseline `pathological-recovery` failed with `RangeError: Maximum call stack size exceeded` -- a cross-approach `1/2/4 MiB` limit-study sweep failed for all four approaches with the same stack-overflow shape - -That makes `pathological-recovery` an explicit parser-limit lane, not merely an unfinished archive row. - -The current checked-in `1 GiB` mitata archive covers the non-pathological lane for all four approaches: - -- `plain-paragraphs` -- `mixed-article` -- `table-heavy` -- `template-heavy` -- `inline-heavy` -- `uri-heavy` -- `unicode-heavy` -- `synthetic-article` -- `outline-heavy` - -Collect one `1 GiB` mitata stress artifact for one approach: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-sys --allow-env=NODE_DISABLE_COLORS \ - --v8-flags=--expose-gc,--max-old-space-size=8192 \ - experiments/event-shape-study/tools/collect_large_input_mitata_report.ts \ - --approach-dir=current-baseline \ - --variant=current-baseline \ - --scenario=mixed-article \ - --size-mib=1024 \ - --total-budget-seconds=110 \ - --out=experiments/event-shape-study/current-baseline/artifacts/mitata-stress-mixed-1GiB.json -``` - -Use the matrix runner when you want the same `1 GiB` mitata stress lane across several approaches: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-run --allow-sys --allow-env=NODE_DISABLE_COLORS \ - experiments/event-shape-study/tools/run_large_input_mitata_matrix.ts \ - --approaches=current-baseline,shared-props \ - --scenarios=plain-paragraphs,mixed-article,pathological-recovery,table-heavy,template-heavy,inline-heavy,uri-heavy,unicode-heavy,synthetic-article,outline-heavy \ - --sizes-mib=1024 \ - --total-budget-seconds=110 -``` - -The matrix runner shells out through `mise`, so it needs `--allow-run`. The single-run collector does not. - -When you need a quick inventory before choosing the next batch, summarize the archive directly: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read \ - experiments/event-shape-study/tools/summarize_large_input_archive.ts \ - --size-mib=1024 -``` - -That summary groups one-shot stress, session stress, and mitata stress coverage by approach so the -next run can target missing scenarios instead of rescanning artifact directories by hand. - If you want to verify the study is fully built before collecting anything new, run the preflight inventory: ```bash @@ -350,6 +187,10 @@ mise x deno@latest -- deno run --no-lock --allow-read \ experiments/event-shape-study/tools/validate_experiment_matrix.ts ``` +This preflight only checks the runnable surface that exists on this branch: the core study docs, +the standard comparison tools, the one-shot large-input tools, the approach-local `code/mod.ts` +snapshots, and the checked-in `16 MiB` mixed-article smoke artifacts. + ## Decision rule The acceptance rule stays the same across all approaches: diff --git a/experiments/event-shape-study/results.md b/experiments/event-shape-study/results.md index 0ed94cf..52889eb 100644 --- a/experiments/event-shape-study/results.md +++ b/experiments/event-shape-study/results.md @@ -7,44 +7,41 @@ checked-in artifacts currently support. |---|---:|---:|---:|---| | `current-baseline` | control | control | control | keep as baseline | | `shared-props` | +0.81% | +4.02% | -0.24% | reject | +| `lazy-position` | -112.62% | -185.19% | -124.96% | reject | | `lazy-position-shared-props` | -107.05% | -0.08% | -149.81% | reject | | `planned-flat-eager-event-shape` | -1.49% | -3.97% | -3.55% | reject | Interpretation: - `shared-props` gives a real memory improvement, but the median target timing win is too small to clear the 5% acceptance bar. +- `lazy-position` shows that getter-based lazy `position` materialization is itself a decisive negative result on this code path, even without shared props mixed in. - `lazy-position-shared-props` fails hard on timing and is useful mainly as a negative result. - `planned-flat-eager-event-shape` keeps all changes inside its approach-local snapshot, but the refreshed direct comparison still lands below the timing bar and regresses retained memory. - the deterministic round-robin check for `planned-flat-eager-event-shape` shifts the target timing median to `+0.95%` with `2/9` significant target wins, but it still misses the acceptance bar and keeps the same memory regression. Archive completion summary: -| Lane | Scenario set | Implemented | Collected across all four approaches | Current status | +| Lane | Scenario set | Implemented | Collected across all five approaches | Current status | |---|---|---:|---:|---| -| Standard comparison | event-shape timing + retained memory | yes | yes | complete | -| `16 MiB` one-shot stress | ten scenarios | yes | partial | only `mixed-article` is checked in across all four approaches | -| `16 MiB` session stress | ten scenarios | yes | partial | `mixed-article` and `synthetic-article` are checked in across all four approaches | -| `1 GiB` mitata stress | ten scenarios | yes | yes for all non-pathological scenarios | nine normal scenarios are complete across all four approaches; only `pathological-recovery` remains outside the normal archive | -| `1 GiB` one-shot stress | ten scenarios | yes | no | archive not started | -| `1 GiB` session stress | ten scenarios | yes | no | archive not started | -| Pathological limit lane | separate parser-limit study | yes | n/a | treat as limit study, not as a normal missing archive row | +| Standard comparison | event-shape timing + retained memory | yes | yes | complete across five approaches | +| `16 MiB` one-shot stress | ten scenarios | yes | partial | only `mixed-article` is checked in across all five approaches | +| `1 GiB` one-shot stress | ten scenarios | yes | no | tooling exists, but the archive is not checked in on this branch | +| Session stress | n/a on this branch | no | no | not implemented on this branch | +| Mitata large-input stress | n/a on this branch | no | no | not implemented on this branch | Large-input archive status: -- comparable `16 MiB` mixed-article one-shot smoke artifacts exist for all four approaches +- comparable `16 MiB` mixed-article one-shot smoke artifacts exist for all five approaches - full one-shot matrix tooling now exists for ten scenario families -- session-stress tooling now exists for the same ten scenario families as the one-shot lane -- budgeted `1 GiB` mitata tooling now exists for the same ten scenario families as the one-shot lane -- the non-pathological `1 GiB` mitata archive is now complete across all four approaches for `plain-paragraphs`, `mixed-article`, `table-heavy`, `template-heavy`, `inline-heavy`, `uri-heavy`, `unicode-heavy`, `synthetic-article`, and `outline-heavy` -- `pathological-recovery` is now clearly a separate parser-limit lane rather than an ordinary large-input benchmark row: baseline was hard-killed at `1 GiB`, baseline `256 MiB` overran a `60s` budget for more than thirty minutes before it was stopped, and a cross-approach `1/2/4 MiB` mitata sweep failed for all four approaches with `RangeError: Maximum call stack size exceeded` -- `16 MiB` session-stress artifacts for `mixed-article` and `synthetic-article` now exist for all four approaches -- the first `16 MiB` session-matrix attempt showed that `pathological-recovery` is not part of the quick lane on this branch: `current-baseline` was still consuming a full CPU core after more than eleven minutes before the run was stopped -- the broader one-shot matrix and the session matrix are still not filled yet on this branch, but the normal `1 GiB` mitata lane is now fully checked in +- a structural preflight now checks that the runnable study surface exists before new collection starts +- session-stress and mitata large-input lanes are part of the longer draft, but they are not implemented on this branch +- the broader one-shot matrix is still mostly unfilled on this branch beyond the checked-in `mixed-article` smoke artifacts Primary artifacts: - [current-baseline/artifacts/report.json](current-baseline/artifacts/report.json) - [shared-props/artifacts/comparison.txt](shared-props/artifacts/comparison.txt) +- [lazy-position/artifacts/comparison.txt](lazy-position/artifacts/comparison.txt) - [lazy-position-shared-props/artifacts/comparison.txt](lazy-position-shared-props/artifacts/comparison.txt) - [planned-flat-eager-event-shape/artifacts/comparison.txt](planned-flat-eager-event-shape/artifacts/comparison.txt) - [cross-candidate-runs/2026-05-17-baseline-vs-flat-eager-round-robin-02/comparisons/round-01--current-baseline--vs--planned-flat-eager-event-shape.txt](cross-candidate-runs/2026-05-17-baseline-vs-flat-eager-round-robin-02/comparisons/round-01--current-baseline--vs--planned-flat-eager-event-shape.txt) \ No newline at end of file diff --git a/experiments/event-shape-study/scenario-roadmap.md b/experiments/event-shape-study/scenario-roadmap.md new file mode 100644 index 0000000..036e85d --- /dev/null +++ b/experiments/event-shape-study/scenario-roadmap.md @@ -0,0 +1,54 @@ +# Scenario Roadmap + +This roadmap names the study lanes that are real on this branch, the ones that are only +partially archived, and the highest-value work still missing from the original event-shape plan. + +## Current branch surface + +The branch supports four runnable study layers today: + +- the standard snapshot-local comparison flow in [methods.md](methods.md) +- the deterministic round-robin check in [methods.md](methods.md) +- the one-shot large-input stress collector and matrix runner in [methods.md](methods.md) +- the structural preflight inventory in [methods.md](methods.md) + +The checked-in large-input archive is intentionally narrow. Each approach keeps one comparable +`16 MiB` `mixed-article` smoke artifact so the archive path stays proven without turning the repo +into a dump of heavyweight stress output. + +## Standard comparison status + +The standard acceptance study is complete for the five checked-in approaches: + +- `current-baseline` +- `shared-props` +- `lazy-position` +- `lazy-position-shared-props` +- `planned-flat-eager-event-shape` + +Those runs are enough to say the current candidates do not clear the acceptance bar documented in +[protocol.md](protocol.md). + +## Large-input archive status + +The one-shot large-input lane is implemented, but the archive is only partially filled: + +- `16 MiB` `mixed-article` smoke artifacts exist for all five approaches +- the broader one-shot matrix can now be rerun on demand instead of being described only in prose +- session-oriented large-input tooling is not present on this branch +- budgeted mitata large-input tooling is not present on this branch + +That means the branch is now internally consistent, but it is still not the final archive shape +from the longer study draft. + +## Highest-value remaining work + +The original event-shape plan now has the missing lazy-position-only verdict filled in, and that +result is a clear rejection. The highest-value remaining work is now archive completion rather than +another first-order event-shape hypothesis. + +If the goal is to finish the study cleanly, the next work should happen in this order: + +1. Fill the non-pathological `16 MiB` one-shot matrix for the five checked-in approaches. +2. Decide whether session and mitata lanes belong on this branch as runnable tooling or only in a later follow-up branch. +3. If lazy positions are revisited at all, test a materially different layout that avoids getter-based per-event memoization. \ No newline at end of file diff --git a/experiments/event-shape-study/tools/run_large_input_stress_matrix.ts b/experiments/event-shape-study/tools/run_large_input_stress_matrix.ts new file mode 100644 index 0000000..90a8518 --- /dev/null +++ b/experiments/event-shape-study/tools/run_large_input_stress_matrix.ts @@ -0,0 +1,241 @@ +/** + * Run the large-input one-shot stress collector across a scenario and size matrix. + * + * This keeps the branch's documented large-input workflow runnable without requiring + * callers to hand-write one collector command per approach, scenario, and size. + */ + +import { fromFileUrl, join, relative } from 'jsr:@std/path'; + +type JobResult = { + approach: string; + scenario: string; + size_mib: number; + profile: 'full' | 'streaming-only'; + output_path: string; + status: 'ok' | 'failed'; + error_message?: string; +}; + +type MatrixRunSummary = { + study: 'event-shape'; + generated_at: string; + approaches: string[]; + scenarios: string[]; + sizes_mib: number[]; + repeats: number; + gib_profile: 'full' | 'streaming-only'; + continue_on_error: boolean; + jobs: JobResult[]; + commands: string[]; +}; + +const STUDY_ROOT = fromFileUrl(new URL('../', import.meta.url)); +const REPO_ROOT = fromFileUrl(new URL('../../../', import.meta.url)); +const DEFAULT_SCENARIOS = [ + 'plain-paragraphs', + 'mixed-article', + 'pathological-recovery', + 'table-heavy', + 'template-heavy', + 'inline-heavy', + 'uri-heavy', + 'unicode-heavy', + 'synthetic-article', + 'outline-heavy', +] as const; + +function getFlag(name: string): string | undefined { + return Deno.args.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3); +} + +function hasFlag(name: string): boolean { + return Deno.args.includes(`--${name}`); +} + +function parseCsvFlag(name: string, fallback: readonly string[]): string[] { + const raw = getFlag(name); + if (raw === undefined) { + return [...fallback]; + } + + const values = raw.split(',').map((value) => value.trim()).filter((value) => value.length > 0); + if (values.length === 0) { + throw new Error(`expected --${name} to contain at least one comma-separated value`); + } + + return values; +} + +function parsePositiveIntFlag(name: string, fallback: number): number { + const raw = getFlag(name); + if (raw === undefined) { + return fallback; + } + + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`expected --${name} to be a positive integer, got: ${raw}`); + } + + return value; +} + +function parseSizeListFlag(name: string, fallback: readonly number[]): number[] { + const raw = getFlag(name); + if (raw === undefined) { + return [...fallback]; + } + + return raw.split(',').map((value) => { + const parsed = Number(value.trim()); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`expected --${name} to contain positive integers, got: ${raw}`); + } + + return parsed; + }); +} + +function parseProfileFlag(name: string, fallback: 'full' | 'streaming-only'): 'full' | 'streaming-only' { + const raw = getFlag(name); + if (raw === undefined) { + return fallback; + } + + if (raw !== 'full' && raw !== 'streaming-only') { + throw new Error(`expected --${name} to be full or streaming-only, got: ${raw}`); + } + + return raw; +} + +function formatSizeLabel(size_mib: number): string { + if (size_mib % 1024 === 0) { + return `${size_mib / 1024}GiB`; + } + + return `${size_mib}MiB`; +} + +function buildOutputPath( + approach: string, + scenario: string, + size_mib: number, + profile: 'full' | 'streaming-only', +): string { + const suffix = formatSizeLabel(size_mib); + const file_name = size_mib >= 1024 && profile === 'full' + ? `stress-${scenario}-${suffix}-full.json` + : `stress-${scenario}-${suffix}.json`; + + return join(STUDY_ROOT, approach, 'artifacts', file_name); +} + +async function runCommand(args: string[]): Promise { + const command = new Deno.Command('mise', { + args, + cwd: REPO_ROOT, + stdout: 'piped', + stderr: 'piped', + env: { + NODE_DISABLE_COLORS: '1', + }, + }); + const output = await command.output(); + const stdout = new TextDecoder().decode(output.stdout); + const stderr = new TextDecoder().decode(output.stderr); + + if (!output.success) { + throw new Error(`command failed: mise ${args.join(' ')}\n${stdout}\n${stderr}`); + } +} + +const approaches = parseCsvFlag('approaches', []); +if (approaches.length === 0) { + throw new Error('expected --approaches to contain at least one approach directory'); +} + +const scenarios = parseCsvFlag('scenarios', DEFAULT_SCENARIOS); +const sizes_mib = parseSizeListFlag('sizes-mib', [16, 1024]); +const repeats = parsePositiveIntFlag('repeats', 1); +const gib_profile = parseProfileFlag('gib-profile', 'streaming-only'); +const continue_on_error = hasFlag('continue-on-error'); + +const commands: string[] = []; +const jobs: JobResult[] = []; + +for (const approach of approaches) { + for (const scenario of scenarios) { + for (const size_mib of sizes_mib) { + const profile = size_mib >= 1024 ? gib_profile : 'full'; + const output_path = buildOutputPath(approach, scenario, size_mib, profile); + const v8_flags = size_mib >= 1024 + ? '--v8-flags=--expose-gc,--max-old-space-size=8192' + : '--v8-flags=--expose-gc'; + const args = [ + 'x', + 'deno@latest', + '--', + 'deno', + 'run', + '--no-lock', + '--allow-read', + '--allow-write', + '--allow-sys', + v8_flags, + 'experiments/event-shape-study/tools/collect_large_input_stress_report.ts', + `--approach-dir=${approach}`, + `--variant=${approach.split('/').at(-1) ?? approach}`, + `--scenario=${scenario}`, + `--profile=${profile}`, + `--size-mib=${size_mib}`, + `--repeats=${repeats}`, + `--out=${relative(REPO_ROOT, output_path)}`, + ]; + commands.push(`mise ${args.join(' ')}`); + + try { + await runCommand(args); + jobs.push({ + approach, + scenario, + size_mib, + profile, + output_path: relative(REPO_ROOT, output_path), + status: 'ok', + }); + } catch (error) { + const error_message = error instanceof Error ? error.message : String(error); + jobs.push({ + approach, + scenario, + size_mib, + profile, + output_path: relative(REPO_ROOT, output_path), + status: 'failed', + error_message, + }); + + if (!continue_on_error) { + throw error; + } + } + } + } +} + +const summary: MatrixRunSummary = { + study: 'event-shape', + generated_at: new Date().toISOString(), + approaches, + scenarios, + sizes_mib, + repeats, + gib_profile, + continue_on_error, + jobs, + commands, +}; + +console.log(JSON.stringify(summary, null, 2)); \ No newline at end of file diff --git a/experiments/event-shape-study/tools/validate_experiment_matrix.ts b/experiments/event-shape-study/tools/validate_experiment_matrix.ts new file mode 100644 index 0000000..dfc2539 --- /dev/null +++ b/experiments/event-shape-study/tools/validate_experiment_matrix.ts @@ -0,0 +1,96 @@ +/** + * Validate that the documented event-shape study surface exists on this branch. + * + * This is a structural preflight. It does not rerun benchmarks. It checks that the + * core docs, study tools, approach-local snapshots, and baseline checked-in smoke + * artifacts are present before new collection starts. + */ + +import { fromFileUrl, join, relative } from 'jsr:@std/path'; + +type CheckResult = { + path: string; + status: 'ok' | 'missing'; + kind: 'doc' | 'tool' | 'approach-file' | 'artifact'; +}; + +const STUDY_ROOT = fromFileUrl(new URL('../', import.meta.url)); +const REPO_ROOT = fromFileUrl(new URL('../../../', import.meta.url)); +const APPROACHES = [ + 'current-baseline', + 'shared-props', + 'lazy-position', + 'lazy-position-shared-props', + 'planned-flat-eager-event-shape', +] as const; +const DOC_PATHS = [ + 'README.md', + 'methods.md', + 'protocol.md', + 'results.md', + 'scenario-roadmap.md', +] as const; +const TOOL_PATHS = [ + 'tools/collect_approach_report.ts', + 'tools/collect_large_input_stress_report.ts', + 'tools/compare_reports.ts', + 'tools/record_approach_artifacts.ts', + 'tools/run_cross_candidate_schedule.ts', + 'tools/run_large_input_stress_matrix.ts', + 'tools/validate_experiment_matrix.ts', +] as const; + +async function pathExists(path: string): Promise { + try { + await Deno.stat(path); + return true; + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + return false; + } + + throw error; + } +} + +async function checkPath(path: string, kind: CheckResult['kind']): Promise { + return { + path: relative(REPO_ROOT, path), + status: await pathExists(path) ? 'ok' : 'missing', + kind, + }; +} + +const results: CheckResult[] = []; + +for (const doc_path of DOC_PATHS) { + results.push(await checkPath(join(STUDY_ROOT, doc_path), 'doc')); +} + +for (const tool_path of TOOL_PATHS) { + results.push(await checkPath(join(STUDY_ROOT, tool_path), 'tool')); +} + +for (const approach of APPROACHES) { + results.push(await checkPath(join(STUDY_ROOT, approach, 'code', 'mod.ts'), 'approach-file')); + results.push(await checkPath(join(STUDY_ROOT, approach, 'artifacts', 'report.json'), 'artifact')); + results.push(await checkPath(join(STUDY_ROOT, approach, 'artifacts', 'stress-mixed-16MiB.json'), 'artifact')); +} + +const missing = results.filter((entry) => entry.status === 'missing'); + +for (const group of ['doc', 'tool', 'approach-file', 'artifact'] as const) { + const group_entries = results.filter((entry) => entry.kind === group); + console.log(`${group}:`); + for (const entry of group_entries) { + console.log(` [${entry.status}] ${entry.path}`); + } + console.log(''); +} + +if (missing.length > 0) { + console.error(`missing ${missing.length} required study paths`); + Deno.exit(1); +} + +console.log(`validated ${results.length} study paths`); \ No newline at end of file