From f2c1009f8e8e706e9e0965941e710a188b77073d Mon Sep 17 00:00:00 2001 From: Okiki Ojo Date: Tue, 19 May 2026 02:30:35 +0000 Subject: [PATCH] bench(event-shape): make the one-shot stress matrix explicit Make the branch's large-input story line up with what is actually runnable today. This adds a batch runner for the one-shot stress collector, a structural preflight that checks the documented study surface before new collection starts, and the study docs that explain what is complete on this branch versus what still belongs to the longer draft. The important behavior change is not a new benchmark result. It is that the branch no longer implies a broader archive than it actually ships. The methods, results, and roadmap now say plainly that this branch implements the standard comparison flow, the deterministic round-robin check, the one-shot large-input lane, and the preflight inventory, while session and mitata lanes are still absent here. The stress matrix tooling also makes the archive plan concrete instead of hand-written. It can sweep the ten documented scenario families, default `1 GiB` runs to the cheaper streaming-only profile, and keep the checked-in `16 MiB` smoke artifacts as the lightweight proof that the path works end to end. Verification: - `mise x deno@latest -- deno check experiments/event-shape-study/tools/run_large_input_stress_matrix.ts experiments/event-shape-study/tools/validate_experiment_matrix.ts` - `mise x deno@latest -- deno run --no-lock --allow-read experiments/event-shape-study/tools/validate_experiment_matrix.ts` --- experiments/event-shape-study/README.md | 16 +- experiments/event-shape-study/methods.md | 171 +------------ experiments/event-shape-study/results.md | 29 +-- .../event-shape-study/scenario-roadmap.md | 54 ++++ .../tools/run_large_input_stress_matrix.ts | 241 ++++++++++++++++++ .../tools/validate_experiment_matrix.ts | 96 +++++++ 6 files changed, 418 insertions(+), 189 deletions(-) create mode 100644 experiments/event-shape-study/scenario-roadmap.md create mode 100644 experiments/event-shape-study/tools/run_large_input_stress_matrix.ts create mode 100644 experiments/event-shape-study/tools/validate_experiment_matrix.ts diff --git a/experiments/event-shape-study/README.md b/experiments/event-shape-study/README.md index 257ffb9..49a2063 100644 --- a/experiments/event-shape-study/README.md +++ b/experiments/event-shape-study/README.md @@ -11,14 +11,15 @@ Current status: - `current-baseline` is the control condition and now has its own approach-local code snapshot. - `shared-props` improves retained memory, but it does not clear the timing bar. +- `lazy-position` isolates lazy `position` materialization and regresses timing even more decisively than the baseline study bar allows. - `lazy-position-shared-props` regresses timing badly and is rejected. - `planned-flat-eager-event-shape` has now been tested and rejected. Every approach directory now also carries an approach-local `code/` snapshot and a checked-in -`artifacts/stress-mixed-16MiB.json` file for comparable large-input smoke coverage. The branch now -also has the full drafted experiment toolchain again: one-shot large-input stress, session-oriented -large-input stress, a budgeted `1 GiB` mitata lane, and a preflight inventory that confirms the -study surface is complete before new collection starts. +`artifacts/stress-mixed-16MiB.json` file for comparable large-input smoke coverage. This branch's +documented toolchain now covers the standard comparison flow, the deterministic round-robin check, +the one-shot large-input stress lane, and a preflight inventory that confirms those study surfaces +exist before new collection starts. The newest artifact paths both reject the candidate: @@ -30,10 +31,9 @@ important point: this candidate still does not clear the `+5%` acceptance bar an regresses retained memory overall. Important scope note: the current checked-in comparisons are statistically grounded for the -standard study sizes, not for `1 GiB`-class stress inputs. Large-input work now has three separate -study-local collection lanes documented in [methods.md](methods.md): one-shot stress artifacts, -session-oriented stress artifacts, and a budgeted `1 GiB` mitata lane. The current branch only keeps -the lighter `16 MiB` mixed-article smoke artifact per approach checked in so far. +standard study sizes, not for `1 GiB`-class stress inputs. The current branch documents and ships +only the one-shot large-input stress lane from the broader draft. It still keeps just the lighter +`16 MiB` mixed-article smoke artifact per approach checked in by default. The study uses this decision rule: diff --git a/experiments/event-shape-study/methods.md b/experiments/event-shape-study/methods.md index 3c0c6fb..910641d 100644 --- a/experiments/event-shape-study/methods.md +++ b/experiments/event-shape-study/methods.md @@ -104,8 +104,8 @@ budget or the acceptance gate. The checked-in archive currently keeps one cheaper large-input smoke artifact per approach at `16 MiB` under `artifacts/stress-mixed-16MiB.json`. That proves the archive path end to end. -The broader scenario matrix is now driven by tooling rather than being implied by the current -checked-in artifact count. +The broader one-shot scenario matrix is now driven by tooling rather than being implied by the +current checked-in artifact count. Use the batch runner when you want to fill a one-shot scenario matrix instead of one-off files: @@ -180,169 +180,6 @@ The one-shot stress collector records per-case failures inside the JSON artifact the whole file when one runtime exception hits a pathological input. That matters because a recorded failure tells us more than a missing artifact would. -## Large-input session stress runs - -One-shot large-input stress answers whether an approach survives large text and how much whole-run -work it does. Session stress answers a different question: whether the same event shape helps when -callers keep a session alive and ask for outline, events, parse, and diagnostics results more than -once. - -The session collector mirrors the case semantics in `session_bench.ts` for the first study-local -workload set: - -- `session.outline() cold` -- `session.outline() warm` -- `session.events() cold` -- `session.events() warm` -- `session.parse() cold` -- `session.parse() warm` -- `session.parseWithDiagnostics() cold` -- `session.parseWithDiagnostics() warm` -- `consumer workflow: session outline -> events -> parse cold` -- `consumer workflow: session outline -> events -> parse warm` - -The session lane now uses the same ten scenario families as the one-shot matrix. -That keeps `16 MiB` and `1 GiB` session coverage aligned with the broader large-input archive, -even though the checked-in session artifacts are still much sparser than the implemented surface. - -Collect one session-stress artifact for one approach: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-sys \ - --v8-flags=--expose-gc \ - experiments/event-shape-study/tools/collect_large_input_session_stress_report.ts \ - --approach-dir=current-baseline \ - --variant=current-baseline \ - --scenario=mixed-article \ - --size-mib=16 \ - --repeats=1 \ - --out=experiments/event-shape-study/current-baseline/artifacts/session-stress-mixed-16MiB.json -``` - -Use the session batch runner when you want the same session scenario set across several approaches: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-run --allow-sys \ - experiments/event-shape-study/tools/run_large_input_session_stress_matrix.ts \ - --approaches=current-baseline,shared-props \ - --scenarios=plain-paragraphs,mixed-article,pathological-recovery,table-heavy,template-heavy,inline-heavy,uri-heavy,unicode-heavy,synthetic-article,outline-heavy \ - --sizes-mib=16 \ - --repeats=1 -``` - -The session matrix runner now gives child collectors a larger V8 heap budget by default with -`--max-old-space-size-mib=8192`. Override that flag if you need a different ceiling for a -particular machine or scenario lane. - -The first full `16 MiB` matrix attempt on this branch exposed a practical budget issue: -`current-baseline` on `pathological-recovery` kept one CPU core busy for more than eleven -minutes before the run was stopped. Treat that scenario as a separately budgeted lane rather -than assuming it belongs in the same quick follow-up batch as `mixed-article` and `synthetic-article`. - -The current checked-in session archive covers only the first non-pathological `16 MiB` slice for all four -approaches: - -- `mixed-article` -- `synthetic-article` - -`pathological-recovery` remains intentionally incomplete on this branch until it is rerun with a -separate time budget. - -Warm session cases intentionally include the cache-priming step before the measured second access. -That matches the benchmark helper semantics in `session_bench.ts`, so the checked-in study artifacts -and the benchmark file describe the same workload shape. - -## Budgeted large-input mitata runs - -The one-shot stress collector is useful for survivability and scale diagnostics, but it does not use -mitata. For the `1 GiB` tier, the study now also has a budgeted mitata lane for streaming parser work. -Its purpose is narrower than the standard event-shape benchmark suite: - -- keep the command under about two minutes on the study machine -- stress real parser traversal on a `1 GiB` input -- stay on streaming parser paths instead of requiring full tree materialization - -The current mitata large-input collector measures two streaming cases: - -- `outlineEvents() streamed count` -- `events() streamed count` - -The collector now lowers mitata's minimum sample count to `1` by default for this lane and records -the checksum from the first measured execution instead of doing a separate full pre-pass. That keeps -the `1 GiB` collector much closer to its intended command budget on expensive scenarios. - -The mitata lane now uses the same ten scenario families as the one-shot and session lanes. The -implementation surface is no longer narrower than the rest of the study, and the checked-in `1 GiB` -archive now covers every non-pathological scenario across all four approaches. - -For limit-study sweeps, the matrix runner also supports `--continue-on-error` so one failing size or -approach does not hide the rest of the boundary. That mode still exits non-zero at the end, but it -keeps going and prints a structured failure summary for every failed job. - -Even with those mitigations, `pathological-recovery` at `1 GiB` currently exceeds the practical -survivability budget for `current-baseline` on this machine: the direct collector run was hard-killed -before it could write an artifact. Treat that scenario as a separate limit study rather than assuming -it belongs in the same normal archive pass as `plain-paragraphs`, `mixed-article`, and `synthetic-article`. - -The follow-up limit-study sweeps showed that the failure is not just a `1 GiB` budget miss. -On this branch and machine: - -- `256 MiB` baseline `pathological-recovery` overran a nominal `60s` budget for more than thirty minutes before it was stopped -- `64 MiB` baseline `pathological-recovery` failed with `RangeError: Maximum call stack size exceeded` -- a cross-approach `1/2/4 MiB` limit-study sweep failed for all four approaches with the same stack-overflow shape - -That makes `pathological-recovery` an explicit parser-limit lane, not merely an unfinished archive row. - -The current checked-in `1 GiB` mitata archive covers the non-pathological lane for all four approaches: - -- `plain-paragraphs` -- `mixed-article` -- `table-heavy` -- `template-heavy` -- `inline-heavy` -- `uri-heavy` -- `unicode-heavy` -- `synthetic-article` -- `outline-heavy` - -Collect one `1 GiB` mitata stress artifact for one approach: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-sys --allow-env=NODE_DISABLE_COLORS \ - --v8-flags=--expose-gc,--max-old-space-size=8192 \ - experiments/event-shape-study/tools/collect_large_input_mitata_report.ts \ - --approach-dir=current-baseline \ - --variant=current-baseline \ - --scenario=mixed-article \ - --size-mib=1024 \ - --total-budget-seconds=110 \ - --out=experiments/event-shape-study/current-baseline/artifacts/mitata-stress-mixed-1GiB.json -``` - -Use the matrix runner when you want the same `1 GiB` mitata stress lane across several approaches: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read --allow-write --allow-run --allow-sys --allow-env=NODE_DISABLE_COLORS \ - experiments/event-shape-study/tools/run_large_input_mitata_matrix.ts \ - --approaches=current-baseline,shared-props \ - --scenarios=plain-paragraphs,mixed-article,pathological-recovery,table-heavy,template-heavy,inline-heavy,uri-heavy,unicode-heavy,synthetic-article,outline-heavy \ - --sizes-mib=1024 \ - --total-budget-seconds=110 -``` - -The matrix runner shells out through `mise`, so it needs `--allow-run`. The single-run collector does not. - -When you need a quick inventory before choosing the next batch, summarize the archive directly: - -```bash -mise x deno@latest -- deno run --no-lock --allow-read \ - experiments/event-shape-study/tools/summarize_large_input_archive.ts \ - --size-mib=1024 -``` - -That summary groups one-shot stress, session stress, and mitata stress coverage by approach so the -next run can target missing scenarios instead of rescanning artifact directories by hand. - If you want to verify the study is fully built before collecting anything new, run the preflight inventory: ```bash @@ -350,6 +187,10 @@ mise x deno@latest -- deno run --no-lock --allow-read \ experiments/event-shape-study/tools/validate_experiment_matrix.ts ``` +This preflight only checks the runnable surface that exists on this branch: the core study docs, +the standard comparison tools, the one-shot large-input tools, the approach-local `code/mod.ts` +snapshots, and the checked-in `16 MiB` mixed-article smoke artifacts. + ## Decision rule The acceptance rule stays the same across all approaches: diff --git a/experiments/event-shape-study/results.md b/experiments/event-shape-study/results.md index 0ed94cf..52889eb 100644 --- a/experiments/event-shape-study/results.md +++ b/experiments/event-shape-study/results.md @@ -7,44 +7,41 @@ checked-in artifacts currently support. |---|---:|---:|---:|---| | `current-baseline` | control | control | control | keep as baseline | | `shared-props` | +0.81% | +4.02% | -0.24% | reject | +| `lazy-position` | -112.62% | -185.19% | -124.96% | reject | | `lazy-position-shared-props` | -107.05% | -0.08% | -149.81% | reject | | `planned-flat-eager-event-shape` | -1.49% | -3.97% | -3.55% | reject | Interpretation: - `shared-props` gives a real memory improvement, but the median target timing win is too small to clear the 5% acceptance bar. +- `lazy-position` shows that getter-based lazy `position` materialization is itself a decisive negative result on this code path, even without shared props mixed in. - `lazy-position-shared-props` fails hard on timing and is useful mainly as a negative result. - `planned-flat-eager-event-shape` keeps all changes inside its approach-local snapshot, but the refreshed direct comparison still lands below the timing bar and regresses retained memory. - the deterministic round-robin check for `planned-flat-eager-event-shape` shifts the target timing median to `+0.95%` with `2/9` significant target wins, but it still misses the acceptance bar and keeps the same memory regression. Archive completion summary: -| Lane | Scenario set | Implemented | Collected across all four approaches | Current status | +| Lane | Scenario set | Implemented | Collected across all five approaches | Current status | |---|---|---:|---:|---| -| Standard comparison | event-shape timing + retained memory | yes | yes | complete | -| `16 MiB` one-shot stress | ten scenarios | yes | partial | only `mixed-article` is checked in across all four approaches | -| `16 MiB` session stress | ten scenarios | yes | partial | `mixed-article` and `synthetic-article` are checked in across all four approaches | -| `1 GiB` mitata stress | ten scenarios | yes | yes for all non-pathological scenarios | nine normal scenarios are complete across all four approaches; only `pathological-recovery` remains outside the normal archive | -| `1 GiB` one-shot stress | ten scenarios | yes | no | archive not started | -| `1 GiB` session stress | ten scenarios | yes | no | archive not started | -| Pathological limit lane | separate parser-limit study | yes | n/a | treat as limit study, not as a normal missing archive row | +| Standard comparison | event-shape timing + retained memory | yes | yes | complete across five approaches | +| `16 MiB` one-shot stress | ten scenarios | yes | partial | only `mixed-article` is checked in across all five approaches | +| `1 GiB` one-shot stress | ten scenarios | yes | no | tooling exists, but the archive is not checked in on this branch | +| Session stress | n/a on this branch | no | no | not implemented on this branch | +| Mitata large-input stress | n/a on this branch | no | no | not implemented on this branch | Large-input archive status: -- comparable `16 MiB` mixed-article one-shot smoke artifacts exist for all four approaches +- comparable `16 MiB` mixed-article one-shot smoke artifacts exist for all five approaches - full one-shot matrix tooling now exists for ten scenario families -- session-stress tooling now exists for the same ten scenario families as the one-shot lane -- budgeted `1 GiB` mitata tooling now exists for the same ten scenario families as the one-shot lane -- the non-pathological `1 GiB` mitata archive is now complete across all four approaches for `plain-paragraphs`, `mixed-article`, `table-heavy`, `template-heavy`, `inline-heavy`, `uri-heavy`, `unicode-heavy`, `synthetic-article`, and `outline-heavy` -- `pathological-recovery` is now clearly a separate parser-limit lane rather than an ordinary large-input benchmark row: baseline was hard-killed at `1 GiB`, baseline `256 MiB` overran a `60s` budget for more than thirty minutes before it was stopped, and a cross-approach `1/2/4 MiB` mitata sweep failed for all four approaches with `RangeError: Maximum call stack size exceeded` -- `16 MiB` session-stress artifacts for `mixed-article` and `synthetic-article` now exist for all four approaches -- the first `16 MiB` session-matrix attempt showed that `pathological-recovery` is not part of the quick lane on this branch: `current-baseline` was still consuming a full CPU core after more than eleven minutes before the run was stopped -- the broader one-shot matrix and the session matrix are still not filled yet on this branch, but the normal `1 GiB` mitata lane is now fully checked in +- a structural preflight now checks that the runnable study surface exists before new collection starts +- session-stress and mitata large-input lanes are part of the longer draft, but they are not implemented on this branch +- the broader one-shot matrix is still mostly unfilled on this branch beyond the checked-in `mixed-article` smoke artifacts Primary artifacts: - [current-baseline/artifacts/report.json](current-baseline/artifacts/report.json) - [shared-props/artifacts/comparison.txt](shared-props/artifacts/comparison.txt) +- [lazy-position/artifacts/comparison.txt](lazy-position/artifacts/comparison.txt) - [lazy-position-shared-props/artifacts/comparison.txt](lazy-position-shared-props/artifacts/comparison.txt) - [planned-flat-eager-event-shape/artifacts/comparison.txt](planned-flat-eager-event-shape/artifacts/comparison.txt) - [cross-candidate-runs/2026-05-17-baseline-vs-flat-eager-round-robin-02/comparisons/round-01--current-baseline--vs--planned-flat-eager-event-shape.txt](cross-candidate-runs/2026-05-17-baseline-vs-flat-eager-round-robin-02/comparisons/round-01--current-baseline--vs--planned-flat-eager-event-shape.txt) \ No newline at end of file diff --git a/experiments/event-shape-study/scenario-roadmap.md b/experiments/event-shape-study/scenario-roadmap.md new file mode 100644 index 0000000..036e85d --- /dev/null +++ b/experiments/event-shape-study/scenario-roadmap.md @@ -0,0 +1,54 @@ +# Scenario Roadmap + +This roadmap names the study lanes that are real on this branch, the ones that are only +partially archived, and the highest-value work still missing from the original event-shape plan. + +## Current branch surface + +The branch supports four runnable study layers today: + +- the standard snapshot-local comparison flow in [methods.md](methods.md) +- the deterministic round-robin check in [methods.md](methods.md) +- the one-shot large-input stress collector and matrix runner in [methods.md](methods.md) +- the structural preflight inventory in [methods.md](methods.md) + +The checked-in large-input archive is intentionally narrow. Each approach keeps one comparable +`16 MiB` `mixed-article` smoke artifact so the archive path stays proven without turning the repo +into a dump of heavyweight stress output. + +## Standard comparison status + +The standard acceptance study is complete for the five checked-in approaches: + +- `current-baseline` +- `shared-props` +- `lazy-position` +- `lazy-position-shared-props` +- `planned-flat-eager-event-shape` + +Those runs are enough to say the current candidates do not clear the acceptance bar documented in +[protocol.md](protocol.md). + +## Large-input archive status + +The one-shot large-input lane is implemented, but the archive is only partially filled: + +- `16 MiB` `mixed-article` smoke artifacts exist for all five approaches +- the broader one-shot matrix can now be rerun on demand instead of being described only in prose +- session-oriented large-input tooling is not present on this branch +- budgeted mitata large-input tooling is not present on this branch + +That means the branch is now internally consistent, but it is still not the final archive shape +from the longer study draft. + +## Highest-value remaining work + +The original event-shape plan now has the missing lazy-position-only verdict filled in, and that +result is a clear rejection. The highest-value remaining work is now archive completion rather than +another first-order event-shape hypothesis. + +If the goal is to finish the study cleanly, the next work should happen in this order: + +1. Fill the non-pathological `16 MiB` one-shot matrix for the five checked-in approaches. +2. Decide whether session and mitata lanes belong on this branch as runnable tooling or only in a later follow-up branch. +3. If lazy positions are revisited at all, test a materially different layout that avoids getter-based per-event memoization. \ No newline at end of file diff --git a/experiments/event-shape-study/tools/run_large_input_stress_matrix.ts b/experiments/event-shape-study/tools/run_large_input_stress_matrix.ts new file mode 100644 index 0000000..90a8518 --- /dev/null +++ b/experiments/event-shape-study/tools/run_large_input_stress_matrix.ts @@ -0,0 +1,241 @@ +/** + * Run the large-input one-shot stress collector across a scenario and size matrix. + * + * This keeps the branch's documented large-input workflow runnable without requiring + * callers to hand-write one collector command per approach, scenario, and size. + */ + +import { fromFileUrl, join, relative } from 'jsr:@std/path'; + +type JobResult = { + approach: string; + scenario: string; + size_mib: number; + profile: 'full' | 'streaming-only'; + output_path: string; + status: 'ok' | 'failed'; + error_message?: string; +}; + +type MatrixRunSummary = { + study: 'event-shape'; + generated_at: string; + approaches: string[]; + scenarios: string[]; + sizes_mib: number[]; + repeats: number; + gib_profile: 'full' | 'streaming-only'; + continue_on_error: boolean; + jobs: JobResult[]; + commands: string[]; +}; + +const STUDY_ROOT = fromFileUrl(new URL('../', import.meta.url)); +const REPO_ROOT = fromFileUrl(new URL('../../../', import.meta.url)); +const DEFAULT_SCENARIOS = [ + 'plain-paragraphs', + 'mixed-article', + 'pathological-recovery', + 'table-heavy', + 'template-heavy', + 'inline-heavy', + 'uri-heavy', + 'unicode-heavy', + 'synthetic-article', + 'outline-heavy', +] as const; + +function getFlag(name: string): string | undefined { + return Deno.args.find((arg) => arg.startsWith(`--${name}=`))?.slice(name.length + 3); +} + +function hasFlag(name: string): boolean { + return Deno.args.includes(`--${name}`); +} + +function parseCsvFlag(name: string, fallback: readonly string[]): string[] { + const raw = getFlag(name); + if (raw === undefined) { + return [...fallback]; + } + + const values = raw.split(',').map((value) => value.trim()).filter((value) => value.length > 0); + if (values.length === 0) { + throw new Error(`expected --${name} to contain at least one comma-separated value`); + } + + return values; +} + +function parsePositiveIntFlag(name: string, fallback: number): number { + const raw = getFlag(name); + if (raw === undefined) { + return fallback; + } + + const value = Number(raw); + if (!Number.isInteger(value) || value <= 0) { + throw new Error(`expected --${name} to be a positive integer, got: ${raw}`); + } + + return value; +} + +function parseSizeListFlag(name: string, fallback: readonly number[]): number[] { + const raw = getFlag(name); + if (raw === undefined) { + return [...fallback]; + } + + return raw.split(',').map((value) => { + const parsed = Number(value.trim()); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error(`expected --${name} to contain positive integers, got: ${raw}`); + } + + return parsed; + }); +} + +function parseProfileFlag(name: string, fallback: 'full' | 'streaming-only'): 'full' | 'streaming-only' { + const raw = getFlag(name); + if (raw === undefined) { + return fallback; + } + + if (raw !== 'full' && raw !== 'streaming-only') { + throw new Error(`expected --${name} to be full or streaming-only, got: ${raw}`); + } + + return raw; +} + +function formatSizeLabel(size_mib: number): string { + if (size_mib % 1024 === 0) { + return `${size_mib / 1024}GiB`; + } + + return `${size_mib}MiB`; +} + +function buildOutputPath( + approach: string, + scenario: string, + size_mib: number, + profile: 'full' | 'streaming-only', +): string { + const suffix = formatSizeLabel(size_mib); + const file_name = size_mib >= 1024 && profile === 'full' + ? `stress-${scenario}-${suffix}-full.json` + : `stress-${scenario}-${suffix}.json`; + + return join(STUDY_ROOT, approach, 'artifacts', file_name); +} + +async function runCommand(args: string[]): Promise { + const command = new Deno.Command('mise', { + args, + cwd: REPO_ROOT, + stdout: 'piped', + stderr: 'piped', + env: { + NODE_DISABLE_COLORS: '1', + }, + }); + const output = await command.output(); + const stdout = new TextDecoder().decode(output.stdout); + const stderr = new TextDecoder().decode(output.stderr); + + if (!output.success) { + throw new Error(`command failed: mise ${args.join(' ')}\n${stdout}\n${stderr}`); + } +} + +const approaches = parseCsvFlag('approaches', []); +if (approaches.length === 0) { + throw new Error('expected --approaches to contain at least one approach directory'); +} + +const scenarios = parseCsvFlag('scenarios', DEFAULT_SCENARIOS); +const sizes_mib = parseSizeListFlag('sizes-mib', [16, 1024]); +const repeats = parsePositiveIntFlag('repeats', 1); +const gib_profile = parseProfileFlag('gib-profile', 'streaming-only'); +const continue_on_error = hasFlag('continue-on-error'); + +const commands: string[] = []; +const jobs: JobResult[] = []; + +for (const approach of approaches) { + for (const scenario of scenarios) { + for (const size_mib of sizes_mib) { + const profile = size_mib >= 1024 ? gib_profile : 'full'; + const output_path = buildOutputPath(approach, scenario, size_mib, profile); + const v8_flags = size_mib >= 1024 + ? '--v8-flags=--expose-gc,--max-old-space-size=8192' + : '--v8-flags=--expose-gc'; + const args = [ + 'x', + 'deno@latest', + '--', + 'deno', + 'run', + '--no-lock', + '--allow-read', + '--allow-write', + '--allow-sys', + v8_flags, + 'experiments/event-shape-study/tools/collect_large_input_stress_report.ts', + `--approach-dir=${approach}`, + `--variant=${approach.split('/').at(-1) ?? approach}`, + `--scenario=${scenario}`, + `--profile=${profile}`, + `--size-mib=${size_mib}`, + `--repeats=${repeats}`, + `--out=${relative(REPO_ROOT, output_path)}`, + ]; + commands.push(`mise ${args.join(' ')}`); + + try { + await runCommand(args); + jobs.push({ + approach, + scenario, + size_mib, + profile, + output_path: relative(REPO_ROOT, output_path), + status: 'ok', + }); + } catch (error) { + const error_message = error instanceof Error ? error.message : String(error); + jobs.push({ + approach, + scenario, + size_mib, + profile, + output_path: relative(REPO_ROOT, output_path), + status: 'failed', + error_message, + }); + + if (!continue_on_error) { + throw error; + } + } + } + } +} + +const summary: MatrixRunSummary = { + study: 'event-shape', + generated_at: new Date().toISOString(), + approaches, + scenarios, + sizes_mib, + repeats, + gib_profile, + continue_on_error, + jobs, + commands, +}; + +console.log(JSON.stringify(summary, null, 2)); \ No newline at end of file diff --git a/experiments/event-shape-study/tools/validate_experiment_matrix.ts b/experiments/event-shape-study/tools/validate_experiment_matrix.ts new file mode 100644 index 0000000..dfc2539 --- /dev/null +++ b/experiments/event-shape-study/tools/validate_experiment_matrix.ts @@ -0,0 +1,96 @@ +/** + * Validate that the documented event-shape study surface exists on this branch. + * + * This is a structural preflight. It does not rerun benchmarks. It checks that the + * core docs, study tools, approach-local snapshots, and baseline checked-in smoke + * artifacts are present before new collection starts. + */ + +import { fromFileUrl, join, relative } from 'jsr:@std/path'; + +type CheckResult = { + path: string; + status: 'ok' | 'missing'; + kind: 'doc' | 'tool' | 'approach-file' | 'artifact'; +}; + +const STUDY_ROOT = fromFileUrl(new URL('../', import.meta.url)); +const REPO_ROOT = fromFileUrl(new URL('../../../', import.meta.url)); +const APPROACHES = [ + 'current-baseline', + 'shared-props', + 'lazy-position', + 'lazy-position-shared-props', + 'planned-flat-eager-event-shape', +] as const; +const DOC_PATHS = [ + 'README.md', + 'methods.md', + 'protocol.md', + 'results.md', + 'scenario-roadmap.md', +] as const; +const TOOL_PATHS = [ + 'tools/collect_approach_report.ts', + 'tools/collect_large_input_stress_report.ts', + 'tools/compare_reports.ts', + 'tools/record_approach_artifacts.ts', + 'tools/run_cross_candidate_schedule.ts', + 'tools/run_large_input_stress_matrix.ts', + 'tools/validate_experiment_matrix.ts', +] as const; + +async function pathExists(path: string): Promise { + try { + await Deno.stat(path); + return true; + } catch (error) { + if (error instanceof Deno.errors.NotFound) { + return false; + } + + throw error; + } +} + +async function checkPath(path: string, kind: CheckResult['kind']): Promise { + return { + path: relative(REPO_ROOT, path), + status: await pathExists(path) ? 'ok' : 'missing', + kind, + }; +} + +const results: CheckResult[] = []; + +for (const doc_path of DOC_PATHS) { + results.push(await checkPath(join(STUDY_ROOT, doc_path), 'doc')); +} + +for (const tool_path of TOOL_PATHS) { + results.push(await checkPath(join(STUDY_ROOT, tool_path), 'tool')); +} + +for (const approach of APPROACHES) { + results.push(await checkPath(join(STUDY_ROOT, approach, 'code', 'mod.ts'), 'approach-file')); + results.push(await checkPath(join(STUDY_ROOT, approach, 'artifacts', 'report.json'), 'artifact')); + results.push(await checkPath(join(STUDY_ROOT, approach, 'artifacts', 'stress-mixed-16MiB.json'), 'artifact')); +} + +const missing = results.filter((entry) => entry.status === 'missing'); + +for (const group of ['doc', 'tool', 'approach-file', 'artifact'] as const) { + const group_entries = results.filter((entry) => entry.kind === group); + console.log(`${group}:`); + for (const entry of group_entries) { + console.log(` [${entry.status}] ${entry.path}`); + } + console.log(''); +} + +if (missing.length > 0) { + console.error(`missing ${missing.length} required study paths`); + Deno.exit(1); +} + +console.log(`validated ${results.length} study paths`); \ No newline at end of file -- 2.51.2