diff --git a/experiments/event-shape-study/lazy-position/README.md b/experiments/event-shape-study/lazy-position/README.md new file mode 100644 index 0000000..d665118 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/README.md @@ -0,0 +1,34 @@ +# Lazy Position + +## Hypothesis + +This approach isolates the lazy `position` idea without also interning or sharing props +objects. The goal is to answer the missing question from the original study plan: whether +eager position allocation is the real hot-path cost, or whether the large regression in the +combined lazy-position-plus-shared-props lane came from the interaction between both ideas. + +## Methodology + +The candidate keeps the public event API shape the same and uses the same study settings as +the other approaches. It reuses the lazy `position` event factory from the combined lane, but +all event props stay as ordinary per-event objects just like the baseline. + +## Expected reading + +If this lane still regresses badly, lazy getter-based `position` materialization is likely the +main reason. If it behaves much better than the combined lane, the earlier failure was at least +partly an interaction effect rather than a clean verdict on lazy positions alone. + +## Code Under Test + +- `code/event_factory.ts` +- `code/block_parser.ts` +- `code/inline_parser.ts` + +## Artifacts + +- `artifacts/report.json` +- `artifacts/comparison.json` +- `artifacts/comparison.txt` +- `artifacts/recording.json` +- `artifacts/stress-mixed-16MiB.json` \ No newline at end of file diff --git a/experiments/event-shape-study/lazy-position/artifacts/commands.txt b/experiments/event-shape-study/lazy-position/artifacts/commands.txt new file mode 100644 index 0000000..1f69f5a --- /dev/null +++ b/experiments/event-shape-study/lazy-position/artifacts/commands.txt @@ -0,0 +1,3 @@ +mise x deno@latest -- deno run --no-lock --allow-run=mise,git --allow-read --allow-write experiments/event-shape-study/tools/collect_approach_report.ts --approach-dir=lazy-position --variant=lazy-position --runs=10 --memory-repeats=5 --out=experiments/event-shape-study/lazy-position/artifacts/report.json +mise x deno@latest -- deno run --no-lock --allow-read experiments/event-shape-study/tools/compare_reports.ts --min-improvement=0.05 --max-regression=0.03 --bootstrap=5000 --alpha=0.05 --format=json experiments/event-shape-study/current-baseline/artifacts/report.json experiments/event-shape-study/lazy-position/artifacts/report.json +mise x deno@latest -- deno run --no-lock --allow-read experiments/event-shape-study/tools/compare_reports.ts --min-improvement=0.05 --max-regression=0.03 --bootstrap=5000 --alpha=0.05 --format=text experiments/event-shape-study/current-baseline/artifacts/report.json experiments/event-shape-study/lazy-position/artifacts/report.json diff --git a/experiments/event-shape-study/lazy-position/artifacts/comparison.json b/experiments/event-shape-study/lazy-position/artifacts/comparison.json new file mode 100644 index 0000000..27c33b5 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/artifacts/comparison.json @@ -0,0 +1,62 @@ +{ + "baseline": { + "variant": "current-baseline", + "branch": "perf-test-improve-event-shapes", + "commit": "ffe74312ed03a2b25f510ffa5786bcd4c65ba4a3", + "path": "experiments/event-shape-study/current-baseline/artifacts/report.json" + }, + "minimum_improvement": 0.05, + "maximum_regression": 0.03, + "bootstrap_iterations": 5000, + "alpha": 0.05, + "decisions": [ + { + "variant": "lazy-position", + "path": "experiments/event-shape-study/lazy-position/artifacts/report.json", + "target_timing_median": -1.1262302459513576, + "memory_median": -1.8518927147822155, + "worst_critical_timing": -1.249626307922272, + "significant_target_wins": 0, + "significant_memory_wins": 0, + "significant_critical_regressions": 9, + "recommended": false, + "top_timing_wins": [], + "top_memory_wins": [], + "critical_regressions": [ + { + "name": "parse(): synthetic article (~35-45 KB)", + "family": "timing", + "estimate": -1.249626307922272, + "ci_lower": -2.1052875911239286, + "ci_upper": -1.1156960988254296, + "p_value": 0, + "adjusted_p_value": 0, + "significant_better": false, + "significant_worse": true + }, + { + "name": "session.parse() warm: same-size mixed (~8 KB)", + "family": "timing", + "estimate": -1.19461429788519, + "ci_lower": -2.2883137656283066, + "ci_upper": -1.105441054976658, + "p_value": 0, + "adjusted_p_value": 0, + "significant_better": false, + "significant_worse": true + }, + { + "name": "parse(): same-size mixed (~8 KB)", + "family": "timing", + "estimate": -1.1673551098292037, + "ci_lower": -1.7253257461590932, + "ci_upper": -1.0555289981041531, + "p_value": 0, + "adjusted_p_value": 0, + "significant_better": false, + "significant_worse": true + } + ] + } + ] +} diff --git a/experiments/event-shape-study/lazy-position/artifacts/comparison.txt b/experiments/event-shape-study/lazy-position/artifacts/comparison.txt new file mode 100644 index 0000000..0b315ff --- /dev/null +++ b/experiments/event-shape-study/lazy-position/artifacts/comparison.txt @@ -0,0 +1,12 @@ +baseline: current-baseline (perf-test-improve-event-shapes ffe74312) +decision rule: target median >= +5.00%, critical regressions > -3.00% disallowed, Holm-adjusted alpha=0.050 + +lazy-position: not recommended +report: experiments/event-shape-study/lazy-position/artifacts/report.json +target timing median: -112.62% (0/9 significant wins) +memory median: -185.19% (0 significant wins) +worst critical timing: -124.96% (9 significant regressions) +critical timing risks: +- parse(): synthetic article (~35-45 KB): -124.96% [-210.53%, -111.57%], p_adj=0.00e+0 +- session.parse() warm: same-size mixed (~8 KB): -119.46% [-228.83%, -110.54%], p_adj=0.00e+0 +- parse(): same-size mixed (~8 KB): -116.74% [-172.53%, -105.55%], p_adj=0.00e+0 diff --git a/experiments/event-shape-study/lazy-position/artifacts/recording.json b/experiments/event-shape-study/lazy-position/artifacts/recording.json new file mode 100644 index 0000000..93fc789 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/artifacts/recording.json @@ -0,0 +1,23 @@ +{ + "study": "event-shape", + "variant": "lazy-position", + "experiment_dir": "experiments/event-shape-study/lazy-position", + "report_path": "experiments/event-shape-study/lazy-position/artifacts/report.json", + "comparison_json": "experiments/event-shape-study/lazy-position/artifacts/comparison.json", + "comparison_text": "experiments/event-shape-study/lazy-position/artifacts/comparison.txt", + "baseline_report": "experiments/event-shape-study/current-baseline/artifacts/report.json", + "generated_at": "2026-05-19T00:47:45.490Z", + "runs": 10, + "memory_repeats": 5, + "minimum_improvement": 0.05, + "maximum_regression": 0.03, + "bootstrap_iterations": 5000, + "alpha": 0.05, + "branch": "experiments", + "commit": "47ef0a13b5b1b6f53dc2b63aa5c2123bfe4dbc86", + "commands": [ + "mise x deno@latest -- deno run --no-lock --allow-run=mise,git --allow-read --allow-write experiments/event-shape-study/tools/collect_approach_report.ts --approach-dir=lazy-position --variant=lazy-position --runs=10 --memory-repeats=5 --out=experiments/event-shape-study/lazy-position/artifacts/report.json", + "mise x deno@latest -- deno run --no-lock --allow-read experiments/event-shape-study/tools/compare_reports.ts --min-improvement=0.05 --max-regression=0.03 --bootstrap=5000 --alpha=0.05 --format=json experiments/event-shape-study/current-baseline/artifacts/report.json experiments/event-shape-study/lazy-position/artifacts/report.json", + "mise x deno@latest -- deno run --no-lock --allow-read experiments/event-shape-study/tools/compare_reports.ts --min-improvement=0.05 --max-regression=0.03 --bootstrap=5000 --alpha=0.05 --format=text experiments/event-shape-study/current-baseline/artifacts/report.json experiments/event-shape-study/lazy-position/artifacts/report.json" + ] +} diff --git a/experiments/event-shape-study/lazy-position/artifacts/report.json b/experiments/event-shape-study/lazy-position/artifacts/report.json new file mode 100644 index 0000000..3f68b05 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/artifacts/report.json @@ -0,0 +1,1267 @@ +{ + "schema_version": 1, + "variant": "lazy-position", + "branch": "experiments", + "commit": "47ef0a13b5b1b6f53dc2b63aa5c2123bfe4dbc86", + "generated_at": "2026-05-19T00:47:44.867Z", + "runs": 10, + "memory_repeats": 5, + "timing_unit": "microseconds_per_iter", + "memory_unit": "bytes", + "environment": { + "uname": "Linux 6.12.76-linuxkit #1 SMP Thu Apr 30 11:19:05 UTC 2026 aarch64 GNU/Linux", + "deno_version": "deno 2.7.14 (stable, release, aarch64-unknown-linux-gnu)\nv8 14.7.173.20-rusty\ntypescript 5.9.2", + "lscpu_summary": { + "Architecture": "aarch64", + "CPU(s)": "8", + "Vendor ID": "Apple", + "Model name": "-", + "Thread(s) per core": "1", + "Socket(s)": "-" + }, + "git_worktree_clean": false + }, + "design": { + "timing_command": "mise x deno@latest -- deno bench --no-lock --allow-sys --allow-env=NODE_DISABLE_COLORS --v8-flags=--expose-gc event_shape_bench.ts", + "memory_command": "mise x deno@latest -- deno run --no-lock --allow-sys --v8-flags=--expose-gc event_shape_memory.ts --repeats=5 --format=json", + "approach_dir": "experiments/event-shape-study/lazy-position", + "code_dir": "experiments/event-shape-study/lazy-position/code", + "independent_process_per_run": true, + "raw_samples_preserved": true, + "bootstrap_ready": true, + "notes": [ + "Each timing run executes the approach-local event_shape_bench.ts in a fresh process.", + "Each memory run executes the approach-local event_shape_memory.ts in a fresh process with explicit repeats.", + "All code under test lives inside the approach-local code snapshot to reduce cross-approach variation from root-directory edits." + ] + }, + "timing": { + "per_run": [ + { + "index": 0, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 520.64, + "events() no position access: same-size mixed (~8 KB)": 1640, + "events() no position access: same-size pathological (~8 KB)": 3150, + "events() offsets only: same-size mixed (~8 KB)": 1230, + "events() offsets only: same-size pathological (~8 KB)": 3090, + "events() all position reads: same-size mixed (~8 KB)": 1450, + "events() all position reads: same-size pathological (~8 KB)": 3030, + "events() enter props reads: same-size mixed (~8 KB)": 1290, + "events() enter props reads: same-size pathological (~8 KB)": 2970, + "events() retained array only: same-size mixed (~8 KB)": 1340, + "session.events() warm retained array: same-size mixed (~8 KB)": 1510, + "events() retained array only: synthetic article (~35-45 KB)": 4600, + "session.events() warm retained array: synthetic article (~35-45 KB)": 5180, + "parse(): same-size mixed (~8 KB)": 1810, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 3630, + "session.events() cold: same-size mixed (~8 KB)": 1370, + "session.events() warm: same-size mixed (~8 KB)": 1440, + "session.parse() cold: same-size mixed (~8 KB)": 1830, + "session.parse() warm: same-size mixed (~8 KB)": 1810, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 3860, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 3870, + "parse(): synthetic article (~35-45 KB)": 7340, + "session.events() warm: synthetic article (~35-45 KB)": 6940, + "session.parse() warm: synthetic article (~35-45 KB)": 7820, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 7090 + } + }, + { + "index": 1, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 527.74, + "events() no position access: same-size mixed (~8 KB)": 1470, + "events() no position access: same-size pathological (~8 KB)": 3270, + "events() offsets only: same-size mixed (~8 KB)": 1530, + "events() offsets only: same-size pathological (~8 KB)": 3100, + "events() all position reads: same-size mixed (~8 KB)": 1780, + "events() all position reads: same-size pathological (~8 KB)": 3170, + "events() enter props reads: same-size mixed (~8 KB)": 1280, + "events() enter props reads: same-size pathological (~8 KB)": 2810, + "events() retained array only: same-size mixed (~8 KB)": 1200, + "session.events() warm retained array: same-size mixed (~8 KB)": 1600, + "events() retained array only: synthetic article (~35-45 KB)": 4510, + "session.events() warm retained array: synthetic article (~35-45 KB)": 4920, + "parse(): same-size mixed (~8 KB)": 1750, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 3340, + "session.events() cold: same-size mixed (~8 KB)": 1210, + "session.events() warm: same-size mixed (~8 KB)": 1340, + "session.parse() cold: same-size mixed (~8 KB)": 1640, + "session.parse() warm: same-size mixed (~8 KB)": 1640, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 3360, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 3370, + "parse(): synthetic article (~35-45 KB)": 5970, + "session.events() warm: synthetic article (~35-45 KB)": 5200, + "session.parse() warm: synthetic article (~35-45 KB)": 5850, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 5780 + } + }, + { + "index": 2, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 484.28, + "events() no position access: same-size mixed (~8 KB)": 1340, + "events() no position access: same-size pathological (~8 KB)": 2990, + "events() offsets only: same-size mixed (~8 KB)": 1260, + "events() offsets only: same-size pathological (~8 KB)": 2860, + "events() all position reads: same-size mixed (~8 KB)": 1360, + "events() all position reads: same-size pathological (~8 KB)": 2990, + "events() enter props reads: same-size mixed (~8 KB)": 1290, + "events() enter props reads: same-size pathological (~8 KB)": 2760, + "events() retained array only: same-size mixed (~8 KB)": 1260, + "session.events() warm retained array: same-size mixed (~8 KB)": 1530, + "events() retained array only: synthetic article (~35-45 KB)": 4520, + "session.events() warm retained array: synthetic article (~35-45 KB)": 5220, + "parse(): same-size mixed (~8 KB)": 1840, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 3560, + "session.events() cold: same-size mixed (~8 KB)": 1350, + "session.events() warm: same-size mixed (~8 KB)": 1560, + "session.parse() cold: same-size mixed (~8 KB)": 2170, + "session.parse() warm: same-size mixed (~8 KB)": 2060, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 4320, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 3640, + "parse(): synthetic article (~35-45 KB)": 6790, + "session.events() warm: synthetic article (~35-45 KB)": 5500, + "session.parse() warm: synthetic article (~35-45 KB)": 6520, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 6660 + } + }, + { + "index": 3, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 520.84, + "events() no position access: same-size mixed (~8 KB)": 1410, + "events() no position access: same-size pathological (~8 KB)": 3230, + "events() offsets only: same-size mixed (~8 KB)": 1170, + "events() offsets only: same-size pathological (~8 KB)": 3040, + "events() all position reads: same-size mixed (~8 KB)": 1410, + "events() all position reads: same-size pathological (~8 KB)": 3190, + "events() enter props reads: same-size mixed (~8 KB)": 1150, + "events() enter props reads: same-size pathological (~8 KB)": 3100, + "events() retained array only: same-size mixed (~8 KB)": 1140, + "session.events() warm retained array: same-size mixed (~8 KB)": 1450, + "events() retained array only: synthetic article (~35-45 KB)": 5040, + "session.events() warm retained array: synthetic article (~35-45 KB)": 5560, + "parse(): same-size mixed (~8 KB)": 1970, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 3670, + "session.events() cold: same-size mixed (~8 KB)": 1530, + "session.events() warm: same-size mixed (~8 KB)": 1500, + "session.parse() cold: same-size mixed (~8 KB)": 1860, + "session.parse() warm: same-size mixed (~8 KB)": 1950, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 3990, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 4070.0000000000005, + "parse(): synthetic article (~35-45 KB)": 8140.000000000001, + "session.events() warm: synthetic article (~35-45 KB)": 5860, + "session.parse() warm: synthetic article (~35-45 KB)": 6710, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 6770 + } + }, + { + "index": 4, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 900.82, + "events() no position access: same-size mixed (~8 KB)": 3090, + "events() no position access: same-size pathological (~8 KB)": 6170, + "events() offsets only: same-size mixed (~8 KB)": 2029.9999999999998, + "events() offsets only: same-size pathological (~8 KB)": 3240, + "events() all position reads: same-size mixed (~8 KB)": 1700, + "events() all position reads: same-size pathological (~8 KB)": 3110, + "events() enter props reads: same-size mixed (~8 KB)": 1590, + "events() enter props reads: same-size pathological (~8 KB)": 4120, + "events() retained array only: same-size mixed (~8 KB)": 2029.9999999999998, + "session.events() warm retained array: same-size mixed (~8 KB)": 2580, + "events() retained array only: synthetic article (~35-45 KB)": 6260, + "session.events() warm retained array: synthetic article (~35-45 KB)": 8330, + "parse(): same-size mixed (~8 KB)": 3360, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 4460, + "session.events() cold: same-size mixed (~8 KB)": 2070, + "session.events() warm: same-size mixed (~8 KB)": 2380, + "session.parse() cold: same-size mixed (~8 KB)": 5440, + "session.parse() warm: same-size mixed (~8 KB)": 3570, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 6310, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 7740, + "parse(): synthetic article (~35-45 KB)": 16780, + "session.events() warm: synthetic article (~35-45 KB)": 7430, + "session.parse() warm: synthetic article (~35-45 KB)": 9240, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 8470 + } + }, + { + "index": 5, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 724.56, + "events() no position access: same-size mixed (~8 KB)": 2580, + "events() no position access: same-size pathological (~8 KB)": 3540, + "events() offsets only: same-size mixed (~8 KB)": 1940, + "events() offsets only: same-size pathological (~8 KB)": 3830, + "events() all position reads: same-size mixed (~8 KB)": 1690, + "events() all position reads: same-size pathological (~8 KB)": 4050, + "events() enter props reads: same-size mixed (~8 KB)": 1840, + "events() enter props reads: same-size pathological (~8 KB)": 3460, + "events() retained array only: same-size mixed (~8 KB)": 1740, + "session.events() warm retained array: same-size mixed (~8 KB)": 1890, + "events() retained array only: synthetic article (~35-45 KB)": 6150, + "session.events() warm retained array: synthetic article (~35-45 KB)": 6400, + "parse(): same-size mixed (~8 KB)": 2780, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 4590, + "session.events() cold: same-size mixed (~8 KB)": 2650, + "session.events() warm: same-size mixed (~8 KB)": 2060, + "session.parse() cold: same-size mixed (~8 KB)": 2390, + "session.parse() warm: same-size mixed (~8 KB)": 4330, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 5650, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 4380, + "parse(): synthetic article (~35-45 KB)": 7820, + "session.events() warm: synthetic article (~35-45 KB)": 8670, + "session.parse() warm: synthetic article (~35-45 KB)": 12430, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 8390 + } + }, + { + "index": 6, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 652.31, + "events() no position access: same-size mixed (~8 KB)": 1930, + "events() no position access: same-size pathological (~8 KB)": 3570, + "events() offsets only: same-size mixed (~8 KB)": 1560, + "events() offsets only: same-size pathological (~8 KB)": 3980, + "events() all position reads: same-size mixed (~8 KB)": 1820, + "events() all position reads: same-size pathological (~8 KB)": 3780, + "events() enter props reads: same-size mixed (~8 KB)": 1480, + "events() enter props reads: same-size pathological (~8 KB)": 3460, + "events() retained array only: same-size mixed (~8 KB)": 1440, + "session.events() warm retained array: same-size mixed (~8 KB)": 1870, + "events() retained array only: synthetic article (~35-45 KB)": 5460, + "session.events() warm retained array: synthetic article (~35-45 KB)": 6640, + "parse(): same-size mixed (~8 KB)": 2090, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 5100, + "session.events() cold: same-size mixed (~8 KB)": 2120, + "session.events() warm: same-size mixed (~8 KB)": 1820, + "session.parse() cold: same-size mixed (~8 KB)": 2370, + "session.parse() warm: same-size mixed (~8 KB)": 3390, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 4630, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 3930, + "parse(): synthetic article (~35-45 KB)": 7240, + "session.events() warm: synthetic article (~35-45 KB)": 6170, + "session.parse() warm: synthetic article (~35-45 KB)": 7280, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 7280 + } + }, + { + "index": 7, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 642.2, + "events() no position access: same-size mixed (~8 KB)": 1760, + "events() no position access: same-size pathological (~8 KB)": 3270, + "events() offsets only: same-size mixed (~8 KB)": 1600, + "events() offsets only: same-size pathological (~8 KB)": 3540, + "events() all position reads: same-size mixed (~8 KB)": 1650, + "events() all position reads: same-size pathological (~8 KB)": 3260, + "events() enter props reads: same-size mixed (~8 KB)": 1320, + "events() enter props reads: same-size pathological (~8 KB)": 2860, + "events() retained array only: same-size mixed (~8 KB)": 1260, + "session.events() warm retained array: same-size mixed (~8 KB)": 1460, + "events() retained array only: synthetic article (~35-45 KB)": 4430, + "session.events() warm retained array: synthetic article (~35-45 KB)": 4870, + "parse(): same-size mixed (~8 KB)": 1710, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 3320, + "session.events() cold: same-size mixed (~8 KB)": 1260, + "session.events() warm: same-size mixed (~8 KB)": 1500, + "session.parse() cold: same-size mixed (~8 KB)": 2360, + "session.parse() warm: same-size mixed (~8 KB)": 2009.9999999999998, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 3880, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 3920, + "parse(): synthetic article (~35-45 KB)": 7660, + "session.events() warm: synthetic article (~35-45 KB)": 5430, + "session.parse() warm: synthetic article (~35-45 KB)": 6600, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 6720 + } + }, + { + "index": 8, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 487.01, + "events() no position access: same-size mixed (~8 KB)": 1460, + "events() no position access: same-size pathological (~8 KB)": 3250, + "events() offsets only: same-size mixed (~8 KB)": 1450, + "events() offsets only: same-size pathological (~8 KB)": 3100, + "events() all position reads: same-size mixed (~8 KB)": 1750, + "events() all position reads: same-size pathological (~8 KB)": 2940, + "events() enter props reads: same-size mixed (~8 KB)": 1250, + "events() enter props reads: same-size pathological (~8 KB)": 3000, + "events() retained array only: same-size mixed (~8 KB)": 1430, + "session.events() warm retained array: same-size mixed (~8 KB)": 1660, + "events() retained array only: synthetic article (~35-45 KB)": 5190, + "session.events() warm retained array: synthetic article (~35-45 KB)": 5750, + "parse(): same-size mixed (~8 KB)": 1960, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 3590, + "session.events() cold: same-size mixed (~8 KB)": 1240, + "session.events() warm: same-size mixed (~8 KB)": 1460, + "session.parse() cold: same-size mixed (~8 KB)": 1920, + "session.parse() warm: same-size mixed (~8 KB)": 2110, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 3780, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 3680, + "parse(): synthetic article (~35-45 KB)": 7710, + "session.events() warm: synthetic article (~35-45 KB)": 5960, + "session.parse() warm: synthetic article (~35-45 KB)": 6800, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 5740 + } + }, + { + "index": 9, + "benchmarks": { + "events() no position access: same-size plain (~8 KB)": 782.1, + "events() no position access: same-size mixed (~8 KB)": 1690, + "events() no position access: same-size pathological (~8 KB)": 3180, + "events() offsets only: same-size mixed (~8 KB)": 1300, + "events() offsets only: same-size pathological (~8 KB)": 2990, + "events() all position reads: same-size mixed (~8 KB)": 1380, + "events() all position reads: same-size pathological (~8 KB)": 2960, + "events() enter props reads: same-size mixed (~8 KB)": 1280, + "events() enter props reads: same-size pathological (~8 KB)": 2830, + "events() retained array only: same-size mixed (~8 KB)": 1430, + "session.events() warm retained array: same-size mixed (~8 KB)": 1690, + "events() retained array only: synthetic article (~35-45 KB)": 4570, + "session.events() warm retained array: synthetic article (~35-45 KB)": 5560, + "parse(): same-size mixed (~8 KB)": 2000, + "parseWithDiagnostics(): same-size pathological (~8 KB)": 3620, + "session.events() cold: same-size mixed (~8 KB)": 1300, + "session.events() warm: same-size mixed (~8 KB)": 1410, + "session.parse() cold: same-size mixed (~8 KB)": 1810, + "session.parse() warm: same-size mixed (~8 KB)": 1740, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": 3580, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": 3630, + "parse(): synthetic article (~35-45 KB)": 7390, + "session.events() warm: synthetic article (~35-45 KB)": 6070, + "session.parse() warm: synthetic article (~35-45 KB)": 6840, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": 6500 + } + } + ], + "summary": { + "events() no position access: same-size plain (~8 KB)": { + "samples": [ + 520.64, + 527.74, + 484.28, + 520.84, + 900.82, + 724.56, + 652.31, + 642.2, + 487.01, + 782.1 + ], + "mean": 624.2500000000001, + "median": 584.97, + "minimum": 484.28, + "maximum": 900.82, + "stdev": 142.10657518605927 + }, + "events() no position access: same-size mixed (~8 KB)": { + "samples": [ + 1640, + 1470, + 1340, + 1410, + 3090, + 2580, + 1930, + 1760, + 1460, + 1690 + ], + "mean": 1837, + "median": 1665, + "minimum": 1340, + "maximum": 3090, + "stdev": 567.9602881266338 + }, + "events() no position access: same-size pathological (~8 KB)": { + "samples": [ + 3150, + 3270, + 2990, + 3230, + 6170, + 3540, + 3570, + 3270, + 3250, + 3180 + ], + "mean": 3562, + "median": 3260, + "minimum": 2990, + "maximum": 6170, + "stdev": 932.3304135337429 + }, + "events() offsets only: same-size mixed (~8 KB)": { + "samples": [ + 1230, + 1530, + 1260, + 1170, + 2029.9999999999998, + 1940, + 1560, + 1600, + 1450, + 1300 + ], + "mean": 1507, + "median": 1490, + "minimum": 1170, + "maximum": 2029.9999999999998, + "stdev": 292.5766756101913 + }, + "events() offsets only: same-size pathological (~8 KB)": { + "samples": [ + 3090, + 3100, + 2860, + 3040, + 3240, + 3830, + 3980, + 3540, + 3100, + 2990 + ], + "mean": 3277, + "median": 3100, + "minimum": 2860, + "maximum": 3980, + "stdev": 377.21346741599774 + }, + "events() all position reads: same-size mixed (~8 KB)": { + "samples": [ + 1450, + 1780, + 1360, + 1410, + 1700, + 1690, + 1820, + 1650, + 1750, + 1380 + ], + "mean": 1599, + "median": 1670, + "minimum": 1360, + "maximum": 1820, + "stdev": 179.03755037545739 + }, + "events() all position reads: same-size pathological (~8 KB)": { + "samples": [ + 3030, + 3170, + 2990, + 3190, + 3110, + 4050, + 3780, + 3260, + 2940, + 2960 + ], + "mean": 3248, + "median": 3140, + "minimum": 2940, + "maximum": 4050, + "stdev": 372.19469218258456 + }, + "events() enter props reads: same-size mixed (~8 KB)": { + "samples": [ + 1290, + 1280, + 1290, + 1150, + 1590, + 1840, + 1480, + 1320, + 1250, + 1280 + ], + "mean": 1377, + "median": 1290, + "minimum": 1150, + "maximum": 1840, + "stdev": 204.18129199316965 + }, + "events() enter props reads: same-size pathological (~8 KB)": { + "samples": [ + 2970, + 2810, + 2760, + 3100, + 4120, + 3460, + 3460, + 2860, + 3000, + 2830 + ], + "mean": 3137, + "median": 2985, + "minimum": 2760, + "maximum": 4120, + "stdev": 427.26650543503484 + }, + "events() retained array only: same-size mixed (~8 KB)": { + "samples": [ + 1340, + 1200, + 1260, + 1140, + 2029.9999999999998, + 1740, + 1440, + 1260, + 1430, + 1430 + ], + "mean": 1427, + "median": 1385, + "minimum": 1140, + "maximum": 2029.9999999999998, + "stdev": 270.59810297438025 + }, + "session.events() warm retained array: same-size mixed (~8 KB)": { + "samples": [ + 1510, + 1600, + 1530, + 1450, + 2580, + 1890, + 1870, + 1460, + 1660, + 1690 + ], + "mean": 1724, + "median": 1630, + "minimum": 1450, + "maximum": 2580, + "stdev": 338.3686352683022 + }, + "events() retained array only: synthetic article (~35-45 KB)": { + "samples": [ + 4600, + 4510, + 4520, + 5040, + 6260, + 6150, + 5460, + 4430, + 5190, + 4570 + ], + "mean": 5073, + "median": 4820, + "minimum": 4430, + "maximum": 6260, + "stdev": 686.8939753217619 + }, + "session.events() warm retained array: synthetic article (~35-45 KB)": { + "samples": [ + 5180, + 4920, + 5220, + 5560, + 8330, + 6400, + 6640, + 4870, + 5750, + 5560 + ], + "mean": 5843, + "median": 5560, + "minimum": 4870, + "maximum": 8330, + "stdev": 1050.1751176721802 + }, + "parse(): same-size mixed (~8 KB)": { + "samples": [ + 1810, + 1750, + 1840, + 1970, + 3360, + 2780, + 2090, + 1710, + 1960, + 2000 + ], + "mean": 2127, + "median": 1965, + "minimum": 1710, + "maximum": 3360, + "stdev": 528.6891546289013 + }, + "parseWithDiagnostics(): same-size pathological (~8 KB)": { + "samples": [ + 3630, + 3340, + 3560, + 3670, + 4460, + 4590, + 5100, + 3320, + 3590, + 3620 + ], + "mean": 3888, + "median": 3625, + "minimum": 3320, + "maximum": 5100, + "stdev": 604.9940311918604 + }, + "session.events() cold: same-size mixed (~8 KB)": { + "samples": [ + 1370, + 1210, + 1350, + 1530, + 2070, + 2650, + 2120, + 1260, + 1240, + 1300 + ], + "mean": 1610, + "median": 1360, + "minimum": 1210, + "maximum": 2650, + "stdev": 494.45817708769755 + }, + "session.events() warm: same-size mixed (~8 KB)": { + "samples": [ + 1440, + 1340, + 1560, + 1500, + 2380, + 2060, + 1820, + 1500, + 1460, + 1410 + ], + "mean": 1647, + "median": 1500, + "minimum": 1340, + "maximum": 2380, + "stdev": 336.1233239287165 + }, + "session.parse() cold: same-size mixed (~8 KB)": { + "samples": [ + 1830, + 1640, + 2170, + 1860, + 5440, + 2390, + 2370, + 2360, + 1920, + 1810 + ], + "mean": 2379, + "median": 2045, + "minimum": 1640, + "maximum": 5440, + "stdev": 1108.8176886515955 + }, + "session.parse() warm: same-size mixed (~8 KB)": { + "samples": [ + 1810, + 1640, + 2060, + 1950, + 3570, + 4330, + 3390, + 2009.9999999999998, + 2110, + 1740 + ], + "mean": 2461, + "median": 2035, + "minimum": 1640, + "maximum": 4330, + "stdev": 939.9934987954852 + }, + "session.parseWithDiagnostics() cold: same-size pathological (~8 KB)": { + "samples": [ + 3860, + 3360, + 4320, + 3990, + 6310, + 5650, + 4630, + 3880, + 3780, + 3580 + ], + "mean": 4336, + "median": 3935, + "minimum": 3360, + "maximum": 6310, + "stdev": 948.5333708181043 + }, + "session.parseWithDiagnostics() warm: same-size pathological (~8 KB)": { + "samples": [ + 3870, + 3370, + 3640, + 4070.0000000000005, + 7740, + 4380, + 3930, + 3920, + 3680, + 3630 + ], + "mean": 4223, + "median": 3895, + "minimum": 3370, + "maximum": 7740, + "stdev": 1266.3863724963423 + }, + "parse(): synthetic article (~35-45 KB)": { + "samples": [ + 7340, + 5970, + 6790, + 8140.000000000001, + 16780, + 7820, + 7240, + 7660, + 7710, + 7390 + ], + "mean": 8284, + "median": 7525, + "minimum": 5970, + "maximum": 16780, + "stdev": 3046.0145180948243 + }, + "session.events() warm: synthetic article (~35-45 KB)": { + "samples": [ + 6940, + 5200, + 5500, + 5860, + 7430, + 8670, + 6170, + 5430, + 5960, + 6070 + ], + "mean": 6323, + "median": 6015, + "minimum": 5200, + "maximum": 8670, + "stdev": 1068.9563965746113 + }, + "session.parse() warm: synthetic article (~35-45 KB)": { + "samples": [ + 7820, + 5850, + 6520, + 6710, + 9240, + 12430, + 7280, + 6600, + 6800, + 6840 + ], + "mean": 7609, + "median": 6820, + "minimum": 5850, + "maximum": 12430, + "stdev": 1926.6288923632617 + }, + "session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)": { + "samples": [ + 7090, + 5780, + 6660, + 6770, + 8470, + 8390, + 7280, + 6720, + 5740, + 6500 + ], + "mean": 6940, + "median": 6745, + "minimum": 5740, + "maximum": 8470, + "stdev": 925.9469627239877 + } + } + }, + "memory": { + "per_run": [ + { + "index": 0, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699632, + "events retained, then read enter props": 665208, + "session warm event cache retained": 759424, + "parse() result retained": 272688, + "parseWithDiagnostics() result retained": 260024 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2520328, + "events retained, then read every position": 2669280, + "events retained, then read enter props": 2472248, + "session warm event cache retained": 2873104, + "parse() result retained": 988144, + "parseWithDiagnostics() result retained": 987752 + } + } + }, + { + "index": 1, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 660784, + "events retained, then read every position": 699728, + "events retained, then read enter props": 656520, + "session warm event cache retained": 744600, + "parse() result retained": 272688, + "parseWithDiagnostics() result retained": 249608 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2520328, + "events retained, then read every position": 2672480, + "events retained, then read enter props": 2472032, + "session warm event cache retained": 2873104, + "parse() result retained": 988144, + "parseWithDiagnostics() result retained": 989168 + } + } + }, + { + "index": 2, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699728, + "events retained, then read enter props": 661040, + "session warm event cache retained": 744632, + "parse() result retained": 272688, + "parseWithDiagnostics() result retained": 252920 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2528120, + "events retained, then read every position": 2669616, + "events retained, then read enter props": 2472032, + "session warm event cache retained": 2873104, + "parse() result retained": 988144, + "parseWithDiagnostics() result retained": 987752 + } + } + }, + { + "index": 3, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699728, + "events retained, then read enter props": 661040, + "session warm event cache retained": 795072, + "parse() result retained": 278208, + "parseWithDiagnostics() result retained": 250072 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2528120, + "events retained, then read every position": 2675912, + "events retained, then read enter props": 2472248, + "session warm event cache retained": 2873104, + "parse() result retained": 988144, + "parseWithDiagnostics() result retained": 986944 + } + } + }, + { + "index": 4, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699728, + "events retained, then read enter props": 660952, + "session warm event cache retained": 744600, + "parse() result retained": 272688, + "parseWithDiagnostics() result retained": 249352 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2520328, + "events retained, then read every position": 2669112, + "events retained, then read enter props": 2472032, + "session warm event cache retained": 2873104, + "parse() result retained": 990392, + "parseWithDiagnostics() result retained": 986376 + } + } + }, + { + "index": 5, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699632, + "events retained, then read enter props": 656520, + "session warm event cache retained": 744632, + "parse() result retained": 272688, + "parseWithDiagnostics() result retained": 249232 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2520328, + "events retained, then read every position": 2669280, + "events retained, then read enter props": 2472248, + "session warm event cache retained": 2873104, + "parse() result retained": 990392, + "parseWithDiagnostics() result retained": 989168 + } + } + }, + { + "index": 6, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699680, + "events retained, then read enter props": 664624, + "session warm event cache retained": 744632, + "parse() result retained": 271952, + "parseWithDiagnostics() result retained": 249704 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2528120, + "events retained, then read every position": 2668160, + "events retained, then read enter props": 2472032, + "session warm event cache retained": 2873104, + "parse() result retained": 990392, + "parseWithDiagnostics() result retained": 987752 + } + } + }, + { + "index": 7, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699728, + "events retained, then read enter props": 664848, + "session warm event cache retained": 744632, + "parse() result retained": 272688, + "parseWithDiagnostics() result retained": 250032 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2520880, + "events retained, then read every position": 2669616, + "events retained, then read enter props": 2472248, + "session warm event cache retained": 2873104, + "parse() result retained": 988144, + "parseWithDiagnostics() result retained": 987752 + } + } + }, + { + "index": 8, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699728, + "events retained, then read enter props": 659160, + "session warm event cache retained": 744632, + "parse() result retained": 249616, + "parseWithDiagnostics() result retained": 251088 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2528648, + "events retained, then read every position": 2669328, + "events retained, then read enter props": 2472248, + "session warm event cache retained": 2873104, + "parse() result retained": 988144, + "parseWithDiagnostics() result retained": 987752 + } + } + }, + { + "index": 9, + "inputs": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": 652576, + "events retained, then read every position": 699728, + "events retained, then read enter props": 664624, + "session warm event cache retained": 759712, + "parse() result retained": 272688, + "parseWithDiagnostics() result retained": 249352 + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": 2528120, + "events retained, then read every position": 2669112, + "events retained, then read enter props": 2472032, + "session warm event cache retained": 2873104, + "parse() result retained": 985920, + "parseWithDiagnostics() result retained": 986424 + } + } + } + ], + "summary": { + "same-size mixed (~8 KB)": { + "events retained, no position reads": { + "samples": [ + 652576, + 660784, + 652576, + 652576, + 652576, + 652576, + 652576, + 652576, + 652576, + 652576 + ], + "mean": 653396.8, + "median": 652576, + "minimum": 652576, + "maximum": 660784, + "stdev": 2595.5975034662056 + }, + "events retained, then read every position": { + "samples": [ + 699632, + 699728, + 699728, + 699728, + 699728, + 699632, + 699680, + 699728, + 699728, + 699728 + ], + "mean": 699704, + "median": 699728, + "minimum": 699632, + "maximum": 699728, + "stdev": 40.792156108742276 + }, + "events retained, then read enter props": { + "samples": [ + 665208, + 656520, + 661040, + 661040, + 660952, + 656520, + 664624, + 664848, + 659160, + 664624 + ], + "mean": 661453.6, + "median": 661040, + "minimum": 656520, + "maximum": 665208, + "stdev": 3337.588856911195 + }, + "session warm event cache retained": { + "samples": [ + 759424, + 744600, + 744632, + 795072, + 744600, + 744632, + 744632, + 744632, + 744632, + 759712 + ], + "mean": 752656.8, + "median": 744632, + "minimum": 744600, + "maximum": 795072, + "stdev": 16146.65827690396 + }, + "parse() result retained": { + "samples": [ + 272688, + 272688, + 272688, + 278208, + 272688, + 272688, + 271952, + 272688, + 249616, + 272688 + ], + "mean": 270859.2, + "median": 272688, + "minimum": 249616, + "maximum": 278208, + "stdev": 7673.071133661033 + }, + "parseWithDiagnostics() result retained": { + "samples": [ + 260024, + 249608, + 252920, + 250072, + 249352, + 249232, + 249704, + 250032, + 251088, + 249352 + ], + "mean": 251138.4, + "median": 249868, + "minimum": 249232, + "maximum": 260024, + "stdev": 3315.088307856804 + } + }, + "synthetic article (~35-45 KB)": { + "events retained, no position reads": { + "samples": [ + 2520328, + 2520328, + 2528120, + 2528120, + 2520328, + 2520328, + 2528120, + 2520880, + 2528648, + 2528120 + ], + "mean": 2524332, + "median": 2524500, + "minimum": 2520328, + "maximum": 2528648, + "stdev": 4110.528487244 + }, + "events retained, then read every position": { + "samples": [ + 2669280, + 2672480, + 2669616, + 2675912, + 2669112, + 2669280, + 2668160, + 2669616, + 2669328, + 2669112 + ], + "mean": 2670189.6, + "median": 2669304, + "minimum": 2668160, + "maximum": 2675912, + "stdev": 2297.06688336815 + }, + "events retained, then read enter props": { + "samples": [ + 2472248, + 2472032, + 2472032, + 2472248, + 2472032, + 2472248, + 2472032, + 2472248, + 2472248, + 2472032 + ], + "mean": 2472140, + "median": 2472140, + "minimum": 2472032, + "maximum": 2472248, + "stdev": 113.84199576606166 + }, + "session warm event cache retained": { + "samples": [ + 2873104, + 2873104, + 2873104, + 2873104, + 2873104, + 2873104, + 2873104, + 2873104, + 2873104, + 2873104 + ], + "mean": 2873104, + "median": 2873104, + "minimum": 2873104, + "maximum": 2873104, + "stdev": 0 + }, + "parse() result retained": { + "samples": [ + 988144, + 988144, + 988144, + 988144, + 990392, + 990392, + 990392, + 988144, + 988144, + 985920 + ], + "mean": 988596, + "median": 988144, + "minimum": 985920, + "maximum": 990392, + "stdev": 1416.7116855592037 + }, + "parseWithDiagnostics() result retained": { + "samples": [ + 987752, + 989168, + 987752, + 986944, + 986376, + 989168, + 987752, + 987752, + 987752, + 986424 + ], + "mean": 987684, + "median": 987752, + "minimum": 986376, + "maximum": 989168, + "stdev": 958.8011032302558 + } + } + } + } +} diff --git a/experiments/event-shape-study/lazy-position/artifacts/stress-mixed-16MiB.json b/experiments/event-shape-study/lazy-position/artifacts/stress-mixed-16MiB.json new file mode 100644 index 0000000..6c07095 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/artifacts/stress-mixed-16MiB.json @@ -0,0 +1,126 @@ +{ + "schema_version": 2, + "variant": "lazy-position", + "approach_dir": "lazy-position", + "generated_at": "2026-05-19T00:48:26.803Z", + "profile": "full", + "scenario": "mixed-article", + "size_bytes": 16777216, + "size_mib": 16, + "repeats": 1, + "environment": { + "deno_version": "deno 2.7.14 / v8 14.7.173.20-rusty / typescript 5.9.2", + "platform": "aarch64-linux" + }, + "cases": [ + { + "name": "events() streamed count", + "samples": [ + { + "index": 0, + "status": "ok", + "elapsed_ms": 5377.971752, + "heap_delta_bytes": -1700960, + "checksum": 4850290 + } + ], + "ok_sample_count": 1, + "failed_sample_count": 0, + "elapsed_ms": { + "samples": [ + 5377.971752 + ], + "mean": 5377.971752, + "median": 5377.971752, + "minimum": 5377.971752, + "maximum": 5377.971752, + "stdev": 0 + }, + "heap_delta_bytes": { + "samples": [ + -1700960 + ], + "mean": -1700960, + "median": -1700960, + "minimum": -1700960, + "maximum": -1700960, + "stdev": 0 + } + }, + { + "name": "parse() child count", + "samples": [ + { + "index": 0, + "status": "ok", + "elapsed_ms": 16694.238966999998, + "heap_delta_bytes": 60328, + "checksum": 477078 + } + ], + "ok_sample_count": 1, + "failed_sample_count": 0, + "elapsed_ms": { + "samples": [ + 16694.238966999998 + ], + "mean": 16694.238966999998, + "median": 16694.238966999998, + "minimum": 16694.238966999998, + "maximum": 16694.238966999998, + "stdev": 0 + }, + "heap_delta_bytes": { + "samples": [ + 60328 + ], + "mean": 60328, + "median": 60328, + "minimum": 60328, + "maximum": 60328, + "stdev": 0 + } + }, + { + "name": "parseWithDiagnostics() tree+diagnostics count", + "samples": [ + { + "index": 0, + "status": "ok", + "elapsed_ms": 10609.558129000001, + "heap_delta_bytes": 99680, + "checksum": 477079 + } + ], + "ok_sample_count": 1, + "failed_sample_count": 0, + "elapsed_ms": { + "samples": [ + 10609.558129000001 + ], + "mean": 10609.558129000001, + "median": 10609.558129000001, + "minimum": 10609.558129000001, + "maximum": 10609.558129000001, + "stdev": 0 + }, + "heap_delta_bytes": { + "samples": [ + 99680 + ], + "mean": 99680, + "median": 99680, + "minimum": 99680, + "maximum": 99680, + "stdev": 0 + } + } + ], + "notes": [ + "Large-input stress runs are intentionally separate from the main acceptance ledger.", + "String sizes are generated on demand so the study can run MiB-scale and GiB-scale scenarios without checking giant fixtures into the repo.", + "This artifact uses the full stress profile, including tree materialization and diagnostics.", + "All requested stress cases completed successfully.", + "Scenario description: A repeated mix of headings, links, templates, tables, and refs." + ] +} diff --git a/experiments/event-shape-study/lazy-position/code/_test_utils/perf_fixtures.ts b/experiments/event-shape-study/lazy-position/code/_test_utils/perf_fixtures.ts new file mode 100644 index 0000000..286663f --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/_test_utils/perf_fixtures.ts @@ -0,0 +1,659 @@ +import type { WikitextEvent } from '../events.ts'; + +import { + blockEvents, + buildTree, + createSession, + events, + inlineEvents, + outlineEvents, + parse, + parseStrictWithDiagnostics, + parseWithDiagnostics, + tokenize, +} from '../mod.ts'; +import { UNICODE_TEXT_FIXTURES } from './unicode_fixtures.ts'; + +const CC_LF = 0x0a; +const CC_CR = 0x0d; +const CC_TAB = 0x09; +const CC_SPACE = 0x20; +const CC_BANG = 0x21; +const CC_HASH = 0x23; +const CC_AMP = 0x26; +const CC_APOSTROPHE = 0x27; +const CC_ASTERISK = 0x2a; +const CC_DASH = 0x2d; +const CC_SLASH = 0x2f; +const CC_COLON = 0x3a; +const CC_SEMICOLON = 0x3b; +const CC_LT = 0x3c; +const CC_EQUALS = 0x3d; +const CC_GT = 0x3e; +const CC_OPEN_BRACKET = 0x5b; +const CC_CLOSE_BRACKET = 0x5d; +const CC_UNDERSCORE = 0x5f; +const CC_OPEN_BRACE = 0x7b; +const CC_PIPE = 0x7c; +const CC_CLOSE_BRACE = 0x7d; +const CC_TILDE = 0x7e; + +export type RangeState = { + get(name: string): number; +}; + +export function repeatBlock(unit: string, repeat: number): string { + return unit.repeat(repeat); +} + +export function repeatToMinimumSize(unit: string, minimum_size: number): string { + const repeat = Math.ceil(minimum_size / unit.length); + return unit.repeat(repeat); +} + +export function cycleInputs(inputs: readonly T[]): () => T { + let index = 0; + + return () => { + const input = inputs[index]; + index = (index + 1) % inputs.length; + return input; + }; +} + +const PLAIN_PARAGRAPH_UNITS = [ + [ + 'Observational astronomy records atmospheric scattering, continental weathering, and navigational corrections across multiple expeditions.', + 'Field notes preserve calibration details, seasonal drift, and cross-check remarks so later readers can reconstruct the original measurement context.', + ].join(' '), + [ + 'Archival restoration teams compare transcription variants, publication histories, and editorial interventions before they publish a stable reference text.', + 'That workflow emphasizes provenance, reproducibility, and careful language around uncertainty instead of collapsing every disagreement into one canonical sentence.', + ].join(' '), + [ + 'Long-form technical prose often mixes descriptive paragraphs, cautious qualifications, and domain-specific terminology while still remaining ordinary text to the tokenizer.', + 'This fixture aims to model that kind of paragraph payload rather than a short pangram repeated until the benchmark reaches its target size.', + ].join(' '), +] as const; + +const PLAIN_WORD_BOUNDARY_STRESS_UNIT = [ + 'a a a a a a a a a a a a a a a a', + 'b b b b b b b b b b b b b b b b', + 'c c c c c c c c c c c c c c c c', +].join(' '); + +export function repeatParagraphText(unit: string, minimum_size: number): string { + return repeatToMinimumSize(`${unit}\n\n`, minimum_size); +} + +export function repeatWordBoundaryStressText(minimum_size: number): string { + return repeatToMinimumSize(`${PLAIN_WORD_BOUNDARY_STRESS_UNIT}\n`, minimum_size); +} + +export const TOKENIZER_SCAN_DELIMITER = Uint8Array.from({ length: 128 }, (_, code) => { + switch (code) { + case CC_LF: + case CC_CR: + case CC_TAB: + case CC_SPACE: + case CC_BANG: + case CC_HASH: + case CC_AMP: + case CC_APOSTROPHE: + case CC_ASTERISK: + case CC_DASH: + case CC_SLASH: + case CC_COLON: + case CC_SEMICOLON: + case CC_LT: + case CC_EQUALS: + case CC_GT: + case CC_OPEN_BRACKET: + case CC_CLOSE_BRACKET: + case CC_UNDERSCORE: + case CC_OPEN_BRACE: + case CC_PIPE: + case CC_CLOSE_BRACE: + case CC_TILDE: + return 1; + + default: + return 0; + } +}); + +export const PLAIN_TEXT_INPUTS = [ + repeatParagraphText(PLAIN_PARAGRAPH_UNITS[0], 9 * 1024), + repeatParagraphText(PLAIN_PARAGRAPH_UNITS[1], 9 * 1024), + repeatParagraphText(PLAIN_PARAGRAPH_UNITS[2], 9 * 1024), +] as const; + +export const PLAIN_TOKEN_DENSITY_STRESS_INPUTS = [ + repeatWordBoundaryStressText(9 * 1024), + repeatToMinimumSize(`${PLAIN_WORD_BOUNDARY_STRESS_UNIT}\t${PLAIN_WORD_BOUNDARY_STRESS_UNIT}\n`, 9 * 1024), +] as const; + +export const HEADING_TEXT_INPUTS = [ + '== Section ==\nParagraph text here.\n'.repeat(100), + '=== Nested ===\nAnother paragraph line.\n'.repeat(90), +] as const; + +export const TABLE_TEXT_INPUTS = [ + '{|\n! H1 !! H2\n|-\n| A || B\n|-\n| C || D\n|}\n'.repeat(50), + '{| class="wikitable"\n! Name !! Value\n|-\n| Alpha || 1\n|-\n| Beta || 2\n|}\n'.repeat(40), +] as const; + +export const LINK_TEXT_INPUTS = [ + "See [[Main Page|home]], '''bold''' and ''italic'' text.\n".repeat(100), + 'Visit [[Earth|planet]] with [https://example.com source] and & notes.\n'.repeat(80), +] as const; + +export const TEMPLATE_TEXT_INPUTS = [ + '{{Infobox|name={{{1}}}|value={{{2|default}}}}}\n'.repeat(100), + '{{Card|title={{PAGENAME}}|body={{{content|fallback}}}}}\n'.repeat(85), +] as const; + +export const MIXED_TEXT_INPUTS = [ + [ + '== Heading ==', + "'''Bold''' and ''italic'' and '''''both'''''.", + '* Bullet item', + '# Ordered item', + ': Indented', + '{|', + '! Header', + '|-', + '| [[Page|link]] || {{template|arg=val}}', + '|}', + '----', + '', + '& { 💩', + '~~~~ __TOC__', + '', + ].join('\n').repeat(50), + [ + '== Another Heading ==', + "A [[Main Page|home]] link with ''italic'' and '''bold'''.", + '; Term', + ': Definition', + '{|', + '! Name !! Count', + '|-', + '| {{Item|name=Alpha}} || 42', + '|}', + 'inline', + '<escaped> ~~ ~~ __TOC__', + '', + ].join('\n').repeat(48), +] as const; + +export const PATHOLOGICAL_TEXT_INPUTS = [ + [ + '[[[[{{{{ → Comment (value: ' hidden ') + +/** + * Literal text content. The leaf node for all inline text that is not + * markup. + */ +export interface Text extends WikistNodeBase { + /** Node type discriminant. */ + readonly type: 'text'; + /** The text content. */ + readonly value: string; +} + +/** + * HTML character entity: `&`, `{`, `{`. + */ +export interface HtmlEntity extends WikistNodeBase { + /** Node type discriminant. */ + readonly type: 'html-entity'; + /** Raw entity text including `&` and `;` (e.g., `"&"`). */ + readonly value: string; +} + +/** + * Content inside `...` tags. Markup within is not + * parsed. + */ +export interface Nowiki extends WikistNodeBase { + /** Node type discriminant. */ + readonly type: 'nowiki'; + /** Raw text content (not parsed for markup). */ + readonly value: string; +} + +/** + * HTML comment: ``. + */ +export interface Comment extends WikistNodeBase { + /** Node type discriminant. */ + readonly type: 'comment'; + /** Comment content between ``. */ + readonly value: string; +} + +// --------------------------------------------------------------------------- +// Reserved +// --------------------------------------------------------------------------- + +/** + * Reserved for future collaboration support. Represents multiple possible + * variants for a range of content, inspired by jujutsu's "conflict as + * value" model. + * + * **Not produced by the core parser.** Exists as a reserved slot so that + * collaboration tooling can represent unresolved conflicts in the tree + * without requiring a breaking AST change later. + */ +export interface Conflict extends WikistNodeBase { + /** Node type discriminant. */ + readonly type: 'conflict'; + /** Each variant is a list of children representing one conflict side. */ + readonly variants: WikistNode[][]; +} + +// --------------------------------------------------------------------------- +// Discriminated union +// --------------------------------------------------------------------------- +// +// WikistNode is a TypeScript "discriminated union": a union of interfaces +// that all share a common `type` field with a unique string literal value. +// This lets the compiler narrow the type inside a switch statement: +// +// switch (node.type) { +// case 'heading': node.level; // TypeScript knows this is a Heading +// case 'text': node.value; // TypeScript knows this is a Text +// case 'bold': node.children; // TypeScript knows this is a Bold +// } +// +// Why this matters: without the union, you would need explicit casts or +// type assertions to access type-specific fields. The discriminated union +// makes the compiler do the narrowing for you, catching mismatches at +// compile time: +// +// if (node.type === 'heading') { +// node.level; // level is accessible here +// node.value; // compile error: Heading has no `value` +// } +// +// The pattern appears throughout this codebase: WikistNode, WikitextEvent, +// and TokenType all use discriminated unions for type-safe branching. + +/** + * Discriminated union of all wikist node types. Switch on `node.type` + * for exhaustive pattern matching. + * + * @example Exhaustive switch + * ```ts + * import type { WikistNode } from './ast.ts'; + * + * function nodeLabel(node: WikistNode): string { + * switch (node.type) { + * case 'root': return 'document'; + * case 'heading': return `h${node.level}`; + * case 'text': return node.value; + * default: return node.type; + * } + * } + * ``` + */ +export type WikistNode = + | Root + | Heading + | Paragraph + | ThematicBreak + | Preformatted + | List + | ListItem + | DefinitionList + | DefinitionTerm + | DefinitionDescription + | Table + | TableCaption + | TableRow + | TableCell + | Bold + | Italic + | BoldItalic + | Wikilink + | ExternalLink + | ImageLink + | CategoryLink + | Template + | TemplateArgument + | Argument + | ParserFunction + | MagicWord + | BehaviorSwitch + | HtmlTag + | HtmlEntity + | Text + | Nowiki + | Comment + | Redirect + | Signature + | Break + | Gallery + | Reference + | Conflict; + +/** + * String literal union of all wikist node type discriminants. + * + * Derived from {@linkcode WikistNode} for type-safe switching and + * mapping. + */ +export type WikistNodeType = WikistNode['type']; + +/** Alias for the root node type returned by `parse()`. */ +export type WikistRoot = Root; + +// --------------------------------------------------------------------------- +// Category aliases +// --------------------------------------------------------------------------- +// +// These union types group nodes by structural category. They are useful +// for generic tree-walking code that cares about "does this node have +// children?" rather than "is this specific node type a heading or a bold?" +// +// isParent(node) narrows WikistNode → WikistParent (has children) +// isLiteral(node) narrows WikistNode → WikistLiteral (has value) +// WikistVoid covers the rest (has neither) + +/** + * Union of all parent node types (nodes with a `children` field). + */ +export type WikistParent = + | Root + | Heading + | Paragraph + | Preformatted + | List + | ListItem + | DefinitionList + | DefinitionTerm + | DefinitionDescription + | Table + | TableCaption + | TableRow + | TableCell + | Bold + | Italic + | BoldItalic + | Wikilink + | ExternalLink + | ImageLink + | Template + | TemplateArgument + | ParserFunction + | HtmlTag + | Redirect + | Gallery + | Reference; + +/** + * Union of all literal node types (nodes with a `value` field). + */ +export type WikistLiteral = + | HtmlEntity + | Text + | Nowiki + | Comment; + +/** + * Union of all void node types (no `children`, no `value`). + */ +export type WikistVoid = + | ThematicBreak + | CategoryLink + | Argument + | MagicWord + | BehaviorSwitch + | Signature + | Break; + +// --------------------------------------------------------------------------- +// Type guards +// --------------------------------------------------------------------------- +// +// Type guards are functions that narrow a general WikistNode to a specific +// type. They return a "type predicate" (e.g., `node is Heading`) that tells +// TypeScript to narrow the type inside an `if` block. +// +// Two kinds of type guards: +// +// 1. Category guards: isParent(node), isLiteral(node) +// Use structural checks ('children' in node, 'value' in node). +// Work with future node types too. +// +// 2. Specific guards: isHeading(node), isText(node), etc. +// Check node.type === 'heading', node.type === 'text', etc. +// Give access to type-specific fields (level, value, target...). +// +// Type guards are especially useful as callbacks: +// const headings = nodes.filter(isHeading); // Heading[] + +/** + * Narrow a {@linkcode WikistNode} to any parent node (has `children`). + * + * Uses the `children` property as a structural check rather than + * enumerating all parent types, so it also works with future parent + * node types. + * + * @example Recursively visiting children + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isParent } from './ast.ts'; + * + * function visit(node: WikistNode, fn: (n: WikistNode) => void) { + * fn(node); + * if (isParent(node)) node.children.forEach(child => visit(child, fn)); + * } + * ``` + * + * @example Filtering parent nodes from a flat list + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isParent } from './ast.ts'; + * + * function parents(nodes: WikistNode[]) { + * return nodes.filter(isParent); + * } + * ``` + */ +export function isParent(node: WikistNode): node is WikistParent { + return 'children' in node; +} + +/** + * Narrow a {@linkcode WikistNode} to any literal node (has `value`). + * + * @example Collecting all literal values + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isLiteral } from './ast.ts'; + * + * function literalValues(nodes: WikistNode[]): string[] { + * return nodes.filter(isLiteral).map(n => n.value); + * } + * ``` + * + * @example Narrowing to access the value field + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isLiteral } from './ast.ts'; + * + * function showNode(node: WikistNode) { + * if (isLiteral(node)) console.log('literal:', node.value); + * } + * ``` + */ +export function isLiteral(node: WikistNode): node is WikistLiteral { + return 'value' in node; +} + +/** Narrow to {@linkcode Root}. */ +export function isRoot(node: WikistNode): node is Root { + return node.type === 'root'; +} + +/** + * Narrow to {@linkcode Heading}. + * + * @example Extracting all headings from a tree + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isHeading, isParent } from './ast.ts'; + * + * function headings(node: WikistNode): WikistNode[] { + * const result: WikistNode[] = []; + * if (isHeading(node)) result.push(node); + * if (isParent(node)) node.children.forEach(c => result.push(...headings(c))); + * return result; + * } + * ``` + */ +export function isHeading(node: WikistNode): node is Heading { + return node.type === 'heading'; +} + +/** Narrow to {@linkcode Paragraph}. */ +export function isParagraph(node: WikistNode): node is Paragraph { + return node.type === 'paragraph'; +} + +/** Narrow to {@linkcode ThematicBreak}. */ +export function isThematicBreak(node: WikistNode): node is ThematicBreak { + return node.type === 'thematic-break'; +} + +/** Narrow to {@linkcode Preformatted}. */ +export function isPreformatted(node: WikistNode): node is Preformatted { + return node.type === 'preformatted'; +} + +/** Narrow to {@linkcode List}. */ +export function isList(node: WikistNode): node is List { + return node.type === 'list'; +} + +/** Narrow to {@linkcode ListItem}. */ +export function isListItem(node: WikistNode): node is ListItem { + return node.type === 'list-item'; +} + +/** Narrow to {@linkcode DefinitionList}. */ +export function isDefinitionList(node: WikistNode): node is DefinitionList { + return node.type === 'definition-list'; +} + +/** Narrow to {@linkcode DefinitionTerm}. */ +export function isDefinitionTerm(node: WikistNode): node is DefinitionTerm { + return node.type === 'definition-term'; +} + +/** Narrow to {@linkcode DefinitionDescription}. */ +export function isDefinitionDescription(node: WikistNode): node is DefinitionDescription { + return node.type === 'definition-description'; +} + +/** Narrow to {@linkcode Table}. */ +export function isTable(node: WikistNode): node is Table { + return node.type === 'table'; +} + +/** Narrow to {@linkcode TableCaption}. */ +export function isTableCaption(node: WikistNode): node is TableCaption { + return node.type === 'table-caption'; +} + +/** Narrow to {@linkcode TableRow}. */ +export function isTableRow(node: WikistNode): node is TableRow { + return node.type === 'table-row'; +} + +/** Narrow to {@linkcode TableCell}. */ +export function isTableCell(node: WikistNode): node is TableCell { + return node.type === 'table-cell'; +} + +/** Narrow to {@linkcode Bold}. */ +export function isBold(node: WikistNode): node is Bold { + return node.type === 'bold'; +} + +/** Narrow to {@linkcode Italic}. */ +export function isItalic(node: WikistNode): node is Italic { + return node.type === 'italic'; +} + +/** Narrow to {@linkcode BoldItalic}. */ +export function isBoldItalic(node: WikistNode): node is BoldItalic { + return node.type === 'bold-italic'; +} + +/** + * Narrow to {@linkcode Wikilink}. + * + * @example Finding all wikilinks in a tree + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isWikilink, isParent } from './ast.ts'; + * + * function links(node: WikistNode): string[] { + * const result: string[] = []; + * if (isWikilink(node)) result.push(node.target); + * if (isParent(node)) node.children.forEach(c => result.push(...links(c))); + * return result; + * } + * ``` + */ +export function isWikilink(node: WikistNode): node is Wikilink { + return node.type === 'wikilink'; +} + +/** Narrow to {@linkcode ExternalLink}. */ +export function isExternalLink(node: WikistNode): node is ExternalLink { + return node.type === 'external-link'; +} + +/** Narrow to {@linkcode ImageLink}. */ +export function isImageLink(node: WikistNode): node is ImageLink { + return node.type === 'image-link'; +} + +/** Narrow to {@linkcode CategoryLink}. */ +export function isCategoryLink(node: WikistNode): node is CategoryLink { + return node.type === 'category-link'; +} + +/** + * Narrow to {@linkcode Template}. + * + * @example Extracting template names + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isTemplate, isParent } from './ast.ts'; + * + * function templateNames(node: WikistNode): string[] { + * const result: string[] = []; + * if (isTemplate(node)) result.push(node.name); + * if (isParent(node)) node.children.forEach(c => result.push(...templateNames(c))); + * return result; + * } + * ``` + */ +export function isTemplate(node: WikistNode): node is Template { + return node.type === 'template'; +} + +/** Narrow to {@linkcode TemplateArgument}. */ +export function isTemplateArgument(node: WikistNode): node is TemplateArgument { + return node.type === 'template-argument'; +} + +/** Narrow to {@linkcode Argument}. */ +export function isArgument(node: WikistNode): node is Argument { + return node.type === 'argument'; +} + +/** Narrow to {@linkcode ParserFunction}. */ +export function isParserFunction(node: WikistNode): node is ParserFunction { + return node.type === 'parser-function'; +} + +/** Narrow to {@linkcode MagicWord}. */ +export function isMagicWord(node: WikistNode): node is MagicWord { + return node.type === 'magic-word'; +} + +/** Narrow to {@linkcode BehaviorSwitch}. */ +export function isBehaviorSwitch(node: WikistNode): node is BehaviorSwitch { + return node.type === 'behavior-switch'; +} + +/** Narrow to {@linkcode HtmlTag}. */ +export function isHtmlTag(node: WikistNode): node is HtmlTag { + return node.type === 'html-tag'; +} + +/** Narrow to {@linkcode HtmlEntity}. */ +export function isHtmlEntity(node: WikistNode): node is HtmlEntity { + return node.type === 'html-entity'; +} + +/** + * Narrow to {@linkcode Text}. + * + * @example Checking for text nodes + * ```ts + * import type { WikistNode } from './ast.ts'; + * import { isText } from './ast.ts'; + * + * function isLeafText(node: WikistNode): boolean { + * return isText(node); + * } + * ``` + */ +export function isText(node: WikistNode): node is Text { + return node.type === 'text'; +} + +/** Narrow to {@linkcode Nowiki}. */ +export function isNowiki(node: WikistNode): node is Nowiki { + return node.type === 'nowiki'; +} + +/** Narrow to {@linkcode Comment}. */ +export function isComment(node: WikistNode): node is Comment { + return node.type === 'comment'; +} + +/** Narrow to {@linkcode Redirect}. */ +export function isRedirect(node: WikistNode): node is Redirect { + return node.type === 'redirect'; +} + +/** Narrow to {@linkcode Signature}. */ +export function isSignature(node: WikistNode): node is Signature { + return node.type === 'signature'; +} + +/** Narrow to {@linkcode Break}. */ +export function isBreak(node: WikistNode): node is Break { + return node.type === 'break'; +} + +/** Narrow to {@linkcode Gallery}. */ +export function isGallery(node: WikistNode): node is Gallery { + return node.type === 'gallery'; +} + +/** Narrow to {@linkcode Reference}. */ +export function isReference(node: WikistNode): node is Reference { + return node.type === 'reference'; +} + +// --------------------------------------------------------------------------- +// Builder functions +// --------------------------------------------------------------------------- +// +// Builder functions create wikist nodes with the correct `type` discriminant +// set automatically. They are the primary way to construct trees +// programmatically (e.g., in tests, code generators, or transformers). +// +// Each builder returns a plain object — no classes, no prototypes. This +// keeps nodes JSON-serializable and structurally compatible with unist. +// +// Example: building a small document tree +// +// const tree = root([ +// heading(2, [text('Hello')]), +// paragraph([text('Some '), bold([text('bold')]), text(' text.')]), +// ]); + +/** + * Create a {@linkcode Root} node. + * + * @example Building a document tree + * ```ts + * import { root, paragraph, text } from './ast.ts'; + * + * const tree = root([paragraph([text('Hello world.')])]); + * tree.type; // 'root' + * tree.children.length; // 1 + * ``` + * + * @example Empty document + * ```ts + * import { root } from './ast.ts'; + * + * const empty = root([]); + * empty.children.length; // 0 + * ``` + */ +export function root(children: WikistNode[]): Root { + return { type: 'root', children }; +} + +/** + * Create a {@linkcode Heading} node. + * + * @example Level 2 heading with text + * ```ts + * import { heading, text } from './ast.ts'; + * + * const h2 = heading(2, [text('Introduction')]); + * h2.level; // 2 + * ``` + * + * @example Level 1 heading with formatted content + * ```ts + * import { heading, bold, text } from './ast.ts'; + * + * const h1 = heading(1, [bold([text('Important')])]); + * h1.level; // 1 + * ``` + */ +export function heading( + level: 1 | 2 | 3 | 4 | 5 | 6, + children: WikistNode[], +): Heading { + return { type: 'heading', level, children }; +} + +/** + * Create a {@linkcode Paragraph} node. + * + * @example Simple paragraph + * ```ts + * import { paragraph, text } from './ast.ts'; + * + * const p = paragraph([text('Some text.')]); + * p.type; // 'paragraph' + * ``` + */ +export function paragraph(children: WikistNode[]): Paragraph { + return { type: 'paragraph', children }; +} + +/** + * Create a {@linkcode ThematicBreak} node. + * + * @example Inserting a horizontal rule + * ```ts + * import { thematicBreak } from './ast.ts'; + * + * const hr = thematicBreak(); + * hr.type; // 'thematic-break' + * ``` + */ +export function thematicBreak(): ThematicBreak { + return { type: 'thematic-break' }; +} + +/** + * Create a {@linkcode Preformatted} node. + * + * @example Preformatted block + * ```ts + * import { preformatted, text } from './ast.ts'; + * + * const pre = preformatted([text(' code here')]); + * pre.type; // 'preformatted' + * ``` + */ +export function preformatted(children: WikistNode[]): Preformatted { + return { type: 'preformatted', children }; +} + +/** + * Create a {@linkcode List} node. + * + * @example Bullet list + * ```ts + * import { list, listItem, text } from './ast.ts'; + * + * const ul = list(false, [listItem('*', [text('item')])]); + * ul.ordered; // false + * ``` + * + * @example Ordered list + * ```ts + * import { list, listItem, text } from './ast.ts'; + * + * const ol = list(true, [listItem('#', [text('first')])]); + * ol.ordered; // true + * ``` + */ +export function list(ordered: boolean, children: ListItem[]): List { + return { type: 'list', ordered, children }; +} + +/** + * Create a {@linkcode ListItem} node. + * + * @example Bullet list item + * ```ts + * import { listItem, text } from './ast.ts'; + * + * const item = listItem('*', [text('bullet point')]); + * item.marker; // '*' + * ``` + */ +export function listItem(marker: string, children: WikistNode[]): ListItem { + return { type: 'list-item', marker, children }; +} + +/** + * Create a {@linkcode DefinitionList} node. + * + * @example Definition list with term and description + * ```ts + * import { definitionList, definitionTerm, definitionDescription, text } from './ast.ts'; + * + * const dl = definitionList([ + * definitionTerm([text('Term')]), + * definitionDescription([text('Description')]), + * ]); + * dl.type; // 'definition-list' + * ``` + */ +export function definitionList( + children: (DefinitionTerm | DefinitionDescription)[], +): DefinitionList { + return { type: 'definition-list', children }; +} + +/** + * Create a {@linkcode DefinitionTerm} node. + * + * @example Simple term + * ```ts + * import { definitionTerm, text } from './ast.ts'; + * + * const dt = definitionTerm([text('Key')]); + * dt.type; // 'definition-term' + * ``` + */ +export function definitionTerm(children: WikistNode[]): DefinitionTerm { + return { type: 'definition-term', children }; +} + +/** + * Create a {@linkcode DefinitionDescription} node. + * + * @example Simple description + * ```ts + * import { definitionDescription, text } from './ast.ts'; + * + * const dd = definitionDescription([text('Value')]); + * dd.type; // 'definition-description' + * ``` + */ +export function definitionDescription(children: WikistNode[]): DefinitionDescription { + return { type: 'definition-description', children }; +} + +/** + * Create a {@linkcode Table} node. + * + * @example Table with one row + * ```ts + * import { table, tableRow, tableCell, text } from './ast.ts'; + * + * const t = table([tableRow([tableCell(false, [text('cell')])])]); + * t.type; // 'table' + * ``` + */ +export function table( + children: (TableCaption | TableRow)[], + attributes?: string, +): Table { + // Optional fields stay omitted instead of being set to `undefined` so the + // serialized tree stays compact and debug output reflects what was actually + // present in source. + return attributes !== undefined + ? { type: 'table', attributes, children } + : { type: 'table', children }; +} + +/** + * Create a {@linkcode TableCaption} node. + * + * @example Caption text + * ```ts + * import { tableCaption, text } from './ast.ts'; + * + * const cap = tableCaption([text('Table title')]); + * cap.type; // 'table-caption' + * ``` + */ +export function tableCaption(children: WikistNode[]): TableCaption { + return { type: 'table-caption', children }; +} + +/** + * Create a {@linkcode TableRow} node. + * + * @example Row with attributes + * ```ts + * import { tableRow, tableCell, text } from './ast.ts'; + * + * const row = tableRow([tableCell(false, [text('data')])], 'class="highlight"'); + * row.attributes; // 'class="highlight"' + * ``` + */ +export function tableRow( + children: TableCell[], + attributes?: string, +): TableRow { + // Table rows use the same omission rule as tables: absent attributes should + // not become noisy `attributes: undefined` fields in snapshots or JSON. + return attributes !== undefined + ? { type: 'table-row', attributes, children } + : { type: 'table-row', children }; +} + +/** + * Create a {@linkcode TableCell} node. + * + * @example Data cell + * ```ts + * import { tableCell, text } from './ast.ts'; + * + * const td = tableCell(false, [text('data')]); + * td.header; // false + * ``` + * + * @example Header cell + * ```ts + * import { tableCell, text } from './ast.ts'; + * + * const th = tableCell(true, [text('Header')]); + * th.header; // true + * ``` + */ +export function tableCell( + header: boolean, + children: WikistNode[], + attributes?: string, +): TableCell { + // Header/data status is structural and always explicit. Attributes are not, + // so we only materialize them when the source actually carried them. + return attributes !== undefined + ? { type: 'table-cell', header, attributes, children } + : { type: 'table-cell', header, children }; +} + +/** + * Create a {@linkcode Bold} node. + * + * @example Bold text + * ```ts + * import { bold, text } from './ast.ts'; + * + * const b = bold([text('strong')]); + * b.type; // 'bold' + * ``` + */ +export function bold(children: WikistNode[]): Bold { + return { type: 'bold', children }; +} + +/** + * Create an {@linkcode Italic} node. + * + * @example Italic text + * ```ts + * import { italic, text } from './ast.ts'; + * + * const em = italic([text('emphasis')]); + * em.type; // 'italic' + * ``` + */ +export function italic(children: WikistNode[]): Italic { + return { type: 'italic', children }; +} + +/** + * Create a {@linkcode BoldItalic} node. + * + * @example Bold italic text + * ```ts + * import { boldItalic, text } from './ast.ts'; + * + * const bi = boldItalic([text('both')]); + * bi.type; // 'bold-italic' + * ``` + */ +export function boldItalic(children: WikistNode[]): BoldItalic { + return { type: 'bold-italic', children }; +} + +/** + * Create a {@linkcode Wikilink} node. + * + * @example Link with display text + * ```ts + * import { wikilink, text } from './ast.ts'; + * + * const link = wikilink('Main Page', [text('home')]); + * link.target; // 'Main Page' + * ``` + * + * @example Link with no display text (target is used) + * ```ts + * import { wikilink } from './ast.ts'; + * + * const link = wikilink('Help:Contents', []); + * link.children.length; // 0 + * ``` + */ +export function wikilink(target: string, children: WikistNode[]): Wikilink { + return { type: 'wikilink', target, children }; +} + +/** + * Create an {@linkcode ExternalLink} node. + * + * @example External link with label + * ```ts + * import { externalLink, text } from './ast.ts'; + * + * const link = externalLink('https://example.com', [text('Example')]); + * link.url; // 'https://example.com' + * ``` + */ +export function externalLink(url: string, children: WikistNode[]): ExternalLink { + return { type: 'external-link', url, children }; +} + +/** + * Create an {@linkcode ImageLink} node. + * + * @example Image with caption + * ```ts + * import { imageLink, text } from './ast.ts'; + * + * const img = imageLink('File:Photo.jpg', [text('A photo')]); + * img.target; // 'File:Photo.jpg' + * ``` + */ +export function imageLink(target: string, children: WikistNode[]): ImageLink { + return { type: 'image-link', target, children }; +} + +/** + * Create a {@linkcode CategoryLink} node. + * + * @example Category with sort key + * ```ts + * import { categoryLink } from './ast.ts'; + * + * const cat = categoryLink('Science', 'Physics'); + * cat.sort_key; // 'Physics' + * ``` + * + * @example Category without sort key + * ```ts + * import { categoryLink } from './ast.ts'; + * + * const cat = categoryLink('Articles'); + * cat.sort_key; // undefined + * ``` + */ +export function categoryLink(target: string, sort_key?: string): CategoryLink { + // Category links are often compared or serialized by tools, so leaving the + // optional sort key absent is cleaner than storing an explicit undefined. + return sort_key !== undefined + ? { type: 'category-link', target, sort_key } + : { type: 'category-link', target }; +} + +/** + * Create a {@linkcode Template} node. + * + * @example Template with arguments + * ```ts + * import { template, templateArgument, text } from './ast.ts'; + * + * const t = template('Infobox', [ + * templateArgument([text('value')]), + * templateArgument([text('named')], 'key'), + * ]); + * t.name; // 'Infobox' + * ``` + * + * @example Template with no arguments + * ```ts + * import { template } from './ast.ts'; + * + * const t = template('Stub', []); + * t.children.length; // 0 + * ``` + */ +export function template(name: string, children: TemplateArgument[]): Template { + return { type: 'template', name, children }; +} + +/** + * Create a {@linkcode TemplateArgument} node. + * + * @example Positional argument + * ```ts + * import { templateArgument, text } from './ast.ts'; + * + * const arg = templateArgument([text('value')]); + * arg.name; // undefined (positional) + * ``` + * + * @example Named argument + * ```ts + * import { templateArgument, text } from './ast.ts'; + * + * const arg = templateArgument([text('bar')], 'foo'); + * arg.name; // 'foo' + * ``` + */ +export function templateArgument( + children: WikistNode[], + name?: string, +): TemplateArgument { + // Named and positional arguments share one node type. Omitting `name` is the + // signal that the argument was positional in source. + return name !== undefined + ? { type: 'template-argument', name, children } + : { type: 'template-argument', children }; +} + +/** + * Create an {@linkcode Argument} node (triple-brace parameter). + * + * @example Parameter with default + * ```ts + * import { argument } from './ast.ts'; + * + * const arg = argument('title', 'Untitled'); + * arg.default; // 'Untitled' + * ``` + * + * @example Parameter without default + * ```ts + * import { argument } from './ast.ts'; + * + * const arg = argument('name'); + * arg.default; // undefined + * ``` + */ +export function argument(name: string, defaultValue?: string): Argument { + return defaultValue !== undefined + ? { type: 'argument', name, default: defaultValue } + : { type: 'argument', name }; +} + +/** + * Create a {@linkcode ParserFunction} node. + * + * @example If parser function + * ```ts + * import { parserFunction, templateArgument, text } from './ast.ts'; + * + * const fn = parserFunction('#if', [ + * templateArgument([text('condition')]), + * templateArgument([text('then')]), + * ]); + * fn.name; // '#if' + * ``` + */ +export function parserFunction( + name: string, + children: TemplateArgument[], +): ParserFunction { + return { type: 'parser-function', name, children }; +} + +/** + * Create a {@linkcode MagicWord} node. + * + * @example Page name magic word + * ```ts + * import { magicWord } from './ast.ts'; + * + * const mw = magicWord('PAGENAME'); + * mw.name; // 'PAGENAME' + * ``` + */ +export function magicWord(name: string): MagicWord { + return { type: 'magic-word', name }; +} + +/** + * Create a {@linkcode BehaviorSwitch} node. + * + * @example TOC switch + * ```ts + * import { behaviorSwitch } from './ast.ts'; + * + * const sw = behaviorSwitch('TOC'); + * sw.name; // 'TOC' + * ``` + */ +export function behaviorSwitch(name: string): BehaviorSwitch { + return { type: 'behavior-switch', name }; +} + +/** + * Create an {@linkcode HtmlTag} node. + * + * @example A div tag with content + * ```ts + * import { htmlTag, text } from './ast.ts'; + * + * const div = htmlTag('div', false, [text('content')], { class: 'note' }); + * div.tag_name; // 'div' + * div.self_closing; // false + * div.attributes; // { class: 'note' } + * ``` + * + * @example A self-closing br tag + * ```ts + * import { htmlTag } from './ast.ts'; + * + * const br = htmlTag('br', true, []); + * br.self_closing; // true + * ``` + */ +export function htmlTag( + tag_name: string, + self_closing: boolean, + children: WikistNode[], + attributes?: Readonly>, +): HtmlTag { + // Self-closing tags still use the same builder so callers can construct one + // consistent node shape and let `self_closing` carry the semantic difference. + return attributes !== undefined + ? { type: 'html-tag', tag_name, self_closing, attributes, children } + : { type: 'html-tag', tag_name, self_closing, children }; +} + +/** + * Create an {@linkcode HtmlEntity} node. + * + * @example Named entity + * ```ts + * import { htmlEntity } from './ast.ts'; + * + * const ent = htmlEntity('&'); + * ent.value; // '&' + * ``` + */ +export function htmlEntity(value: string): HtmlEntity { + return { type: 'html-entity', value }; +} + +/** + * Create a {@linkcode Text} node. + * + * @example Simple text leaf + * ```ts + * import { text } from './ast.ts'; + * + * const t = text('Hello world'); + * t.value; // 'Hello world' + * ``` + * + * @example Empty text node + * ```ts + * import { text } from './ast.ts'; + * + * const t = text(''); + * t.value; // '' + * ``` + */ +export function text(value: string): Text { + return { type: 'text', value }; +} + +/** + * Create a {@linkcode Nowiki} node. + * + * @example Nowiki content + * ```ts + * import { nowiki } from './ast.ts'; + * + * const nw = nowiki('[[not a link]]'); + * nw.value; // '[[not a link]]' + * ``` + */ +export function nowiki(value: string): Nowiki { + return { type: 'nowiki', value }; +} + +/** + * Create a {@linkcode Comment} node. + * + * @example HTML comment + * ```ts + * import { comment } from './ast.ts'; + * + * const c = comment('hidden note'); + * c.value; // 'hidden note' + * ``` + */ +export function comment(value: string): Comment { + return { type: 'comment', value }; +} + +/** + * Create a {@linkcode Redirect} node. + * + * @example Page redirect + * ```ts + * import { redirect, wikilink } from './ast.ts'; + * + * const r = redirect('Main Page', [wikilink('Main Page', [])]); + * r.target; // 'Main Page' + * ``` + */ +export function redirect(target: string, children: WikistNode[]): Redirect { + return { type: 'redirect', target, children }; +} + +/** + * Create a {@linkcode Signature} node. + * + * @example Four-tilde signature (username + timestamp) + * ```ts + * import { signature } from './ast.ts'; + * + * const sig = signature(4); + * sig.tildes; // 4 + * ``` + */ +export function signature(tildes: 3 | 4 | 5): Signature { + return { type: 'signature', tildes }; +} + +/** + * Create a {@linkcode Break} node (explicit `
` line break). + * + * Named `lineBreak` to avoid collision with the `break` reserved word. + * + * @example Line break + * ```ts + * import { lineBreak } from './ast.ts'; + * + * const br = lineBreak(); + * br.type; // 'break' + * ``` + */ +export function lineBreak(): Break { + // `lineBreak` is the exported name because `break` would collide with the + // JavaScript keyword at call sites. + return { type: 'break' }; +} + +/** + * Create a {@linkcode Gallery} node. + * + * @example Gallery with attributes + * ```ts + * import { gallery, text } from './ast.ts'; + * + * const g = gallery([text('File:A.png')], { mode: 'packed' }); + * g.attributes; // { mode: 'packed' } + * ``` + */ +export function gallery( + children: WikistNode[], + attributes?: Readonly>, +): Gallery { + // Like htmlTag(), the gallery builder keeps optional attributes sparse so + // tooling sees the same shape whether the tree came from parsing or manual + // construction. + return attributes !== undefined + ? { type: 'gallery', attributes, children } + : { type: 'gallery', children }; +} + +/** + * Create a {@linkcode Reference} node. + * + * @example Named reference + * ```ts + * import { reference, text } from './ast.ts'; + * + * const ref = reference([text('Source text.')], 'cite1', 'note'); + * ref.name; // 'cite1' + * ref.group; // 'note' + * ``` + * + * @example Anonymous reference + * ```ts + * import { reference, text } from './ast.ts'; + * + * const ref = reference([text('Inline citation.')]); + * ref.name; // undefined + * ``` + */ +export function reference( + children: WikistNode[], + name?: string, + group?: string, +): Reference { + // References have two independent optional metadata fields. Building the node + // step by step keeps the runtime shape obvious and avoids attaching metadata + // that the caller did not actually provide. + const node: Reference = { type: 'reference', children }; + if (name !== undefined) { + return group !== undefined + ? { type: 'reference', name, group, children } + : { type: 'reference', name, children }; + } + return node; +} diff --git a/experiments/event-shape-study/lazy-position/code/block_parser.ts b/experiments/event-shape-study/lazy-position/code/block_parser.ts new file mode 100644 index 0000000..364e373 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/block_parser.ts @@ -0,0 +1,1507 @@ +/** + * Block-level parser that turns a token stream into structural events. + * + * The tokenizer only marks raw source pieces such as `==`, `*`, `{|`, and + * plain text. This file is the next step. It reads those tokens line by line + * and answers a more useful question: what kind of block does this line begin? + * + * In practical terms, this stage decides things like: + * + * - does this line start a heading? + * - is this the start of a bullet list or numbered list? + * - are we entering a table? + * - should this text become a paragraph? + * + * The output is still an event stream rather than a tree. That keeps it useful + * for streaming callers and lets later stages, especially the inline parser, + * enrich the structure without rebuilding everything from scratch. + * + * ``` + * TextSource -> tokenize() -> blockEvents() -> [inline parser] -> [consumers] + * ``` + * + * The parser is line-oriented. That means the first meaningful token on a line + * usually decides what kind of block the line belongs to. + * + * ``` + * first token on line result + * ------------------- ------------------------- + * HEADING_MARKER heading + * BULLET bullet list item + * HASH numbered list item + * SEMICOLON definition-list term + * COLON definition-list description or indent + * TABLE_OPEN table + * THEMATIC_BREAK thematic break + * PREFORMATTED_MARKER preformatted block + * anything else paragraph + * ``` + * + * One important limit to keep in mind is that this file is still only doing + * block structure. If a heading contains `[[Mars]]` or `'''bold'''`, this + * stage does not parse those inline details yet. It emits text ranges inside + * the heading and leaves inline meaning for the later inline parser. + * + * Like the rest of the pipeline, this parser never throws. If the input is + * messy, it emits recovery events when needed and still closes open blocks so + * the event stream stays usable. + * + * @example Parsing a heading followed by a paragraph + * ```ts + * import { blockEvents } from './block_parser.ts'; + * import { tokenize } from './tokenizer.ts'; + * + * const source = '== Title ==\nSome text.'; + * const events = [...blockEvents(source, tokenize(source))]; + * ``` + * + * @example Parsing nested bullet lines + * ```ts + * import { blockEvents } from './block_parser.ts'; + * import { tokenize } from './tokenizer.ts'; + * + * const source = '* A\n** B'; + * const events = [...blockEvents(source, tokenize(source))]; + * ``` + * + * @module + */ + +import type { TextSource } from './text_source.ts'; +import type { Token } from './token.ts'; +import type { Position, Point, WikitextEvent } from './events.ts'; +import { TokenType } from './token.ts'; +import { + DiagnosticCode, + errorEvent, +} from './events.ts'; +import { + enterEventFromPoints, + exitEventFromPoints, + textEventFromPoints, +} from './event_factory.ts'; + +/** + * Optional switches for block-event generation. + * + * The block parser always keeps recovering structurally so the event stream + * stays usable. The only question here is whether this caller also wants the + * block-owned recovery diagnostics preserved in that stream. + */ +export interface BlockEventOptions { + /** Whether block-stage diagnostics such as unclosed-table warnings are emitted. */ + readonly diagnostics?: boolean; +} + +// --------------------------------------------------------------------------- +// Position helpers +// --------------------------------------------------------------------------- + +/** Build a source point from the current line, column, and offset. */ +function point(line: number, column: number, offset: number): Point { + return { line, column, offset }; +} + +/** Build a source range from two points. */ +function pos(start: Point, end: Point): Position { + return { start, end }; +} + +/** Build an empty range at one point. */ +function zeroPos(pt: Point): Position { + return { start: pt, end: pt }; +} + +// --------------------------------------------------------------------------- +// Line tracker +// --------------------------------------------------------------------------- +// +// Tokens only store start and end offsets. Event positions also need line and +// column, so this tracker keeps that extra running state as tokens are consumed. + +interface LineTracker { + /** 1-based logical line number for the current token cursor. */ + line: number; + /** Offset of the start of the current line. */ + line_offset: number; +} + +/** Turn an offset into a full source point using the current line-tracking state. */ +function pointAt(tracker: LineTracker, offset: number): Point { + return point(tracker.line, 1 + offset - tracker.line_offset, offset); +} + +/** Move line tracking forward after consuming a newline token. */ +function advanceLine(tracker: LineTracker, newlineEnd: number): void { + tracker.line++; + tracker.line_offset = newlineEnd; +} + +// --------------------------------------------------------------------------- +// Token buffer +// --------------------------------------------------------------------------- +// +// The block parser sometimes needs to inspect the current token before it +// decides which block parser to enter. This wrapper keeps the current token and +// line-tracking state together so the parser can peek and consume cleanly. + +interface TokenBuffer { + /** Underlying token iterator from the tokenizer stage. */ + iter: Iterator; + /** Current token under the parser cursor, or `null` at the end. */ + current: Token | null; + /** Tracks line/column from newline tokens. */ + tracker: LineTracker; + /** Whether this parse lane wants block-stage diagnostics. */ + emit_diagnostics: boolean; +} + +/** + * Create a token buffer whose cursor starts at the first token. + * + * This keeps three pieces of state together because block parsing needs all of + * them at once: the current token, the line tracker used for event positions, + * and whether this caller asked for block-stage diagnostics. + */ +function createBuffer(tokens: Iterable, emit_diagnostics: boolean): TokenBuffer { + const iter = tokens[Symbol.iterator](); + const first = iter.next(); + return { + iter, + current: first.done ? null : first.value, + tracker: { line: 1, line_offset: 0 }, + emit_diagnostics, + }; +} + +/** + * Consume the current token and move to the next one. + * + * Newlines are where line/column state advances, so this helper is the single + * place that keeps token consumption and source-position tracking in sync. + */ +function advance(buf: TokenBuffer): void { + if (buf.current && buf.current.type === TokenType.NEWLINE) { + advanceLine(buf.tracker, buf.current.end); + } + const next = buf.iter.next(); + buf.current = next.done ? null : next.value; +} + +/** Read the current token without consuming it. */ +function peek(buf: TokenBuffer): Token | null { + return buf.current; +} + +/** Return the current token and advance the buffer. */ +function consume(buf: TokenBuffer): Token | null { + const tok = buf.current; + if (tok) advance(buf); + return tok; +} + +// --------------------------------------------------------------------------- +// List stack management +// --------------------------------------------------------------------------- +// +// Wikitext lists are line-oriented: each line's marker prefix determines +// the nesting level and list type. For example: +// +// * → depth 1, bullet +// ** → depth 2, bullet +// *# → depth 1 bullet, depth 2 ordered +// +// The block parser maintains a stack of open list levels. When a new line's +// prefix diverges from the stack, we close excess levels and open new ones. + +/** Information about one open list depth. */ +interface ListLevel { + /** 'bullet', 'ordered', 'definition-term', or 'definition-description'. */ + kind: string; + /** Node type for the wrapping list: 'list' or 'definition-list'. */ + list_type: string; + /** Whether the list is ordered (only for 'list'). */ + ordered: boolean; +} + +const DEFAULT_LIST_LEVEL: ListLevel = { + kind: 'bullet', + list_type: 'list', + ordered: false, +}; + +/** + * Fixed marker-to-list metadata map used while parsing list prefixes. + * + * This is a small hot mapping, not general control flow. A null-prototype + * object keeps the vocabulary explicit, avoids inherited keys from the normal + * object prototype chain, and lets `Object.hasOwn(...)` answer "is this one of + * our markers?" without consulting names such as `toString` or `constructor`. + */ +const LIST_LEVEL_LOOKUP: Partial> = Object.assign( + Object.create(null), + { + '*': DEFAULT_LIST_LEVEL, + '#': { kind: 'ordered', list_type: 'list', ordered: true }, + ';': { kind: 'definition-term', list_type: 'definition-list', ordered: false }, + ':': { kind: 'definition-description', list_type: 'definition-list', ordered: false }, + }, +); + +const LIST_MARKER_CHAR_LOOKUP: Partial> = Object.assign( + Object.create(null), + { + [TokenType.BULLET]: '*', + [TokenType.HASH]: '#', + [TokenType.SEMICOLON]: ';', + [TokenType.COLON]: ':', + }, +); + +/** + * One merged text range that still maps exactly back to the original source. + * + * The block parser sees many small tokenizer tokens such as whitespace, text, + * punctuation, and delimiter leftovers. The later inline parser does not care + * about those original token boundaries. It cares about a simpler question: + * which exact source bytes belong to this block-level text run? + * + * `TextSpan` is the answer to that question. Each span records one contiguous + * `[start, end)` slice of source that the block parser has decided belongs to + * the current heading, paragraph line, list item line, table cell, or + * preformatted line. + * + * A concrete example is easier than the earlier placeholder A/B/gap diagram. + * Suppose the source content we want to keep is the line `alpha beta`, and the + * tokenizer handed this parser three adjacent content tokens: + * + * [0,5) = 'alpha' + * [5,6) = ' ' + * [6,10) = 'beta' + * + * Those three tokens become one span because each token starts exactly where + * the previous token ended: + * + * pending span: [0,10) + * + * Now compare that with a case where a structural boundary appears in the + * middle. If a paragraph continues on the next physical line, the newline is a + * real block-parser boundary, so we do not merge across it: + * + * source: alpha beta\nsecond line + * 012345678901234567890 + * ^ newline at offset 10 + * + * spans: [0,10) and [11,22) + * + * Read that as: merge adjacent content bytes, but stop as soon as a newline, + * cell separator, or real gap means the bytes no longer belong to one local + * text run. + * + * The important invariant is source fidelity. A span may merge neighboring + * tokens, but it must never invent bytes, skip bytes that belong to content, + * or cross a structural boundary such as a newline or an inline cell + * separator. + */ +interface TextSpan { + /** Inclusive start offset of a merged text range. */ + start: number; + /** Exclusive end offset of a merged text range. */ + end: number; +} + +/** + * Record one merged text range without allocating per-call closure state. + * + * The earlier perf pass used local `flushSpan()` helpers inside several hot + * block parsers. Those helpers were small, but they still created one closure + * per parser invocation. This shared helper keeps the same merge behavior while + * avoiding that repeated setup work. + * + * Correctness rule: this helper only merges contiguous token ranges. It does + * not trim trailing whitespace. Outside headings, trailing spaces are part of + * the original source range and must remain visible to downstream consumers. + * + * The sentinel values `spanStart = -1` and `spanEnd = -1` mean "there is no + * pending span right now." Callers build a pending span as they walk tokens, + * then call `pushTextSpan()` only when they hit one of three events: + * + * 1. a real gap in offsets + * 2. a structural boundary such as newline or cell separator + * 3. the end of the current block-local collection loop + */ +function pushTextSpan( + spans: TextSpan[], + start: number, + end: number, +): void { + if (start === -1) return; + spans.push({ start, end }); +} + +function markerToLevel(marker: string): ListLevel { + return Object.hasOwn(LIST_LEVEL_LOOKUP, marker) + ? LIST_LEVEL_LOOKUP[marker]! + : DEFAULT_LIST_LEVEL; +} + +/** + * Return whether two marker levels can share the same open list wrapper. + * + * Raw marker characters are slightly too specific for this check. `;` and `:` + * produce different child node types, but they still belong to the same + * `definition-list` wrapper at a given depth. + */ +function canReuseListLevel(openLevel: ListLevel, nextLevel: ListLevel): boolean { + if (openLevel.list_type !== nextLevel.list_type) { + return false; + } + + if (openLevel.list_type === 'definition-list') { + return true; + } + + return openLevel.ordered === nextLevel.ordered; +} + +// --------------------------------------------------------------------------- +// Block parser generator +// --------------------------------------------------------------------------- + +/** + * Consume a token stream and yield block-level events. + * + * Reads tokens produced by {@linkcode tokenize} and emits enter/exit pairs + * for headings, paragraphs, lists, definition lists, tables, thematic + * breaks, and preformatted blocks. Inline content is emitted as raw text + * events for a downstream inline parser to process. + * + * The generator never throws. Malformed or unexpected token sequences + * produce recovery error events and the parser continues. + * + * Current diagnostic scope is intentionally narrow. This stage only reports + * block-owned recovery facts, such as reaching EOF before a table closed. It + * does not try to attach tree anchors itself because those depend on later + * tree materialization. + * + * @param source - The text source backing the tokens (for offset resolution). + * @param tokens - Token iterable, typically from `tokenize(source)`. + */ +export function* blockEvents( + source: TextSource, + tokens: Iterable, + options: BlockEventOptions = {}, +): Generator { + const buf = createBuffer(tokens, options.diagnostics === true); + + // Wrap the root document in enter/exit. + const startPt = pointAt(buf.tracker, 0); + yield enterEventFromPoints('root', {}, startPt, startPt); + + while (peek(buf) !== null) { + const tok = peek(buf)!; + + // TODO: snapshot recording point for incremental reparsing. + // A BlockSnapshot captured here (before dispatch) would record the + // token buffer position, line tracker state, and open block stack, + // letting the incremental parser restart from any block boundary. + + // Skip newlines between blocks (blank lines). + if (tok.type === TokenType.NEWLINE) { + advance(buf); + continue; + } + + // Skip EOF. + if (tok.type === TokenType.EOF) { + advance(buf); + continue; + } + + // Dispatch on the first token of the line. + switch (tok.type) { + case TokenType.HEADING_MARKER: + yield* parseHeading(buf, source); + break; + + case TokenType.BULLET: + case TokenType.HASH: + case TokenType.SEMICOLON: + case TokenType.COLON: + yield* parseList(buf, source); + break; + + case TokenType.TABLE_OPEN: + yield* parseTable(buf, source); + break; + + case TokenType.THEMATIC_BREAK: + yield* parseThematicBreak(buf); + break; + + case TokenType.PREFORMATTED_MARKER: + yield* parsePreformatted(buf, source); + break; + + default: + yield* parseParagraph(buf, source); + break; + } + } + + const endPt = pointAt(buf.tracker, source.length); + yield exitEventFromPoints('root', endPt, endPt); +} + +// --------------------------------------------------------------------------- +// Heading parser +// --------------------------------------------------------------------------- +// +// Wikitext headings: `== Title ==` +// The opening `=` count sets the level (1-6). A closing `=` run on the +// same line is optional. Content between them is inline text. +// +// Strategy: collect all tokens on the line, then trim a trailing close +// marker (HEADING_MARKER_CLOSE or EQUALS) and surrounding whitespace +// from the end. This avoids premature close detection for mid-content +// equals signs like `== a=b ==`. + +function* parseHeading( + buf: TokenBuffer, + _source: TextSource, +): Generator { + const marker = consume(buf)!; + const level = Math.min(6, Math.max(1, marker.end - marker.start)) as + 1 | 2 | 3 | 4 | 5 | 6; + + const startPt = pointAt(buf.tracker, marker.start); + + // Collect all tokens on this line (until NEWLINE or EOF). + const lineTokens: Token[] = []; + + while (peek(buf) !== null) { + const t = peek(buf)!; + if (t.type === TokenType.NEWLINE || t.type === TokenType.EOF) break; + lineTokens.push(t); + advance(buf); + } + + let contentStartIndex = 0; + let contentEndIndex = lineTokens.length; + + // Trim trailing whitespace. + while ( + contentEndIndex > contentStartIndex && + lineTokens[contentEndIndex - 1].type === TokenType.WHITESPACE + ) { + contentEndIndex--; + } + + // Trim trailing close marker (HEADING_MARKER_CLOSE or EQUALS). + // This scan is intentionally end-biased so inner text like `a=b` survives as + // heading content instead of being mistaken for the closing marker. + let endOffset = marker.end; + if ( + contentEndIndex > contentStartIndex && + (lineTokens[contentEndIndex - 1].type === TokenType.HEADING_MARKER_CLOSE || + lineTokens[contentEndIndex - 1].type === TokenType.EQUALS) + ) { + const closeTok = lineTokens[contentEndIndex - 1]; + contentEndIndex--; + endOffset = closeTok.end; + } + + // Trim whitespace between content and the (now-removed) close marker. + while ( + contentEndIndex > contentStartIndex && + lineTokens[contentEndIndex - 1].type === TokenType.WHITESPACE + ) { + contentEndIndex--; + } + + // Trim leading whitespace after the heading marker. + while ( + contentStartIndex < contentEndIndex && + lineTokens[contentStartIndex].type === TokenType.WHITESPACE + ) { + contentStartIndex++; + } + + // Use endOffset from the last remaining token if we have content. + if (contentStartIndex < contentEndIndex) { + endOffset = Math.max(endOffset, lineTokens[contentEndIndex - 1].end); + } + + const endPt = pointAt(buf.tracker, endOffset); + yield enterEventFromPoints('heading', { level }, startPt, endPt); + + // The inline parser only cares about source ranges, not original tokenizer + // token boundaries. Merging contiguous spans here avoids re-merging the same + // text immediately in the next stage. + if (contentStartIndex < contentEndIndex) { + yield* emitTextSpans(buf.tracker, [{ + start: lineTokens[contentStartIndex].start, + end: lineTokens[contentEndIndex - 1].end, + }]); + } + + yield exitEventFromPoints('heading', startPt, endPt); +} + +// --------------------------------------------------------------------------- +// Paragraph parser +// --------------------------------------------------------------------------- +// +// A paragraph is a sequence of lines that don't start with a block +// delimiter. The paragraph ends at a blank line (two consecutive +// newlines), a block-starting token, or EOF. +// +// The comment that matters here is: a paragraph can span many physical lines, +// but this block parser still splits its text spans at each newline. +// +// Concrete example: +// +// source: +// Alpha beta +// Gamma delta +// +// paragraph node: +// one paragraph containing both lines +// +// emitted text spans: +// [Alpha beta] then [Gamma delta] +// +// Diagram: +// +// paragraph +// | +// +-- line 1 content span +// +-- newline boundary +// +-- line 2 content span +// +// Why not merge the whole paragraph into one giant span? The important rule is +// more precise than "paragraphs are line-based". +// +// Today, one `text` event means one contiguous source slice. A continued +// paragraph line break sits between those slices as a real newline byte, and +// this block stage treats that newline as paragraph structure rather than as +// emitted text. So the current handoff is: +// +// enter(paragraph) +// text("Alpha beta") +// text("Gamma delta") +// exit(paragraph) +// +// not: +// +// enter(paragraph) +// text("Alpha beta\nGamma delta") +// exit(paragraph) +// +// A future discontiguous block-to-inline handoff could keep both line slices in +// one logical group, but that would be a new internal contract. It would no +// longer be the same thing as one ordinary contiguous text span. + +/** + * Lookup table for token types that start a new block and therefore terminate + * a running paragraph. + * + * This is a fixed string vocabulary, so a null-prototype object is a tighter + * fit than a `Set` for the hot membership check inside paragraph parsing. + * `Object.create(null)` removes the usual prototype chain, and + * `Object.hasOwn(...)` keeps the check on the table's own keys instead of + * inherited names such as `toString`. + */ +const BLOCK_START_TOKEN_LOOKUP: Partial> = Object.assign( + Object.create(null), + { + [TokenType.HEADING_MARKER]: true, + [TokenType.BULLET]: true, + [TokenType.HASH]: true, + [TokenType.SEMICOLON]: true, + [TokenType.COLON]: true, + [TokenType.TABLE_OPEN]: true, + [TokenType.TABLE_CLOSE]: true, + [TokenType.THEMATIC_BREAK]: true, + [TokenType.PREFORMATTED_MARKER]: true, + }, +); + +function* parseParagraph( + buf: TokenBuffer, + _source: TextSource, +): Generator { + const firstTok = peek(buf)!; + const startPt = pointAt(buf.tracker, firstTok.start); + + const contentSpans: TextSpan[] = []; + let spanStart = -1; + let spanEnd = -1; + let _endOffset = firstTok.start; + let sawNewline = false; + + // Walkthrough for the three span variables used below: + // + // spanStart = where the current pending span begins + // spanEnd = where the current pending span currently ends + // contentSpans = finished spans we already decided to keep + // + // Sentinel state: + // + // spanStart = -1 + // spanEnd = -1 + // + // means "we are not currently building a span." + // + // Example with concrete offsets: + // + // source line: alpha beta + // 0123456789 + // + // tokens seen: [0,5) 'alpha' + // [5,6) ' ' + // [6,10) 'beta' + // + // state change: + // start with no pending span + // read [0,5) -> pending becomes [0,5) + // read [5,6) -> still adjacent, extend to [0,6) + // read [6,10) -> still adjacent, extend to [0,10) + // end of line -> flush [0,10) into contentSpans + + while (peek(buf) !== null) { + const t = peek(buf)!; + + if (t.type === TokenType.EOF) break; + + // A newline followed by a block-start token or another newline + // (blank line) ends the paragraph. + if (t.type === TokenType.NEWLINE) { + if (sawNewline) { + // Double newline (blank line) — end paragraph. + break; + } + sawNewline = true; + _endOffset = t.end; + advance(buf); + + // Check what follows the newline. + const next = peek(buf); + if (next === null) break; + if (next.type === TokenType.EOF) break; + if (next.type === TokenType.NEWLINE) break; + if (Object.hasOwn(BLOCK_START_TOKEN_LOOKUP, next.type)) break; + + // Each physical line becomes its own merged text span. + // + // Example: + // source: Alpha beta\nGamma + // before newline: pending span is [Alpha beta] + // newline found: flush that span, then restart on the next line + // + // So the paragraph keeps going, but the current line-local span does not. + // The reason is not that the next line is outside the paragraph. The + // reason is that one current span must stay contiguous in source, while + // the continuation newline remains structural instead of becoming text. + pushTextSpan(contentSpans, spanStart, spanEnd); + spanStart = -1; + spanEnd = -1; + + // The newline is part of the paragraph content (continuation line). + // We don't emit newline tokens as text — they're structural separators + // within the paragraph's inline content. + continue; + } + + sawNewline = false; + if (spanStart === -1) { + spanStart = t.start; + spanEnd = t.end; + } else if (t.start === spanEnd) { + spanEnd = t.end; + } else { + pushTextSpan(contentSpans, spanStart, spanEnd); + spanStart = t.start; + spanEnd = t.end; + } + + _endOffset = t.end; + advance(buf); + } + + // The loop keeps one pending span in local variables for the common fast + // path. Flush it once at the end so the caller sees the final line segment + // even when the paragraph ended because of EOF or a block-start token. + pushTextSpan(contentSpans, spanStart, spanEnd); + + // Don't emit empty paragraphs. + if (contentSpans.length === 0) return; + + const endPt = pointAt(buf.tracker, contentSpans[contentSpans.length - 1].end); + yield enterEventFromPoints('paragraph', {}, startPt, endPt); + + // Inline markup is deliberately left unresolved here. Paragraph parsing owns + // block boundaries; the later inline stage owns links, templates, emphasis, + // and other nested inline syntax. + // + // Performance rule: emit one text event per contiguous span instead of one + // per tokenizer token. The inline parser only needs accurate source ranges, + // so coarser text events avoid redundant merge work in the next stage. + // + // That still leaves one event per physical paragraph line today, because the + // newline between continuation lines is structural rather than emitted text. + // Crossing that boundary with one logical group would require a different + // internal handoff shape than plain contiguous `text(start_offset, end_offset)`. + yield* emitTextSpans(buf.tracker, contentSpans); + + yield exitEventFromPoints('paragraph', startPt, endPt); +} + +// --------------------------------------------------------------------------- +// List parser +// --------------------------------------------------------------------------- +// +// Wikitext lists are line-oriented. Each list line starts with one or more +// marker characters (*#;:). The number and type of markers determines the +// nesting structure. +// +// Once the marker prefix has been consumed, list item text uses the same span +// model as paragraph text, just on a smaller scope: one physical list line. +// The markers and the optional space after them are structure, not content. +// +// Example: +// * A → list(ordered=false) > list-item(marker='*') +// ** B → list(ordered=false) > list-item(marker='*') > list(ordered=false) > list-item(marker='**') +// *# C → list(ordered=false) > list-item(marker='*') > list(ordered=true) > list-item(marker='*#') +// +// The parser processes all consecutive list lines as one group, managing +// a stack of open list/list-item nodes. + +function* parseList( + buf: TokenBuffer, + _source: TextSource, +): Generator { + // The open stack tracks which lists and items are currently open. + // Each entry is { level: ListLevel, had_item: boolean }. + const openStack: { level: ListLevel; marker_char: string }[] = []; + + // Think of each list line as a prefix rewrite against the previous line: + // + // previous: * * + // current : * # + // │ └─ depth 2 changed kind, so close to depth 1 then reopen + // └── depth 1 stayed compatible and remains open + + // Process consecutive list lines. + while (peek(buf) !== null) { + const t = peek(buf)!; + + // Only list markers start a list line. + if ( + t.type !== TokenType.BULLET && + t.type !== TokenType.HASH && + t.type !== TokenType.SEMICOLON && + t.type !== TokenType.COLON + ) { + break; + } + + // Collect marker characters for this line. + const markers: string[] = []; + let markersEndOffset = t.start; + + while (peek(buf) !== null) { + const m = peek(buf)!; + if ( + m.type !== TokenType.BULLET && + m.type !== TokenType.HASH && + m.type !== TokenType.SEMICOLON && + m.type !== TokenType.COLON + ) { + break; + } + const char = tokenToMarkerChar(m.type); + markers.push(char); + markersEndOffset = m.end; + advance(buf); + } + + const depth = markers.length; + + // Close levels deeper than the current line's depth. + yield* closeLevels(buf, openStack, depth); + + // If any shared depth changes list wrapper meaning, close back to the + // first incompatible depth and reopen from there. This keeps `;` and `:` + // inside one definition-list wrapper while still splitting `*` and `#` + // into different list wrappers. + const sharedDepth = Math.min(openStack.length, depth); + for (let i = 0; i < sharedDepth; i++) { + const nextLevel = markerToLevel(markers[i]); + if (!canReuseListLevel(openStack[i].level, nextLevel)) { + yield* closeLevels(buf, openStack, i); + break; + } + } + + // Open new levels or adjust existing levels. + for (let i = openStack.length; i < depth; i++) { + const markerChar = markers[i]; + const lvl = markerToLevel(markerChar); + const lvlPt = pointAt(buf.tracker, markersEndOffset); + // Open the wrapping list node. + if (lvl.list_type === 'list') { + yield enterEventFromPoints('list', { ordered: lvl.ordered }, lvlPt, lvlPt); + } else { + yield enterEventFromPoints('definition-list', {}, lvlPt, lvlPt); + } + + openStack.push({ level: lvl, marker_char: markerChar }); + } + + // Determine the item node type. + const lastMarker = markers[markers.length - 1]; + const lastLevel = markerToLevel(lastMarker); + const fullMarker = markers.join(''); + + const itemPt = pointAt(buf.tracker, markersEndOffset); + + // Open the list item. + if (lastLevel.kind === 'definition-term') { + yield enterEventFromPoints('definition-term', {}, itemPt, itemPt); + } else if (lastLevel.kind === 'definition-description') { + yield enterEventFromPoints('definition-description', {}, itemPt, itemPt); + } else { + yield enterEventFromPoints('list-item', { marker: fullMarker }, itemPt, itemPt); + } + + // Skip whitespace after markers. + while (peek(buf) !== null && peek(buf)!.type === TokenType.WHITESPACE) { + advance(buf); + } + + // Collect inline content until newline or EOF. + const contentSpans: TextSpan[] = []; + let spanStart = -1; + let spanEnd = -1; + let lineEndOffset = markersEndOffset; + + // This loop is the paragraph span algorithm applied to one list line. + // The only practical difference is the stop condition: list item content + // ends at the next newline instead of continuing across later lines. + + while (peek(buf) !== null) { + const ct = peek(buf)!; + if (ct.type === TokenType.NEWLINE || ct.type === TokenType.EOF) break; + if (spanStart === -1) { + spanStart = ct.start; + spanEnd = ct.end; + } else if (ct.start === spanEnd) { + spanEnd = ct.end; + } else { + pushTextSpan(contentSpans, spanStart, spanEnd); + spanStart = ct.start; + spanEnd = ct.end; + } + + lineEndOffset = ct.end; + advance(buf); + } + + // List item content follows the same rule as paragraphs: merge contiguous + // source ranges, but preserve exact source bytes inside those ranges. + // + // Concrete example: + // + // source: * item text here + // ^ structural marker + // ^^^^^^^^^^^^^^ content span that gets emitted + // + // The marker and the space after it help define list structure, so they do + // not become part of the emitted text span. + pushTextSpan(contentSpans, spanStart, spanEnd); + + yield* emitTextSpans(buf.tracker, contentSpans); + + const itemEndPt = pointAt(buf.tracker, lineEndOffset); + + // Close the list item. + if (lastLevel.kind === 'definition-term') { + yield exitEventFromPoints('definition-term', itemEndPt, itemEndPt); + } else if (lastLevel.kind === 'definition-description') { + yield exitEventFromPoints('definition-description', itemEndPt, itemEndPt); + } else { + yield exitEventFromPoints('list-item', itemEndPt, itemEndPt); + } + + // Consume the newline if present. + if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) { + advance(buf); + } + } + + // Close all remaining open levels. + yield* closeLevels(buf, openStack, 0); +} + +/** Close list levels from the stack down to `targetDepth`. */ +function* closeLevels( + buf: TokenBuffer, + stack: { level: ListLevel; marker_char: string }[], + targetDepth: number, +): Generator { + while (stack.length > targetDepth) { + const entry = stack.pop()!; + // When no more tokens remain, use the tracker's current position + // (line_offset tracks the last known newline boundary). + const closePt = peek(buf) + ? pointAt(buf.tracker, peek(buf)!.start) + : point(buf.tracker.line, 1, buf.tracker.line_offset); + // Close the wrapping list. + if (entry.level.list_type === 'list') { + yield exitEventFromPoints('list', closePt, closePt); + } else { + yield exitEventFromPoints('definition-list', closePt, closePt); + } + } +} + +/** Convert a list marker token type back into the source marker character. */ +function tokenToMarkerChar(type: TokenType): string { + return Object.hasOwn(LIST_MARKER_CHAR_LOOKUP, type) + ? LIST_MARKER_CHAR_LOOKUP[type]! + : '*'; +} + +// --------------------------------------------------------------------------- +// Table parser +// --------------------------------------------------------------------------- +// +// Wikitext tables: +// {| attributes → table open +// |+ caption → table caption +// |- attributes → row separator +// | cell → data cell +// || cell → inline data cell separator +// ! cell → header cell +// !! cell → inline header cell separator +// |} → table close +// +// Rows are implicit: the first cell after `{|` or `|+` starts an +// implicit row. `|-` explicitly starts a new row. + +function* parseTable( + buf: TokenBuffer, + source: TextSource, +): Generator { + const openTok = consume(buf)!; // TABLE_OPEN + const startPt = pointAt(buf.tracker, openTok.start); + + // Collect attributes after {| on the same line. + const attrTokens: Token[] = []; + while (peek(buf) !== null) { + const t = peek(buf)!; + if (t.type === TokenType.NEWLINE || t.type === TokenType.EOF) break; + attrTokens.push(t); + advance(buf); + } + const attributes = attrTokens.length > 0 + ? joinTokenText(source, attrTokens).trim() + : undefined; + + yield enterEventFromPoints('table', attributes !== undefined ? { attributes } : {}, startPt, startPt); + + // Consume trailing newline. + if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) { + advance(buf); + } + + let rowOpen = false; + let cellOpen = false; + + // Tables are mostly driven one physical line at a time: + // + // {| open table + // |+ caption + // |- explicit row boundary + // |/! cell line, with implicit row creation if needed + // |} close table + // + // `rowOpen` and `cellOpen` let recovery close the right structure when the + // source omits an expected row separator or table terminator. + + // Process table body line by line until TABLE_CLOSE or EOF. + while (peek(buf) !== null) { + const t = peek(buf)!; + + if (t.type === TokenType.EOF) break; + + // Table close: |} + if (t.type === TokenType.TABLE_CLOSE) { + if (cellOpen) { + yield* closeCell(buf); + cellOpen = false; + } + if (rowOpen) { + yield* closeRow(buf); + rowOpen = false; + } + const closePt = pointAt(buf.tracker, t.end); + advance(buf); + yield exitEventFromPoints('table', closePt, closePt); + // Consume trailing newline. + if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) { + advance(buf); + } + return; + } + + // Skip blank lines inside table. + if (t.type === TokenType.NEWLINE) { + advance(buf); + continue; + } + + // Table row separator: |- + if (t.type === TokenType.TABLE_ROW) { + if (cellOpen) { + yield* closeCell(buf); + cellOpen = false; + } + if (rowOpen) { + yield* closeRow(buf); + rowOpen = false; + } + advance(buf); + const rowPt = pointAt(buf.tracker, t.start); + + // Row attributes on the same line. + const rowAttrTokens: Token[] = []; + while (peek(buf) !== null) { + const rt = peek(buf)!; + if (rt.type === TokenType.NEWLINE || rt.type === TokenType.EOF) break; + rowAttrTokens.push(rt); + advance(buf); + } + const rowAttrs = rowAttrTokens.length > 0 + ? joinTokenText(source, rowAttrTokens).trim() + : undefined; + + yield enterEventFromPoints( + 'table-row', + rowAttrs !== undefined ? { attributes: rowAttrs } : {}, + rowPt, + rowPt, + ); + rowOpen = true; + + // Consume newline. + if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) { + advance(buf); + } + continue; + } + + // Table caption: |+ + if (t.type === TokenType.TABLE_CAPTION) { + if (cellOpen) { + yield* closeCell(buf); + cellOpen = false; + } + advance(buf); + const capPt = pointAt(buf.tracker, t.start); + + yield enterEventFromPoints('table-caption', {}, capPt, capPt); + + // Caption content until newline or EOF. + yield* emitLineContent(buf); + + const capEndPt = peek(buf) + ? pointAt(buf.tracker, peek(buf)!.start) + : pointAt(buf.tracker, source.length); + yield exitEventFromPoints('table-caption', capEndPt, capEndPt); + + // Consume newline. + if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) { + advance(buf); + } + continue; + } + + // Header cell: ! at line start + if (t.type === TokenType.TABLE_HEADER_CELL) { + if (cellOpen) { + yield* closeCell(buf); + cellOpen = false; + } + if (!rowOpen) { + // The first cell line after `{|` implicitly starts a row even without + // an explicit `|-` line. + const rowPt = pointAt(buf.tracker, t.start); + yield enterEventFromPoints('table-row', {}, rowPt, rowPt); + rowOpen = true; + } + advance(buf); + yield* parseTableCells(buf, source, true); + cellOpen = false; // parseTableCells handles open/close + // Consume newline. + if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) { + advance(buf); + } + continue; + } + + // Data cell: | at line start + if (t.type === TokenType.PIPE) { + if (cellOpen) { + yield* closeCell(buf); + cellOpen = false; + } + if (!rowOpen) { + const rowPt = pointAt(buf.tracker, t.start); + yield enterEventFromPoints('table-row', {}, rowPt, rowPt); + rowOpen = true; + } + advance(buf); + yield* parseTableCells(buf, source, false); + cellOpen = false; + // Consume newline. + if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) { + advance(buf); + } + continue; + } + + // Anything else inside the table is recovery territory. Advancing keeps the + // parser moving so malformed table content does not trap the loop. + advance(buf); + } + + // End of input: close any open structures. + if (cellOpen) { + yield* closeCell(buf); + } + if (rowOpen) { + yield* closeRow(buf); + } + const endPt = pointAt(buf.tracker, source.length); + if (buf.emit_diagnostics) { + yield unclosedTableDiagnostic(endPt); + } + yield exitEventFromPoints('table', endPt, endPt); +} + +/** + * Report that the block parser reached end of input before a table closed. + * + * This happens when the source opens a table with `{|` but never reaches the + * matching `|}` before EOF. + * + * Example input: + * + * ```text + * {| class="wikitable" + * | Planet + * | Mars + * ``` + * + * The parser still closes the table in recovery mode so downstream consumers + * get a usable tree. Consumers can respond in a few different ways depending + * on their product goals: + * + * - show a warning and keep rendering the recovered table + * - offer a quick fix that inserts a closing `|}` + * - ignore the warning in best-effort batch processing that only needs a + * stable structure + * + * None of those responses is mandatory. The parser's contract is only that it + * records the recovery and still produces valid output. + */ +function unclosedTableDiagnostic(end: Point): WikitextEvent { + return errorEvent('Unclosed table at end of input', zeroPos(end), { + severity: 'warning', + code: DiagnosticCode.UNCLOSED_TABLE, + recoverable: true, + source: 'block', + }); +} + +/** Parse cells on one line, handling `||` and `!!` inline separators. */ +function* parseTableCells( + buf: TokenBuffer, + source: TextSource, + header: boolean, +): Generator { + // Parse the first cell and any inline-separated cells on this line. + const separator = header ? TokenType.DOUBLE_BANG : TokenType.DOUBLE_PIPE; + + while (true) { + const cellPt = peek(buf) + ? pointAt(buf.tracker, peek(buf)!.start) + : pointAt(buf.tracker, source.length); + + yield enterEventFromPoints('table-cell', { header }, cellPt, cellPt); + + // Skip leading whitespace. + while (peek(buf) !== null && peek(buf)!.type === TokenType.WHITESPACE) { + advance(buf); + } + + // Collect cell content until separator, newline, or EOF. + const contentSpans: TextSpan[] = []; + let spanStart = -1; + let spanEnd = -1; + let hitSeparator = false; + + // Table cell spans use the same pending-span state as paragraphs and list + // items, but with one extra boundary: `||` or `!!` must split cells even + // when the bytes on both sides are adjacent in the original source. + // + // Concrete example: + // + // source: | A || B + // ^^^^ first cell content + // ^^ separator + // ^^ second cell content + // + // The separator is structure, not content, so we flush the first cell span + // before consuming `||` and then start a fresh span for `B`. + + while (peek(buf) !== null) { + const ct = peek(buf)!; + if (ct.type === TokenType.NEWLINE || ct.type === TokenType.EOF) break; + if (ct.type === separator) { + // Inline separators like `||` and `!!` are real structure boundaries. + // Flush the current merged span before consuming the separator so the + // cell content range stays faithful to the original source. + pushTextSpan(contentSpans, spanStart, spanEnd); + spanStart = -1; + spanEnd = -1; + hitSeparator = true; + advance(buf); + break; + } + // Also handle `!!` as separator in header context when seeing DOUBLE_BANG + // even from a data cell start (mixed usage). + if (header && ct.type === TokenType.DOUBLE_BANG) { + pushTextSpan(contentSpans, spanStart, spanEnd); + spanStart = -1; + spanEnd = -1; + hitSeparator = true; + advance(buf); + break; + } + if (spanStart === -1) { + spanStart = ct.start; + spanEnd = ct.end; + } else if (ct.start === spanEnd) { + spanEnd = ct.end; + } else { + pushTextSpan(contentSpans, spanStart, spanEnd); + spanStart = ct.start; + spanEnd = ct.end; + } + + advance(buf); + } + + pushTextSpan(contentSpans, spanStart, spanEnd); + + yield* emitTextSpans(buf.tracker, contentSpans); + + const cellEndPt = contentSpans.length > 0 + ? pointAt(buf.tracker, contentSpans[contentSpans.length - 1].end) + : cellPt; + yield exitEventFromPoints('table-cell', cellEndPt, cellEndPt); + + // `||` and `!!` mean there is another cell on the same physical line. + if (!hitSeparator) break; + } +} + +/** Close a table cell at the current cursor position. */ +function* closeCell(buf: TokenBuffer): Generator { + const pt = peek(buf) + ? pointAt(buf.tracker, peek(buf)!.start) + : point(buf.tracker.line, 1, buf.tracker.line_offset); + yield exitEventFromPoints('table-cell', pt, pt); +} + +/** Close a table row at the current cursor position. */ +function* closeRow(buf: TokenBuffer): Generator { + const pt = peek(buf) + ? pointAt(buf.tracker, peek(buf)!.start) + : point(buf.tracker.line, 1, buf.tracker.line_offset); + yield exitEventFromPoints('table-row', pt, pt); +} + +/** Emit text events for tokens until newline or EOF. */ +function* emitLineContent( + buf: TokenBuffer, +): Generator { + const lineSpans: TextSpan[] = []; + let spanStart = -1; + let spanEnd = -1; + + while (peek(buf) !== null) { + const t = peek(buf)!; + if (t.type === TokenType.NEWLINE || t.type === TokenType.EOF) break; + // Used for simple single-line payloads such as captions where the block + // container is already known and only raw text needs to be forwarded. + if (spanStart === -1) { + spanStart = t.start; + spanEnd = t.end; + } else if (t.start === spanEnd) { + spanEnd = t.end; + } else { + pushTextSpan(lineSpans, spanStart, spanEnd); + spanStart = t.start; + spanEnd = t.end; + } + + advance(buf); + } + + // Captions and similar single-line payloads do not need token granularity. + // One merged range is enough unless a real gap appears in the underlying + // tokens. + // + // Example: + // |+ caption text + // ^^^^^^^^^^^^ one merged line-local span + // + // This helper exists so caption handling can reuse the same span model as + // other block text paths without duplicating the state machine again. + pushTextSpan(lineSpans, spanStart, spanEnd); + yield* emitTextSpans(buf.tracker, lineSpans); +} + +/** + * Emit already-merged text spans as text events. + * + * The important invariant is simple: these spans must still cover the exact + * bytes the block parser decided belong to the block. This helper is only an + * event materialization step. It must not normalize spacing, trim content, or + * reinterpret structure. + * + * Every span passed here is expected to be line-local. That is why one current + * `LineTracker` state is enough to reconstruct both points for the event. + * Callers split on newlines earlier, then `emitTextSpans()` converts each + * finished `[start, end)` range into a proper `text` event. + * + * Example: + * + * spans from caller: [12,20) and [24,31) + * emitted events: text(12,20) and text(24,31) + * + * This helper does not decide where spans begin or end. It only turns already + * approved spans into event objects with correct positions. + */ +function* emitTextSpans( + tracker: LineTracker, + spans: readonly TextSpan[], +): Generator { + for (const span of spans) { + const eventStart = pointAt(tracker, span.start); + const eventEnd = pointAt(tracker, span.end); + yield textEventFromPoints(span.start, span.end, eventStart, eventEnd); + } +} + +/** Concatenate text of tokens by slicing from the source. */ +function joinTokenText(source: TextSource, tokens: Token[]): string { + if (tokens.length === 0) return ''; + const start = tokens[0].start; + const end = tokens[tokens.length - 1].end; + return source.slice(start, end); +} + +// --------------------------------------------------------------------------- +// Thematic break parser +// --------------------------------------------------------------------------- + +function* parseThematicBreak( + buf: TokenBuffer, +): Generator { + const tok = consume(buf)!; + const startPt = pointAt(buf.tracker, tok.start); + const endPt = pointAt(buf.tracker, tok.end); + yield enterEventFromPoints('thematic-break', {}, startPt, endPt); + yield exitEventFromPoints('thematic-break', startPt, endPt); +} + +// --------------------------------------------------------------------------- +// Preformatted block parser +// --------------------------------------------------------------------------- +// +// Lines starting with a space are preformatted (rendered as
).
+// Consecutive preformatted lines form one preformatted block.
+//
+// This is the strictest source-fidelity path in the block parser. After the
+// leading structural marker space, the rest of each line is treated as literal
+// content. That means the span collector must preserve trailing spaces instead
+// of trimming or normalizing them.
+
+function* parsePreformatted(
+  buf: TokenBuffer,
+  source: TextSource,
+): Generator {
+  const firstTok = peek(buf)!;
+  const startPt = pointAt(buf.tracker, firstTok.start);
+
+  yield enterEventFromPoints('preformatted', {}, startPt, startPt);
+
+  // Process consecutive preformatted lines.
+  while (peek(buf) !== null && peek(buf)!.type === TokenType.PREFORMATTED_MARKER) {
+    // Skip the preformatted marker (leading space).
+    advance(buf);
+
+    // Emit content of this line.
+    const lineSpans: TextSpan[] = [];
+    let spanStart = -1;
+    let spanEnd = -1;
+
+    // Read this as: skip the one structural marker byte, then preserve every
+    // remaining byte on the line exactly as authored.
+    //
+    // Example:
+    //   source:  " pre  text  "
+    //             ^ structural marker, not emitted
+    //              ^^^^^^^^^^^ literal content, including trailing spaces
+
+    while (peek(buf) !== null) {
+      const t = peek(buf)!;
+      if (t.type === TokenType.NEWLINE || t.type === TokenType.EOF) break;
+      // The leading space is structural and already consumed, so the emitted
+      // text starts with the first token after that marker.
+      if (spanStart === -1) {
+        spanStart = t.start;
+        spanEnd = t.end;
+      } else if (t.start === spanEnd) {
+        spanEnd = t.end;
+      } else {
+        pushTextSpan(lineSpans, spanStart, spanEnd);
+        spanStart = t.start;
+        spanEnd = t.end;
+      }
+
+      advance(buf);
+    }
+
+    // Preformatted content is the strictest source-fidelity case in this file.
+    // After the leading marker space, every remaining byte on the line counts
+    // as user content, including trailing spaces.
+    pushTextSpan(lineSpans, spanStart, spanEnd);
+    yield* emitTextSpans(buf.tracker, lineSpans);
+
+    // Consume newline.
+    if (peek(buf) !== null && peek(buf)!.type === TokenType.NEWLINE) {
+      advance(buf);
+    }
+  }
+
+  const endPt = peek(buf)
+    ? pointAt(buf.tracker, peek(buf)!.start)
+    : pointAt(buf.tracker, source.length);
+  yield exitEventFromPoints('preformatted', endPt, endPt);
+}
diff --git a/experiments/event-shape-study/lazy-position/code/event_factory.ts b/experiments/event-shape-study/lazy-position/code/event_factory.ts
new file mode 100644
index 0000000..3c67539
--- /dev/null
+++ b/experiments/event-shape-study/lazy-position/code/event_factory.ts
@@ -0,0 +1,92 @@
+import type { TokenType } from './token.ts';
+import type {
+	EnterEvent,
+	ExitEvent,
+	Point,
+	Position,
+	TextEvent,
+	TokenEvent,
+} from './events.ts';
+
+type LazyPositionEvent = {
+	position: Position;
+};
+
+function defineLazyPosition(
+	event: T,
+	start: Point,
+	end: Point,
+): T & LazyPositionEvent {
+	let position: Position | undefined;
+
+	return Object.defineProperty(event, 'position', {
+		enumerable: true,
+		configurable: false,
+		get(): Position {
+			position ??= { start, end };
+			return position;
+		},
+	}) as T & LazyPositionEvent;
+}
+
+/**
+ * Create an eager enter event directly from source points.
+ *
+ * This keeps the public event surface unchanged while moving the nested
+ * position-object allocation into one constructor path with a stable property
+ * layout.
+ */
+export function enterEventFromPoints(
+	node_type: string,
+	props: Readonly>,
+	start: Point,
+	end: Point,
+): EnterEvent {
+	return defineLazyPosition({
+		kind: 'enter',
+		node_type,
+		props,
+	}, start, end);
+}
+
+/** Create an eager exit event directly from source points. */
+export function exitEventFromPoints(
+	node_type: string,
+	start: Point,
+	end: Point,
+): ExitEvent {
+	return defineLazyPosition({
+		kind: 'exit',
+		node_type,
+	}, start, end);
+}
+
+/** Create an eager text event directly from source points. */
+export function textEventFromPoints(
+	start_offset: number,
+	end_offset: number,
+	start: Point,
+	end: Point,
+): TextEvent {
+	return defineLazyPosition({
+		kind: 'text',
+		start_offset,
+		end_offset,
+	}, start, end);
+}
+
+/** Create an eager token event directly from source points. */
+export function tokenEventFromPoints(
+	token_type: TokenType,
+	start_offset: number,
+	end_offset: number,
+	start: Point,
+	end: Point,
+): TokenEvent {
+	return defineLazyPosition({
+		kind: 'token',
+		token_type,
+		start_offset,
+		end_offset,
+	}, start, end);
+}
diff --git a/experiments/event-shape-study/lazy-position/code/event_shape_bench.ts b/experiments/event-shape-study/lazy-position/code/event_shape_bench.ts
new file mode 100644
index 0000000..0f0b626
--- /dev/null
+++ b/experiments/event-shape-study/lazy-position/code/event_shape_bench.ts
@@ -0,0 +1,145 @@
+/**
+ * Benchmarks that isolate event representation access patterns.
+ *
+ * The goal is to make event-shape experiments comparable without changing the
+ * parser pipeline or mixing ad-hoc memory checks into hot loops.
+ *
+ * @module bench
+ */
+// deno-lint-ignore-file no-import-prefix no-unversioned-import
+
+import { bench, do_not_optimize, run, summary } from 'npm:mitata';
+
+import {
+	cycleInputs,
+	drainEnterProps,
+	drainEventsNoPosition,
+	drainEventsOffsetsOnly,
+	drainEventsWithPosition,
+	drainRetainedEventCount,
+	drainSessionEventsCold,
+	drainSessionEventsWarm,
+	drainSessionParseCold,
+	drainSessionParseWarm,
+	drainSessionParseWithDiagnosticsCold,
+	drainSessionParseWithDiagnosticsWarm,
+	drainSessionRetainedEventCountWarm,
+	drainStatelessParseWithDiagnostics,
+	SAME_SIZE_MIXED_TEXT,
+	SAME_SIZE_PATHOLOGICAL_TEXT,
+	SAME_SIZE_PLAIN_TEXT,
+	SYNTHETIC_ARTICLE_INPUTS,
+} from './_test_utils/perf_fixtures.ts';
+import { parse } from './mod.ts';
+
+const nextSyntheticArticle = cycleInputs(SYNTHETIC_ARTICLE_INPUTS);
+
+summary(() => {
+	bench('events() no position access: same-size plain (~8 KB)', () => {
+		do_not_optimize(drainEventsNoPosition(SAME_SIZE_PLAIN_TEXT));
+	}).gc('inner');
+
+	bench('events() no position access: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainEventsNoPosition(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('events() no position access: same-size pathological (~8 KB)', () => {
+		do_not_optimize(drainEventsNoPosition(SAME_SIZE_PATHOLOGICAL_TEXT));
+	}).gc('inner');
+});
+
+summary(() => {
+	bench('events() offsets only: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainEventsOffsetsOnly(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('events() offsets only: same-size pathological (~8 KB)', () => {
+		do_not_optimize(drainEventsOffsetsOnly(SAME_SIZE_PATHOLOGICAL_TEXT));
+	}).gc('inner');
+
+	bench('events() all position reads: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainEventsWithPosition(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('events() all position reads: same-size pathological (~8 KB)', () => {
+		do_not_optimize(drainEventsWithPosition(SAME_SIZE_PATHOLOGICAL_TEXT));
+	}).gc('inner');
+
+	bench('events() enter props reads: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainEnterProps(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('events() enter props reads: same-size pathological (~8 KB)', () => {
+		do_not_optimize(drainEnterProps(SAME_SIZE_PATHOLOGICAL_TEXT));
+	}).gc('inner');
+});
+
+summary(() => {
+	bench('events() retained array only: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainRetainedEventCount(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('session.events() warm retained array: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainSessionRetainedEventCountWarm(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('events() retained array only: synthetic article (~35-45 KB)', () => {
+		do_not_optimize(drainRetainedEventCount(nextSyntheticArticle()));
+	}).gc('inner');
+
+	bench('session.events() warm retained array: synthetic article (~35-45 KB)', () => {
+		do_not_optimize(drainSessionRetainedEventCountWarm(nextSyntheticArticle()));
+	}).gc('inner');
+});
+
+summary(() => {
+	bench('parse(): same-size mixed (~8 KB)', () => {
+		do_not_optimize(parse(SAME_SIZE_MIXED_TEXT).children.length);
+	}).gc('inner');
+
+	bench('parseWithDiagnostics(): same-size pathological (~8 KB)', () => {
+		do_not_optimize(drainStatelessParseWithDiagnostics(SAME_SIZE_PATHOLOGICAL_TEXT));
+	}).gc('inner');
+
+	bench('session.events() cold: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainSessionEventsCold(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('session.events() warm: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainSessionEventsWarm(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('session.parse() cold: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainSessionParseCold(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('session.parse() warm: same-size mixed (~8 KB)', () => {
+		do_not_optimize(drainSessionParseWarm(SAME_SIZE_MIXED_TEXT));
+	}).gc('inner');
+
+	bench('session.parseWithDiagnostics() cold: same-size pathological (~8 KB)', () => {
+		do_not_optimize(drainSessionParseWithDiagnosticsCold(SAME_SIZE_PATHOLOGICAL_TEXT));
+	}).gc('inner');
+
+	bench('session.parseWithDiagnostics() warm: same-size pathological (~8 KB)', () => {
+		do_not_optimize(drainSessionParseWithDiagnosticsWarm(SAME_SIZE_PATHOLOGICAL_TEXT));
+	}).gc('inner');
+	
+	bench('parse(): synthetic article (~35-45 KB)', () => {
+		do_not_optimize(parse(nextSyntheticArticle()).children.length);
+	}).gc('inner');
+
+	bench('session.events() warm: synthetic article (~35-45 KB)', () => {
+		do_not_optimize(drainSessionEventsWarm(nextSyntheticArticle()));
+	}).gc('inner');
+
+	bench('session.parse() warm: synthetic article (~35-45 KB)', () => {
+		do_not_optimize(drainSessionParseWarm(nextSyntheticArticle()));
+	}).gc('inner');
+
+	bench('session.parseWithDiagnostics() warm: synthetic article (~35-45 KB)', () => {
+		do_not_optimize(drainSessionParseWithDiagnosticsWarm(nextSyntheticArticle()));
+	}).gc('inner');
+});
+
+await run();
\ No newline at end of file
diff --git a/experiments/event-shape-study/lazy-position/code/event_shape_memory.ts b/experiments/event-shape-study/lazy-position/code/event_shape_memory.ts
new file mode 100644
index 0000000..0111cfd
--- /dev/null
+++ b/experiments/event-shape-study/lazy-position/code/event_shape_memory.ts
@@ -0,0 +1,262 @@
+/**
+ * Retained-memory measurements for event-shape experiments.
+ *
+ * This script stays outside mitata so heap snapshots are not mixed into hot
+ * benchmark callbacks. It keeps the measurement cases aligned with
+ * event_shape_bench.ts so timing and retained-memory comparisons tell the same
+ * story.
+ */
+
+import {
+	drainSessionRetainedEventCountWarm,
+	SAME_SIZE_MIXED_TEXT,
+	sumEnterPropKeyCounts,
+	sumEventPositionOffsets,
+	SYNTHETIC_ARTICLE_INPUTS,
+} from './_test_utils/perf_fixtures.ts';
+import type { WikitextEvent } from './events.ts';
+import { createSession, events, parse, parseWithDiagnostics } from './mod.ts';
+
+type MeasurementCase = {
+	name: string;
+	run: (input: string) => unknown;
+	consume: (value: unknown) => number;
+};
+
+type MeasurementSummary = {
+	name: string;
+	samples: number[];
+	median: number;
+	minimum: number;
+	maximum: number;
+};
+
+type InputSummary = {
+	name: string;
+	cases: MeasurementSummary[];
+};
+
+type OutputFormat = 'text' | 'json';
+
+function asEventList(value: unknown): WikitextEvent[] {
+	return value as WikitextEvent[];
+}
+
+const DEFAULT_REPEATS = 5;
+const INPUTS = [
+	{
+		name: 'same-size mixed (~8 KB)',
+		value: SAME_SIZE_MIXED_TEXT,
+	},
+	{
+		name: 'synthetic article (~35-45 KB)',
+		value: SYNTHETIC_ARTICLE_INPUTS[0],
+	},
+] as const;
+
+const MEASUREMENT_CASES: readonly MeasurementCase[] = [
+	{
+		name: 'events retained, no position reads',
+		run(input) {
+			return Array.from(events(input));
+		},
+		consume(value) {
+			return (value as unknown[]).length;
+		},
+	},
+	{
+		name: 'events retained, then read every position',
+		run(input) {
+			const retained = Array.from(events(input));
+			return retained;
+		},
+		consume(value) {
+			const retained = asEventList(value);
+			return retained.length + sumEventPositionOffsets(retained);
+		},
+	},
+	{
+		name: 'events retained, then read enter props',
+		run(input) {
+			const retained = Array.from(events(input));
+			return retained;
+		},
+		consume(value) {
+			const retained = asEventList(value);
+			return retained.length + sumEnterPropKeyCounts(retained);
+		},
+	},
+	{
+		name: 'session warm event cache retained',
+		run(input) {
+			const session = createSession(input);
+			Array.from(session.events());
+			return session;
+		},
+		consume(value) {
+			return Array.from((value as ReturnType).events()).length;
+		},
+	},
+	{
+		name: 'parse() result retained',
+		run(input) {
+			return parse(input);
+		},
+		consume(value) {
+			return (value as ReturnType).children.length;
+		},
+	},
+	{
+		name: 'parseWithDiagnostics() result retained',
+		run(input) {
+			return parseWithDiagnostics(input);
+		},
+		consume(value) {
+			const result = value as ReturnType;
+			return result.tree.children.length + result.diagnostics.length;
+		},
+	},
+] as const;
+
+function forceGc(): void {
+	for (let index = 0; index < 3; index++) {
+		globalThis.gc?.();
+	}
+}
+
+function heapUsed(): number {
+	forceGc();
+	return Deno.memoryUsage().heapUsed;
+}
+
+function measureRetainedHeap(case_def: MeasurementCase, input: string): number {
+	forceGc();
+	const before = heapUsed();
+
+	const retained = case_def.run(input);
+	const checksum = case_def.consume(retained);
+
+	if (checksum < 0) {
+		throw new Error('unexpected negative checksum');
+	}
+
+	const after = heapUsed();
+	return after - before;
+}
+
+function parseRepeats(): number {
+	const repeats_arg = Deno.args.find((arg) => arg.startsWith('--repeats='));
+
+	if (repeats_arg === undefined) {
+		return DEFAULT_REPEATS;
+	}
+
+	const value = Number(repeats_arg.slice('--repeats='.length));
+	if (!Number.isInteger(value) || value <= 0) {
+		throw new Error(`expected a positive integer for --repeats, got: ${repeats_arg}`);
+	}
+
+	return value;
+}
+
+function parseFormat(): OutputFormat {
+	const format_arg = Deno.args.find((arg) => arg.startsWith('--format='));
+
+	if (format_arg === undefined) {
+		return 'text';
+	}
+
+	const value = format_arg.slice('--format='.length);
+	if (value === 'json' || value === 'text') {
+		return value;
+	}
+
+	throw new Error(`expected --format=text or --format=json, got: ${format_arg}`);
+}
+
+function median(values: readonly number[]): number {
+	const sorted = [...values].sort((left, right) => left - right);
+	const middle = Math.floor(sorted.length / 2);
+
+	if (sorted.length % 2 === 0) {
+		return Math.round((sorted[middle - 1]! + sorted[middle]!) / 2);
+	}
+
+	return sorted[middle]!;
+}
+
+function summarize(case_def: MeasurementCase, input: string, repeats: number): MeasurementSummary {
+	const samples: number[] = [];
+
+	for (let index = 0; index < repeats; index++) {
+		samples.push(measureRetainedHeap(case_def, input));
+	}
+
+	return {
+		name: case_def.name,
+		samples,
+		median: median(samples),
+		minimum: Math.min(...samples),
+		maximum: Math.max(...samples),
+	};
+}
+
+function formatBytes(bytes: number): string {
+	if (bytes >= 1024 * 1024) {
+		return `${(bytes / (1024 * 1024)).toFixed(2)} MiB`;
+	}
+
+	if (bytes >= 1024) {
+		return `${(bytes / 1024).toFixed(2)} KiB`;
+	}
+
+	return `${bytes} B`;
+}
+
+const repeats = parseRepeats();
+const format = parseFormat();
+const summaries: InputSummary[] = [];
+
+for (const input of INPUTS) {
+	const input_summary: InputSummary = {
+		name: input.name,
+		cases: [],
+	};
+
+	for (const case_def of MEASUREMENT_CASES) {
+		const summary = summarize(case_def, input.value, repeats);
+		input_summary.cases.push(summary);
+	}
+
+	summaries.push(input_summary);
+}
+
+if (format === 'json') {
+	console.log(JSON.stringify({
+		generated_at: new Date().toISOString(),
+		repeats,
+		unit: 'bytes',
+		inputs: summaries.map((input) => ({
+			name: input.name,
+			cases: input.cases.map((summary) => ({
+				name: summary.name,
+				samples_bytes: summary.samples,
+				median_bytes: summary.median,
+				minimum_bytes: summary.minimum,
+				maximum_bytes: summary.maximum,
+			})),
+		})),
+	}, null, 2));
+	Deno.exit(0);
+}
+
+for (const input of summaries) {
+	console.log(`\n# ${input.name}`);
+	console.log(`repeats=${repeats}`);
+
+	for (const summary of input.cases) {
+		console.log(
+			`${summary.name}: median=${formatBytes(summary.median)}, min=${formatBytes(summary.minimum)}, max=${formatBytes(summary.maximum)}, samples=[${summary.samples.join(', ')}]`,
+		);
+	}
+}
\ No newline at end of file
diff --git a/experiments/event-shape-study/lazy-position/code/events.ts b/experiments/event-shape-study/lazy-position/code/events.ts
new file mode 100644
index 0000000..0b0a454
--- /dev/null
+++ b/experiments/event-shape-study/lazy-position/code/events.ts
@@ -0,0 +1,842 @@
+/**
+ * Event types for the parser's main output stream.
+ *
+ * This parser is designed around events first, not tree nodes first. That can
+ * sound abstract, so here is the practical version.
+ *
+ * Sometimes a caller wants the full tree. Sometimes it only wants the first
+ * heading, a table of contents, or a stream it can transform on the fly.
+ * Building a full AST for every case forces extra work even when the caller
+ * does not need it. Events are the cheaper middle layer that all of those
+ * outputs can share. The tree builder is just one consumer of that stream.
+ *
+ * Think of the stream as a running narration of what the parser is finding:
+ *
+ * - "a heading starts here"
+ * - "this text belongs inside it"
+ * - "the heading ends here"
+ * - "a paragraph starts now"
+ *
+ * For `== Hello ==\nText`, that looks like this:
+ *
+ * ```ts
+ * enter('heading', { level: 2 })
+ *   text(3, 8)   // "Hello"
+ * exit('heading')
+ * enter('paragraph')
+ *   text(12, 16) // "Text"
+ * exit('paragraph')
+ * ```
+ *
+ * The important rule is that open and close events stay properly nested. In
+ * plain English, if something starts inside a heading, it also has to finish
+ * before the heading finishes. That is what parser docs often call stack
+ * discipline.
+ *
+ * ```ts
+ * enter('heading')
+ *   enter('wikilink')
+ *   exit('wikilink')
+ * exit('heading')
+ * ```
+ *
+ * You should never see this broken order:
+ *
+ * ```ts
+ * enter('heading')
+ *   enter('wikilink')
+ * exit('heading')   // wrong
+ * exit('wikilink')  // wrong
+ * ```
+ *
+ * Text and token events store source ranges, not copied strings. That means an
+ * event says "the text is from offset 3 to offset 8" instead of carrying a new
+ * string like `"Hello"`. A caller can recover the real text later with
+ * `slice(source, start, end)` when it actually needs it.
+ *
+ * The five event kinds are:
+ *
+ * | Kind    | What it means |
+ * |---------|----------------|
+ * | `enter` | a node starts here |
+ * | `exit`  | the matching node ends here |
+ * | `text`  | plain text from the source |
+ * | `token` | a raw tokenizer token surfaced in the stream |
+ * | `error` | recovery information when the parser had to keep going through bad input |
+ *
+ * @example Processing a stream of events
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ *
+ * function showEvents(events: Iterable) {
+ *   for (const evt of events) {
+ *     switch (evt.kind) {
+ *       case 'enter': console.log(`open ${evt.node_type}`); break;
+ *       case 'exit': console.log(`close ${evt.node_type}`); break;
+ *       case 'text': console.log(`text [${evt.start_offset}..${evt.end_offset})`); break;
+ *       case 'token': console.log(`token ${evt.token_type}`); break;
+ *       case 'error': console.log(`error ${evt.message}`); break;
+ *     }
+ *   }
+ * }
+ * ```
+ *
+ * @example Tracking the current nesting depth
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ *
+ * function maxDepth(events: Iterable): number {
+ *   let depth = 0;
+ *   let max = 0;
+ *
+ *   for (const evt of events) {
+ *     if (evt.kind === 'enter') {
+ *       depth++;
+ *       max = Math.max(max, depth);
+ *     } else if (evt.kind === 'exit') {
+ *       depth--;
+ *     }
+ *   }
+ *
+ *   return max;
+ * }
+ * ```
+ *
+ * @module
+ */
+
+import type { TokenType } from './token.ts';
+
+/**
+ * Diagnostic severity level for parser recovery events.
+ */
+export type DiagnosticSeverity = 'error' | 'warning';
+
+/**
+ * Stable machine-readable diagnostic codes emitted by the parser itself.
+ *
+ * This object behaves like an enum without using a TypeScript `enum` runtime.
+ * Callers can compare against these values directly:
+ *
+ * ```ts
+ * import { DiagnosticCode } from './events.ts';
+ *
+ * if (event.kind === 'error' && event.code === DiagnosticCode.UNCLOSED_TABLE) {
+ *   // offer a quick fix, surface a warning, or log telemetry
+ * }
+ * ```
+ *
+ * The keys group the recoveries the parser currently knows how to describe in
+ * a stable way. The human-readable `message` still explains the specific case,
+ * but the code is the field consumers should match on.
+ *
+ * This object is frozen on purpose. It is the parser's own stable vocabulary,
+ * not a plugin registration table. Callers that need custom codes can still
+ * emit any string through `ErrorEvent.code` or `ParseDiagnostic.code` without
+ * mutating the shared parser-owned constant map.
+ */
+/** Public map shape for the parser's stable diagnostic-code vocabulary. */
+export type DiagnosticCodeMap = Readonly<{
+  UNCLOSED_TABLE: 'UNCLOSED_TABLE';
+  INLINE_TAG_UNTERMINATED_OPENER: 'INLINE_TAG_UNTERMINATED_OPENER';
+  INLINE_TAG_MISSING_CLOSE: 'INLINE_TAG_MISSING_CLOSE';
+  TREE_MISMATCHED_EXIT: 'TREE_MISMATCHED_EXIT';
+  TREE_ORPHAN_EXIT: 'TREE_ORPHAN_EXIT';
+  TREE_EOF_AUTOCLOSE: 'TREE_EOF_AUTOCLOSE';
+}>;
+
+const DIAGNOSTIC_CODE_VALUES: DiagnosticCodeMap = {
+  /**
+   * The block parser reached end of input before a table closed with `|}`.
+   *
+   * Typical input:
+   *
+   * ```text
+   * {| class="wikitable"
+   * | Cell
+   * ```
+   *
+   * Reasonable responses include surfacing a warning in an editor, offering a
+   * quick fix that inserts `|}`, or ignoring it in a best-effort preview that
+   * only needs a usable recovered tree.
+   */
+  UNCLOSED_TABLE: 'UNCLOSED_TABLE',
+
+  /**
+   * The inline parser reached end of input before an HTML-like opener reached
+   * its closing `>`.
+   *
+   * Typical input:
+   *
+   * ```text
+   * body
+   * ```
+   *
+   * The parser keeps the opener as structurally real, recovers by extending
+   * the node to the end of the current text range, and emits this warning.
+   */
+  INLINE_TAG_MISSING_CLOSE: 'INLINE_TAG_MISSING_CLOSE',
+
+  /**
+   * The tree builder saw an exit for one node while a different node was still
+   * open, so it auto-closed the inner node first to restore nesting.
+   */
+  TREE_MISMATCHED_EXIT: 'TREE_MISMATCHED_EXIT',
+
+  /**
+   * The tree builder saw an exit event that did not match any open node and
+   * had to drop it at the root boundary.
+   */
+  TREE_ORPHAN_EXIT: 'TREE_ORPHAN_EXIT',
+
+  /**
+   * The event stream ended while one or more nodes were still open, so the
+   * tree builder auto-closed them at their last known end point.
+   */
+  TREE_EOF_AUTOCLOSE: 'TREE_EOF_AUTOCLOSE',
+} as const;
+
+/**
+ * Stable machine-readable diagnostic codes emitted by the parser today.
+ *
+ * Match on these values when a consumer needs parser-owned diagnostics that
+ * stay stable across messages and formatting changes.
+ */
+export const DiagnosticCode: DiagnosticCodeMap = Object.freeze(DIAGNOSTIC_CODE_VALUES);
+
+/**
+ * Known machine-readable diagnostic codes emitted by the current parser.
+ *
+ * Callers may still encounter custom string codes from tests, adapters, or
+ * future extensions, so event and parse-result shapes keep `code` open to any
+ * string. This alias is the stable subset owned by the parser today.
+ */
+export type KnownDiagnosticCode = typeof DiagnosticCode[keyof typeof DiagnosticCode];
+
+// ---------------------------------------------------------------------------
+// Position types (unist-compatible)
+// ---------------------------------------------------------------------------
+//
+// These types track where each event came from in the original source text.
+// They follow the unist (Universal Syntax Tree) spec used by the unified
+// ecosystem (remark, rehype, etc.), so wikist trees are compatible with
+// unist utilities like `unist-util-position`.
+//
+// All measurements use UTF-16 code units, the native string indexing of
+// JavaScript. This matches the Language Server Protocol (LSP), which also
+// uses UTF-16 positions. No conversion needed when integrating with editors
+// like VS Code.
+
+/**
+ * A single place in the source text.
+ *
+ * This stores three views of the same location: line number, column number,
+ * and absolute offset from the start of the input. All measurements use the
+ * same UTF-16 indexing JavaScript strings already use, so positions line up
+ * with `charCodeAt()`, `slice()`, and most editor integrations.
+ */
+export interface Point {
+  /** 1-indexed line number. */
+  readonly line: number;
+  /** 1-indexed column, in UTF-16 code units from start of line. */
+  readonly column: number;
+  /** 0-indexed offset in UTF-16 code units from start of input. */
+  readonly offset: number;
+}
+
+/**
+ * A source range with a start point and an end point.
+ */
+export interface Position {
+  /** Inclusive start point. */
+  readonly start: Point;
+  /** Exclusive end point (first character *after* the range). */
+  readonly end: Point;
+}
+
+// ---------------------------------------------------------------------------
+// Event variants
+// ---------------------------------------------------------------------------
+//
+// The five event kinds form a discriminated union on the `kind` field.
+// Consumers switch on `evt.kind` for exhaustive handling:
+//
+//   enter  -> a node is opening (carries type + properties)
+//   exit   -> the most recently opened node of that type is closing
+//   text   -> a range of literal text (offsets into the source)
+//   token  -> a raw tokenizer token surfaced in the event stream
+//   error  -> a recovery point (the parser never throws)
+//
+// Enter/exit pairs always nest properly. If you see:
+//   enter('bold') -> enter('italic') -> exit('italic') -> exit('bold')
+// the nesting is correct. You will never see exit('bold') before
+// exit('italic') -- that would break stack discipline.
+//
+// Why not carry text strings directly? Because events are produced during
+// parsing when millions of characters are being scanned. Allocating a new
+// string for every text span would create GC pressure. Instead, text
+// events carry start_offset/end_offset pairs. The consumer calls
+// `slice(source, start, end)` only when it actually needs the string
+// content (e.g., to render HTML or build a node value).
+//
+// Error events deserve special attention: the parser never throws. If it
+// encounters malformed wikitext like an unclosed `{{template`, it recovers
+// by treating the `{{` as literal text and optionally emits an ErrorEvent.
+// This means consumers always get a complete event stream for any input.
+
+/**
+ * Signals that a node of the given type is being opened.
+ *
+ * Every `EnterEvent` will have a matching {@linkcode ExitEvent} with the same
+ * `node_type`, forming a well-nested stack. `props` carries node-specific
+ * fields (e.g., `{ level: 2 }` for a heading, `{ ordered: true }` for a
+ * list).
+ */
+export interface EnterEvent {
+  /** Discriminant for the event union. */
+  readonly kind: 'enter';
+  /** The AST node type being opened (e.g., `'heading'`, `'template'`). */
+  readonly node_type: string;
+  /**
+   * Node-specific properties attached at open time.
+   *
+   * Keyed by field name, values are the property values for the node
+   * being opened (e.g., `{ level: 3 }` for a heading, `{ ordered: true }`
+   * for a list). Empty object when the node has no extra fields.
+   */
+  readonly props: Readonly>;
+  /** Source range covered by this event. */
+  readonly position: Position;
+}
+
+/**
+ * Signals that the most recently opened node of the given type is being
+ * closed. Matches the corresponding `EnterEvent`.
+ */
+export interface ExitEvent {
+  /** Discriminant for the event union. */
+  readonly kind: 'exit';
+  /** The AST node type being closed. */
+  readonly node_type: string;
+  /** Source range of the closing delimiter / boundary. */
+  readonly position: Position;
+}
+
+/**
+ * A range of literal text content, expressed as offsets into the
+ * {@linkcode TextSource}.
+ *
+ * The event does not carry the text string itself: only the start and end
+ * offsets. Consumers call `slice(source, evt.start_offset, evt.end_offset)`
+ * to resolve the string value on demand. This range-first design avoids
+ * per-event string allocation and prevents memory retention hazards from
+ * keeping substrings alive.
+ *
+ * For example, given the source `"== Hello =="`, a text event for the word
+ * "Hello" would carry `start_offset: 3` and `end_offset: 8`, without ever
+ * allocating the string `"Hello"` until a consumer asks for it.
+ */
+export interface TextEvent {
+  /** Discriminant for the event union. */
+  readonly kind: 'text';
+  /** Inclusive start offset (UTF-16 code units into the TextSource). */
+  readonly start_offset: number;
+  /** Exclusive end offset (UTF-16 code units into the TextSource). */
+  readonly end_offset: number;
+  /** Source position of this text range. */
+  readonly position: Position;
+}
+
+/**
+ * A raw token event, exposing the lowest-level tokenizer output in the
+ * event stream. Primarily used by consumers that need token-level
+ * granularity without running the tokenizer separately.
+ */
+export interface TokenEvent {
+  /** Discriminant for the event union. */
+  readonly kind: 'token';
+  /** The token type from the tokenizer. */
+  readonly token_type: TokenType;
+  /** Inclusive start offset (UTF-16 code units into the TextSource). */
+  readonly start_offset: number;
+  /** Exclusive end offset (UTF-16 code units into the TextSource). */
+  readonly end_offset: number;
+  /** Source position of this token. */
+  readonly position: Position;
+}
+
+/**
+ * Optional error event emitted at recovery points. The parser never throws:
+ * it always produces valid output for any input. When it encounters malformed
+ * wikitext (unclosed templates, mismatched tags, etc.), it recovers and
+ * optionally emits an `ErrorEvent` so consumers can log, surface, or ignore
+ * the issue.
+ *
+ * The optional metadata fields (`severity`, `code`, `recoverable`, `source`,
+ * `details`) support richer diagnostics. They are all optional so that the
+ * simplest error case is just a message and a position.
+ */
+export interface ErrorEvent {
+  /** Discriminant for the event union. */
+  readonly kind: 'error';
+  /** Human-readable description of what was recovered from. */
+  readonly message: string;
+  /**
+   * Diagnostic severity. Defaults to `'error'`.
+   *
+   * Use `'warning'` for softer recoveries that may still be semantically
+   * acceptable for many consumers.
+   */
+  readonly severity?: DiagnosticSeverity;
+  /**
+   * Stable machine-readable code for programmatic filtering and telemetry.
+    *
+    * When the parser owns the recovery, prefer matching against
+    * {@linkcode DiagnosticCode}. That keeps consumers stable even if the
+    * human-readable `message` changes.
+   */
+  readonly code?: KnownDiagnosticCode | string;
+  /**
+   * Indicates whether parsing continued with a deterministic recovery path.
+   */
+  readonly recoverable?: boolean;
+  /**
+   * Parser stage that emitted this diagnostic.
+   */
+  readonly source?: 'tokenizer' | 'block' | 'inline' | 'tree';
+  /**
+   * Optional structured details for advanced consumers.
+   */
+  readonly details?: Readonly>;
+  /** Source position where the error was detected. */
+  readonly position: Position;
+}
+
+/**
+ * Optional metadata for {@linkcode ErrorEvent} construction.
+ */
+export interface ErrorEventOptions {
+  /** Severity level for this diagnostic. */
+  readonly severity?: DiagnosticSeverity;
+  /**
+   * Stable machine-readable code.
+   *
+   * Use a {@linkcode DiagnosticCode} member for parser-owned recoveries.
+   */
+  readonly code?: KnownDiagnosticCode | string;
+  /** Whether the parser recovered and continued. */
+  readonly recoverable?: boolean;
+  /** Parser stage that emitted the diagnostic. */
+  readonly source?: 'tokenizer' | 'block' | 'inline' | 'tree';
+  /** Optional structured metadata payload. */
+  readonly details?: Readonly>;
+}
+
+// ---------------------------------------------------------------------------
+// Union type
+// ---------------------------------------------------------------------------
+
+/**
+ * Discriminated union of all event types in the wikitext event stream.
+ *
+ * Switch on `evt.kind` for exhaustive handling:
+ *
+ * ```ts
+ * switch (evt.kind) {
+ *   case 'enter': // open a node
+ *   case 'exit':  // close a node
+ *   case 'text':  // text content (offsets)
+ *   case 'token': // raw token
+ *   case 'error': // recovery point
+ * }
+ * ```
+ *
+ * TypeScript will narrow the type inside each branch, giving access to the
+ * fields specific to that event kind (e.g., `evt.node_type` is only available
+ * inside the `'enter'` and `'exit'` branches).
+ */
+export type WikitextEvent =
+  | EnterEvent
+  | ExitEvent
+  | TextEvent
+  | TokenEvent
+  | ErrorEvent;
+
+// ---------------------------------------------------------------------------
+// Event constructors
+// ---------------------------------------------------------------------------
+//
+// Factory functions for creating event objects. Each returns a fresh
+// immutable object. These are the primary way to build events — prefer
+// these over hand-constructing event objects, because they enforce the
+// correct `kind` discriminant and field names.
+
+/**
+ * Create an {@linkcode EnterEvent}.
+ *
+ * @example Opening a heading node
+ * ```ts
+ * import { enterEvent } from './events.ts';
+ *
+ * const evt = enterEvent('heading', { level: 2 }, {
+ *   start: { line: 1, column: 1, offset: 0 },
+ *   end: { line: 1, column: 14, offset: 13 },
+ * });
+ * ```
+ *
+ * @example Opening a simple paragraph
+ * ```ts
+ * import { enterEvent } from './events.ts';
+ *
+ * const evt = enterEvent('paragraph', {}, {
+ *   start: { line: 3, column: 1, offset: 20 },
+ *   end: { line: 3, column: 1, offset: 20 },
+ * });
+ * ```
+ *
+ * @param node_type - The AST node type being opened.
+ * @param props - Node-specific fields.
+ * @param position - Source range.
+ */
+export function enterEvent(
+  node_type: string,
+  props: Readonly>,
+  position: Position,
+): EnterEvent {
+  return { kind: 'enter', node_type, props, position };
+}
+
+/**
+ * Create an `ExitEvent`.
+ *
+ * @example Closing a heading node
+ * ```ts
+ * import { exitEvent } from './events.ts';
+ *
+ * const evt = exitEvent('heading', {
+ *   start: { line: 1, column: 1, offset: 0 },
+ *   end: { line: 1, column: 14, offset: 13 },
+ * });
+ * ```
+ *
+ * @example Closing a paragraph
+ * ```ts
+ * import { exitEvent } from './events.ts';
+ *
+ * const evt = exitEvent('paragraph', {
+ *   start: { line: 5, column: 1, offset: 50 },
+ *   end: { line: 5, column: 1, offset: 50 },
+ * });
+ * ```
+ *
+ * @param node_type - The AST node type being closed.
+ * @param position - Source range of the closing boundary.
+ */
+export function exitEvent(
+  node_type: string,
+  position: Position,
+): ExitEvent {
+  return { kind: 'exit', node_type, position };
+}
+
+/**
+ * Create a `TextEvent` with range-first offsets.
+ *
+ * @example A text range for inline content
+ * ```ts
+ * import { textEvent } from './events.ts';
+ *
+ * const evt = textEvent(3, 10, {
+ *   start: { line: 1, column: 4, offset: 3 },
+ *   end: { line: 1, column: 11, offset: 10 },
+ * });
+ * ```
+ *
+ * @example An empty text range
+ * ```ts
+ * import { textEvent } from './events.ts';
+ *
+ * const evt = textEvent(5, 5, {
+ *   start: { line: 1, column: 6, offset: 5 },
+ *   end: { line: 1, column: 6, offset: 5 },
+ * });
+ * ```
+ *
+ * @param start_offset - Inclusive start offset (UTF-16 code units).
+ * @param end_offset - Exclusive end offset (UTF-16 code units).
+ * @param position - Source position.
+ */
+export function textEvent(
+  start_offset: number,
+  end_offset: number,
+  position: Position,
+): TextEvent {
+  return { kind: 'text', start_offset, end_offset, position };
+}
+
+/**
+ * Create a `TokenEvent`.
+ *
+ * @example A heading marker token event
+ * ```ts
+ * import { tokenEvent } from './events.ts';
+ * import { TokenType } from './token.ts';
+ *
+ * const evt = tokenEvent(TokenType.HEADING_MARKER, 0, 2, {
+ *   start: { line: 1, column: 1, offset: 0 },
+ *   end: { line: 1, column: 3, offset: 2 },
+ * });
+ * ```
+ *
+ * @example A newline token event
+ * ```ts
+ * import { tokenEvent } from './events.ts';
+ * import { TokenType } from './token.ts';
+ *
+ * const evt = tokenEvent(TokenType.NEWLINE, 13, 14, {
+ *   start: { line: 1, column: 14, offset: 13 },
+ *   end: { line: 2, column: 1, offset: 14 },
+ * });
+ * ```
+ *
+ * @param token_type - The token type from the tokenizer.
+ * @param start_offset - Inclusive start offset (UTF-16 code units).
+ * @param end_offset - Exclusive end offset (UTF-16 code units).
+ * @param position - Source position.
+ */
+export function tokenEvent(
+  token_type: TokenType,
+  start_offset: number,
+  end_offset: number,
+  position: Position,
+): TokenEvent {
+  return { kind: 'token', token_type, start_offset, end_offset, position };
+}
+
+/**
+ * Create an {@linkcode ErrorEvent}.
+ *
+ * The simplest form takes just a message and position. Pass an
+ * {@linkcode ErrorEventOptions} object to attach structured diagnostic
+ * metadata (severity, machine-readable code, recovery status, source
+ * stage, and arbitrary details).
+ *
+ * The message should explain the local recovery in plain English. The code is
+ * what downstream tooling should usually match on.
+ *
+ * For example, an editor may decide to:
+ *
+ * - show a gutter warning for malformed input
+ * - offer a quick fix for a known missing delimiter
+ * - ignore the diagnostic during tolerant preview rendering
+ *
+ * Those responses are consumer choices, not parser requirements.
+ *
+ * @example Emitting an error for an unclosed template
+ * ```ts
+ * import { errorEvent } from './events.ts';
+ *
+ * const evt = errorEvent('Unclosed template at end of input', {
+ *   start: { line: 5, column: 1, offset: 42 },
+ *   end: { line: 5, column: 3, offset: 44 },
+ * });
+ * ```
+ *
+ * @example An error for malformed table syntax
+ * ```ts
+ * import { errorEvent } from './events.ts';
+ *
+ * const evt = errorEvent('Malformed table row', {
+ *   start: { line: 10, column: 1, offset: 100 },
+ *   end: { line: 10, column: 3, offset: 102 },
+ * }, {
+ *   severity: 'warning',
+ *   code: 'TABLE_ROW_MALFORMED',
+ *   recoverable: true,
+ *   source: 'block',
+ * });
+ * ```
+ *
+ * @param message - Human-readable description.
+ * @param position - Source position where the error was detected.
+ * @param options - Optional structured diagnostic metadata.
+ */
+export function errorEvent(
+  message: string,
+  position: Position,
+  options: ErrorEventOptions = {},
+): ErrorEvent {
+  // Object.assign merges the base fields with any optional metadata in a
+  // single allocation. Only the fields present in `options` are included,
+  // so a simple errorEvent('msg', pos) produces { kind, message, position }
+  // without undefined keys for severity, code, etc.
+  return Object.assign(
+    { kind: 'error' as const, message, position },
+    options,
+  );
+}
+
+// ---------------------------------------------------------------------------
+// Type guards
+// ---------------------------------------------------------------------------
+//
+// Type guards let consumers narrow a `WikitextEvent` to a specific variant.
+// TypeScript's type system uses the return type `event is EnterEvent` (a
+// "type predicate") to narrow the type inside an `if` block or `.filter()`.
+//
+// These are thin wrappers around `event.kind === '...'`, but they're useful
+// for passing as callbacks (e.g., `events.filter(isEnterEvent)`) where an
+// inline arrow function would be noisier.
+
+/**
+ * Check whether a `WikitextEvent` is an `EnterEvent`.
+ *
+ * @example Filtering for enter events
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isEnterEvent } from './events.ts';
+ *
+ * function countEnters(events: WikitextEvent[]): number {
+ *   return events.filter(isEnterEvent).length;
+ * }
+ * ```
+ *
+ * @example Narrowing in a switch
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isEnterEvent } from './events.ts';
+ *
+ * function handleEvent(evt: WikitextEvent) {
+ *   if (isEnterEvent(evt)) {
+ *     evt.node_type; // string (narrowed)
+ *     evt.props;    // Record (narrowed)
+ *   }
+ * }
+ * ```
+ */
+export function isEnterEvent(event: WikitextEvent): event is EnterEvent {
+  return event.kind === 'enter';
+}
+
+/**
+ * Check whether a `WikitextEvent` is an `ExitEvent`.
+ *
+ * @example Filtering for exit events
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isExitEvent } from './events.ts';
+ *
+ * const exits = events.filter(isExitEvent);
+ * ```
+ *
+ * @example Narrowing
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isExitEvent } from './events.ts';
+ *
+ * function handle(evt: WikitextEvent) {
+ *   if (isExitEvent(evt)) {
+ *     evt.node_type; // narrowed
+ *   }
+ * }
+ * ```
+ */
+export function isExitEvent(event: WikitextEvent): event is ExitEvent {
+  return event.kind === 'exit';
+}
+
+/**
+ * Check whether a `WikitextEvent` is a `TextEvent`.
+ *
+ * @example Filtering for text events
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isTextEvent } from './events.ts';
+ *
+ * const texts = events.filter(isTextEvent);
+ * ```
+ *
+ * @example Narrowing
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isTextEvent } from './events.ts';
+ *
+ * function handle(evt: WikitextEvent) {
+ *   if (isTextEvent(evt)) {
+ *     evt.start_offset; // narrowed
+ *   }
+ * }
+ * ```
+ */
+export function isTextEvent(event: WikitextEvent): event is TextEvent {
+  return event.kind === 'text';
+}
+
+/**
+ * Check whether a `WikitextEvent` is a `TokenEvent`.
+ *
+ * @example Filtering for token events
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isTokenEvent } from './events.ts';
+ *
+ * const tokens = events.filter(isTokenEvent);
+ * ```
+ *
+ * @example Narrowing
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isTokenEvent } from './events.ts';
+ *
+ * function handle(evt: WikitextEvent) {
+ *   if (isTokenEvent(evt)) {
+ *     evt.token_type; // narrowed
+ *   }
+ * }
+ * ```
+ */
+export function isTokenEvent(event: WikitextEvent): event is TokenEvent {
+  return event.kind === 'token';
+}
+
+/**
+ * Check whether a `WikitextEvent` is an `ErrorEvent`.
+ *
+ * @example Collecting parse errors
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isErrorEvent } from './events.ts';
+ *
+ * const errors = events.filter(isErrorEvent);
+ * ```
+ *
+ * @example Narrowing
+ * ```ts
+ * import type { WikitextEvent } from './events.ts';
+ * import { isErrorEvent } from './events.ts';
+ *
+ * function handle(evt: WikitextEvent) {
+ *   if (isErrorEvent(evt)) {
+ *     evt.message; // narrowed
+ *   }
+ * }
+ * ```
+ */
+export function isErrorEvent(event: WikitextEvent): event is ErrorEvent {
+  return event.kind === 'error';
+}
diff --git a/experiments/event-shape-study/lazy-position/code/filter.ts b/experiments/event-shape-study/lazy-position/code/filter.ts
new file mode 100644
index 0000000..17f1472
--- /dev/null
+++ b/experiments/event-shape-study/lazy-position/code/filter.ts
@@ -0,0 +1,405 @@
+/**
+ * Tree and event filtering helpers.
+ *
+ * The parser's event stream is already useful on its own, but many consumers
+ * still want small convenience helpers for common tree queries such as "find
+ * every template" or "walk every node in document order".
+ *
+ * This module keeps those helpers deliberately small and predictable. Nothing
+ * here changes parser behavior. These are consumer utilities built on top of
+ * the core data shapes.
+ *
+ * @example Collecting all templates from a tree
+ * ```ts
+ * import { filterTemplates } from './filter.ts';
+ * import { parse } from './parse.ts';
+ *
+ * const tree = parse('{{Infobox|name=value}}');
+ * const templates = filterTemplates(tree);
+ * ```
+ *
+ * @module
+ */
+
+import type {
+  Argument,
+  BehaviorSwitch,
+  CategoryLink,
+  ExternalLink,
+  ImageLink,
+  List,
+  MagicWord,
+  ParserFunction,
+  Reference,
+  Redirect,
+  Table,
+  Template,
+  Wikilink,
+  WikistNode,
+  WikistNodeType,
+  WikistParent,
+  WikistRoot,
+} from './ast.ts';
+import type { WikitextEvent } from './events.ts';
+import type { ParseDiagnostic, ParseDiagnosticAnchor } from './tree_builder.ts';
+
+import { isParent } from './ast.ts';
+
+/**
+ * Context passed to a tree visitor.
+ */
+export interface VisitContext {
+  /** Parent node that owns the visited node, if one exists. */
+  readonly parent?: WikistParent;
+  /** Child index inside the parent, if one exists. */
+  readonly index?: number;
+}
+
+/**
+ * Callback shape used by {@linkcode visit}.
+ */
+export type VisitHandler = (node: WikistNode, context: VisitContext) => void;
+
+/**
+ * Resolved location of a node reached through a tree path.
+ */
+export interface TreePathResolution {
+  /** Node reached by following the path. */
+  readonly node: WikistNode;
+  /** Parent that owns the resolved node, when one exists. */
+  readonly parent?: WikistParent;
+  /** Child index inside the parent, when one exists. */
+  readonly index?: number;
+}
+
+/**
+ * Node types whose user-facing identity can be matched by {@linkcode matches}.
+ *
+ * Some of these nodes expose that identity through `name`, others through
+ * `target`, but they all support the same normalized name comparison helper.
+ */
+export type MatchableNode =
+  | Argument
+  | BehaviorSwitch
+  | CategoryLink
+  | ImageLink
+  | MagicWord
+  | ParserFunction
+  | Redirect
+  | Template
+  | Wikilink;
+
+/**
+ * Fixed mapping from node type to the field used by `matches()`.
+ *
+ * Some nodes expose their user-visible identity through `name`, others through
+ * `target`. Centralizing that distinction here keeps the comparison logic
+ * simple and avoids repeating a large switch in the hot path.
+ */
+const MATCH_NAME_FIELD_LOOKUP: Partial> = Object.assign(
+  Object.create(null),
+  {
+    template: 'name',
+    'parser-function': 'name',
+    'magic-word': 'name',
+    'behavior-switch': 'name',
+    argument: 'name',
+    wikilink: 'target',
+    'image-link': 'target',
+    'category-link': 'target',
+    redirect: 'target',
+  },
+);
+
+/**
+ * Visit every node in pre-order depth-first order.
+ *
+ * The visitor runs on the current node before its children. That matches the
+ * usual "walk the tree top-down" mental model used by most syntax-tree tools.
+ */
+export function visit(node: WikistNode, visitor: VisitHandler): void {
+  walk(node, visitor, undefined, undefined);
+}
+
+/**
+ * Collect every node of a given type from a tree.
+ *
+ * This is the simplest recursive query helper in the module. It intentionally
+ * walks the public tree shape instead of relying on internal parser details, so
+ * it stays useful for both parsed trees and trees built by hand in tests or
+ * downstream tooling.
+ */
+export function filter(
+  tree: WikistNode,
+  type: Type,
+): Extract[] {
+  const matches: Extract[] = [];
+
+  visit(tree, (node) => {
+    if (node.type === type) {
+      matches.push(node as Extract);
+    }
+  });
+
+  return matches;
+}
+
+/**
+ * Collect all template nodes.
+ */
+export function filterTemplates(tree: WikistNode): Template[] {
+  return filter(tree, 'template');
+}
+
+/**
+ * Collect visible link nodes.
+ *
+ * This includes wiki links, external links, and redirects. Image embeds and
+ * category assignments have their own dedicated helpers.
+ */
+export function filterLinks(
+  tree: WikistNode,
+): Array {
+  return [
+    ...filter(tree, 'wikilink'),
+    ...filter(tree, 'external-link'),
+    ...filter(tree, 'redirect'),
+  ];
+}
+
+/**
+ * Collect all image-link nodes.
+ */
+export function filterImages(tree: WikistNode): ImageLink[] {
+  return filter(tree, 'image-link');
+}
+
+/**
+ * Collect all list nodes.
+ */
+export function filterLists(tree: WikistNode): List[] {
+  return filter(tree, 'list');
+}
+
+/**
+ * Collect all table nodes.
+ */
+export function filterTables(tree: WikistNode): Table[] {
+  return filter(tree, 'table');
+}
+
+/**
+ * Collect all category-link nodes.
+ */
+export function filterCategories(tree: WikistNode): CategoryLink[] {
+  return filter(tree, 'category-link');
+}
+
+/**
+ * Collect all reference nodes.
+ */
+export function filterReferences(tree: WikistNode): Reference[] {
+  return filter(tree, 'reference');
+}
+
+/**
+ * Compare a node name or target against a candidate string.
+ *
+ * The normalization is intentionally conservative: trim outer whitespace,
+ * treat underscores like spaces, collapse repeated spaces, and compare
+ * case-insensitively. That makes common wiki-name matching less fragile
+ * without claiming full MediaWiki title normalization.
+ */
+export function matches(node: MatchableNode, name: string): boolean {
+  return normalizeName(readMatchName(node)) === normalizeName(name);
+}
+
+/**
+ * Lazily filter an event iterable.
+ *
+ * This is the event-stream counterpart to `filter()`. It preserves laziness so
+ * callers can keep streaming large event sources instead of materializing the
+ * whole stream up front.
+ */
+export function* filterEvents(
+  events: Iterable,
+  predicate: (event: WikitextEvent) => boolean,
+): Generator {
+  for (const event of events) {
+    if (predicate(event)) {
+      yield event;
+    }
+  }
+}
+
+/**
+ * Collect the full event slices for every subtree of a given node type.
+ *
+ * Each returned array starts with the matching `enter` event and ends with the
+ * matching `exit` event. Nested matches are preserved as separate arrays.
+ *
+ * The algorithm keeps a small stack of active matching groups. Every incoming
+ * event is appended to all currently open groups, and a new group starts when
+ * a matching `enter` appears.
+ *
+ * Example for collecting `wikilink` slices:
+ *
+ * ```text
+ * enter(paragraph)
+ * text("A ")
+ * enter(wikilink)   -> start new group
+ * text("Mars")      -> appended to that group
+ * exit(wikilink)    -> close and store that group
+ * ```
+ */
+export function collectEvents(
+  events: Iterable,
+  node_type: string,
+): WikitextEvent[][] {
+  const active: WikitextEvent[][] = [];
+  const result: WikitextEvent[][] = [];
+
+  for (const event of events) {
+    if (event.kind === 'enter' && event.node_type === node_type) {
+      for (const group of active) {
+        group.push(event);
+      }
+      active.push([event]);
+      continue;
+    }
+
+    for (const group of active) {
+      group.push(event);
+    }
+
+    if (event.kind === 'exit' && event.node_type === node_type && active.length > 0) {
+      const group = active.pop();
+      if (group !== undefined) {
+        result.push(group);
+      }
+    }
+  }
+
+  return result;
+}
+
+/**
+ * Resolve a root-relative child-index path into a concrete node location.
+ *
+ * This helper is the low-level building block behind tree-path diagnostic
+ * anchors. Diagnostics preserve the path while the tree is being built, and
+ * higher-level helpers can turn that path back into a real node reference plus
+ * parent/index context.
+ *
+ * ```text
+ * root
+ * ├─ paragraph        path [0]
+ * │  └─ bold          path [0, 0]
+ * └─ table            path [1]
+ * ```
+ *
+ * If the path no longer matches the tree shape, the function returns
+ * `undefined` instead of guessing.
+ * That fail-closed behavior matters when callers hold onto a path longer than
+ * the tree it came from, or accidentally resolve it against a different tree.
+ */
+export function resolveTreePath(
+  tree: WikistRoot,
+  tree_path: readonly number[],
+): TreePathResolution | undefined {
+  let current: WikistNode = tree;
+  let parent: WikistParent | undefined;
+  let index: number | undefined;
+
+  if (tree_path.length === 0) {
+    return { node: tree };
+  }
+
+  for (const child_index of tree_path) {
+    if (!isParent(current)) return undefined;
+    if (child_index < 0 || child_index >= current.children.length) {
+      return undefined;
+    }
+
+    parent = current;
+    index = child_index;
+    current = current.children[child_index];
+  }
+
+  return {
+    node: current,
+    parent,
+    index,
+  };
+}
+
+/**
+ * Resolve one diagnostic anchor to the nearest concrete node location.
+ *
+ * Diagnostics currently expose one narrow anchor kind: `tree-path`. It is a
+ * snapshot of the nearest route through the final materialized tree. That is
+ * enough for editor hints, inspections, and recovery UIs today without
+ * promising edit-stable anchor behavior before session edit tracking exists.
+ *
+ * The helper returns `undefined` for stale anchors instead of guessing. That
+ * matters when a caller accidentally resolves an anchor against a different
+ * tree instance or after reshaping the tree.
+ */
+export function resolveDiagnosticAnchor(
+  tree: WikistRoot,
+  anchor: ParseDiagnosticAnchor,
+): TreePathResolution | undefined {
+  switch (anchor.kind) {
+    case 'tree-path':
+      return resolveTreePath(tree, anchor.path);
+  }
+}
+
+/**
+ * Resolve the nearest node location for one parse diagnostic.
+ *
+ * This is a convenience wrapper over {@linkcode resolveDiagnosticAnchor} so
+ * callers do not need to manually thread `diagnostic.anchor` through every use
+ * site.
+ *
+ * Today those diagnostics mostly come from block-parser recovery events and
+ * tree-builder recovery steps. `parse()` intentionally drops them,
+ * `parseWithDiagnostics()` preserves them for inspection, and
+ * `parseWithRecovery()` adds an explicit boolean summary on top of the same
+ * diagnostics.
+ */
+export function locateDiagnostic(
+  tree: WikistRoot,
+  diagnostic: ParseDiagnostic,
+): TreePathResolution | undefined {
+  return resolveDiagnosticAnchor(tree, diagnostic.anchor);
+}
+
+function walk(
+  node: WikistNode,
+  visitor: VisitHandler,
+  parent: WikistParent | undefined,
+  index: number | undefined,
+): void {
+  visitor(node, { parent, index });
+
+  if (!isParent(node)) return;
+
+  for (let child_index = 0; child_index < node.children.length; child_index++) {
+    walk(node.children[child_index], visitor, node, child_index);
+  }
+}
+
+function readMatchName(node: MatchableNode): string {
+  return MATCH_NAME_FIELD_LOOKUP[node.type] === 'name'
+    ? (node as Argument | BehaviorSwitch | MagicWord | ParserFunction | Template).name
+    : (node as CategoryLink | ImageLink | Redirect | Wikilink).target;
+}
+
+function normalizeName(value: string): string {
+  return value
+    .trim()
+    .replaceAll('_', ' ')
+    .replace(/\s+/g, ' ')
+    .toLowerCase();
+}
\ No newline at end of file
diff --git a/experiments/event-shape-study/lazy-position/code/inline_parser.ts b/experiments/event-shape-study/lazy-position/code/inline_parser.ts
new file mode 100644
index 0000000..ea4bf0d
--- /dev/null
+++ b/experiments/event-shape-study/lazy-position/code/inline_parser.ts
@@ -0,0 +1,2489 @@
+/**
+ * Inline event enrichment over block-parser text ranges.
+ *
+ * The block parser decides the large document shape: headings, paragraphs,
+ * lists, tables, and other block nodes. It intentionally does not decide what
+ * inline markup means inside those blocks. This module is the next stage. It
+ * reads the block parser's text ranges and expands them into finer-grained
+ * inline enter/exit/text events.
+ *
+ * This stage is still event-first. It does not build tree nodes directly.
+ * Instead it emits the same event stream shape the future `events()` API and
+ * tree builder will consume.
+ *
+ * Performance matters here. The block parser emits raw text spans without
+ * inline meaning attached, so this module first merges adjacent spans before
+ * scanning them. The scanner then works in absolute source offsets and uses
+ * `charCodeAt()` directly. It only slices strings when a node actually needs a
+ * convenience string field such as `target`, `name`, or `value`.
+ *
+ * A concrete example helps here. Suppose the block parser has already decided
+ * that this source belongs to one paragraph line:
+ *
+ * ```text
+ * Hello [[Mars|planet]] world
+ * ```
+ *
+ * It may hand the inline parser one merged text range covering that whole line.
+ * The inline parser then walks left to right inside that range, keeping plain
+ * text plain until it reaches `[[`, and only then emitting link structure.
+ *
+ * The high-level flow inside one merged text group looks like this:
+ *
+ * ```text
+ * merged text range: "Hello [[Mars|planet]] world"
+ *
+ *   plain text   opener         plain text
+ *   "Hello "     "[["           " world"
+ *       |          |                |
+ *       +-- emit text when opener appears
+ *                  +-- emit wikilink events for Mars|planet
+ *                                   +-- emit trailing text at end
+ * ```
+ *
+ * That matters because most source bytes are still ordinary text. The parser
+ * stays fast by treating plain text as the default and only paying extra work
+ * when a real opener is present.
+ *
+ * The current implementation covers:
+ * - apostrophe emphasis (`''`, `'''`, `'''''`)
+ * - wikilinks, category links, and image links
+ * - bracketed external links and bare URLs
+ * - templates, parser functions, and triple-brace arguments
+ * - comments, HTML entities, behavior switches, and signatures
+ * - `
`, ``, ``, and generic HTML / extension tags + * + * Recovery rule: when a construct cannot be closed safely, it falls back to + * plain text rather than throwing or inventing structure. + * + * @example Enriching block-level events with inline markup + * ```ts + * import { inlineEvents } from './inline_parser.ts'; + * import { blockEvents } from './block_parser.ts'; + * import { tokenize } from './tokenizer.ts'; + * + * const source = "A [[Page|link]] with ''italic'' text."; + * const events = [...inlineEvents(source, blockEvents(source, tokenize(source)))]; + * ``` + * + * @module + */ + +import type { TextSource } from './text_source.ts'; +import type { Point, Position, TextEvent, WikitextEvent } from './events.ts'; +import { + DiagnosticCode, + errorEvent, +} from './events.ts'; +import { + enterEventFromPoints, + exitEventFromPoints, + textEventFromPoints, +} from './event_factory.ts'; + +const CC_LF = 0x0a; +const CC_CR = 0x0d; +const CC_SPACE = 0x20; +const CC_TAB = 0x09; +const CC_BANG = 0x21; +const CC_HASH = 0x23; +const CC_AMP = 0x26; +const CC_PERCENT = 0x25; +const CC_APOSTROPHE = 0x27; +const CC_OPEN_PAREN = 0x28; +const CC_CLOSE_PAREN = 0x29; +const CC_PLUS = 0x2b; +const CC_COMMA = 0x2c; +const CC_DASH = 0x2d; +const CC_PERIOD = 0x2e; +const CC_SLASH = 0x2f; +const CC_COLON = 0x3a; +const CC_SEMICOLON = 0x3b; +const CC_LT = 0x3c; +const CC_EQUALS = 0x3d; +const CC_QUESTION = 0x3f; +const CC_GT = 0x3e; +const CC_AT = 0x40; +const CC_OPEN_BRACKET = 0x5b; +const CC_CLOSE_BRACKET = 0x5d; +const CC_OPEN_BRACE = 0x7b; +const CC_CLOSE_BRACE = 0x7d; +const CC_UNDERSCORE = 0x5f; +const CC_PIPE = 0x7c; +const CC_TILDE = 0x7e; +const CC_DOUBLE_QUOTE = 0x22; +const CC_SINGLE_QUOTE = 0x27; + +/** + * Return whether an ASCII code point can begin one of the inline constructs + * this parser knows how to match. + * + * Keeping the cases in one switch makes the lookup table easier to audit than + * duplicating a long boolean expression inside `Uint8Array.from(...)`. The + * table still exists because the hot path wants O(1) membership checks while + * scanning very large plain-text ranges. + * + * Two of the starts are worth calling out because they are less obvious than + * `[` or `<`: + * - `:` is here because bare URI detection commits at the scheme separator + * - `{` is here so templates and triple-brace arguments stay on the fast path + */ +function isInlineSpecialStart(code: number): boolean { + switch (code) { + case CC_LT: + case CC_UNDERSCORE: + case CC_TILDE: + case CC_AMP: + case CC_OPEN_BRACKET: + case CC_APOSTROPHE: + case CC_COLON: + case CC_OPEN_BRACE: + return true; + + default: + return false; + } +} + +const INLINE_SPECIAL_START = Uint8Array.from({ length: 128 }, (_, code) => + isInlineSpecialStart(code) ? 1 : 0, +); + +/** + * Shared context for scanning one merged text group. + * + * The block parser may hand us several adjacent text events that are really one + * continuous inline region. This context stores the source range, the point + * where that region began, and enough line-start information to rebuild precise + * `Position` values on demand without storing a `Point` for every code unit. + * + * Example: + * + * ```text + * source: "Hello\n[[Mars]]" + * offsets: 01234567890123 + * 0 5 6 13 + * + * merged group: [0, 14) + * start_point: line 1, column 1, offset 0 + * line_starts: [0, 6] + * ``` + * + * With that information, the parser can answer questions like "what point is + * offset 9?" without storing a full point object for offsets 0 through 14. + */ +/** + * @internal + * Shared scan context for one merged inline text group. + */ +export interface TextGroupContext { + /** The original source text backing this inline scan. */ + source: TextSource; + /** Inclusive start offset of the merged text group. */ + start_offset: number; + /** Exclusive end offset of the merged text group. */ + end_offset: number; + /** Source position for `start_offset`, used as the anchor for later points. */ + start_point: Point; + /** Absolute offsets where each logical line in this text group begins. */ + line_starts: number[]; + /** Whether inline recovery should emit diagnostic events. */ + diagnostics: boolean; +} + +/** + * Internal switches for one inline-enrichment lane. + * + * This mirrors the public diagnostics choice so the inline parser does not do + * extra diagnostic work when the caller only wanted the cheap default tree. + * Materialization policy is intentionally not part of the event layer. + */ +export interface InlineEventOptions { + /** Whether inline-stage diagnostics are emitted into the event stream. */ + readonly diagnostics?: boolean; +} + +/** + * Result of recognizing one inline construct at the current cursor. + * + * `end_offset` tells the caller where scanning should resume. `events` contains + * the full enter/exit/text sequence for the construct that matched. + * + * @internal + */ +export interface SpecialMatch { + /** Inclusive start offset of the recognized construct. */ + start_offset?: number; + /** Exclusive end offset of the recognized construct. */ + end_offset: number; + /** Events emitted for the recognized inline construct. */ + events: WikitextEvent[]; +} + +/** + * Parsed shape of one opening HTML-like tag. + * + * This is used for generic tags plus special cases such as `nowiki`, `ref`, + * and `br`. We keep both the original tag name and a lowercase copy so later + * matching can stay case-insensitive without reslicing repeatedly. + * + * @internal + */ +export interface TagOpen { + /** Discriminant for successful opener recognition. */ + kind: 'parsed'; + /** Tag name exactly as it appeared in source. */ + tag_name: string; + /** Lowercased tag name used for comparisons. */ + tag_name_lower: string; + /** Exclusive end offset of the parsed opening tag. */ + end_offset: number; + /** Whether the opening tag ended with `/>`. */ + self_closing: boolean; + /** Parsed attributes when the tag carried any. */ + attributes?: Readonly>; +} + +/** + * A plausible tag opener that never reached its closing `>`. + * + * This is the boundary between "the source definitely opened a tag" and + * "the source only looked like it might start one". The inline parser uses + * this shape to preserve the original bytes as text while still reporting a + * recovery diagnostic. + * + * @internal + */ +export interface UnterminatedTagOpen { + /** Discriminant for opener recovery before any tag node is committed. */ + kind: 'unterminated'; + /** Tag name exactly as it appeared before EOF or range end. */ + tag_name: string; + /** Lowercased tag name used for diagnostics. */ + tag_name_lower: string; +} + +/** + * Parsed shape of a closing HTML-like tag such as ``. + * + * @internal + */ +export interface TagClose { + /** Lowercased tag name used for close/open matching. */ + tag_name_lower: string; + /** Exclusive end offset of the parsed closing tag. */ + end_offset: number; +} + +/** + * Range of the matching close tag for a non-self-closing HTML-like node. + * + * @internal + */ +export interface TagBoundary { + /** Inclusive start offset of the closing tag. */ + start_offset: number; + /** Exclusive end offset of the closing tag. */ + end_offset: number; +} + +/** + * Enrich block-parser text spans with inline markup events. + * + * Consecutive text events are merged before scanning because callers may still + * hand this stage smaller neighboring spans, and inline constructs can cross + * those boundaries. Parsing each span in isolation would miss multi-span + * constructs such as `[[link|text]]` and `{{template}}`. + */ +export function* inlineEvents( + source: TextSource, + events: Iterable, + options: InlineEventOptions = {}, +): Generator { + let pending_text: TextEvent[] = []; + + for (const event of events) { + if (event.kind === 'text') { + // Today, one text group is only allowed to grow across contiguous source + // slices. Paragraph continuation lines therefore arrive as separate + // groups because the newline between them is structural and omitted from + // the text stream. A future discontiguous handoff experiment would change + // this exact check: it would keep the paragraph lines together in one + // logical group without pretending the omitted newline is plain text. + if ( + pending_text.length > 0 && + pending_text[pending_text.length - 1].end_offset !== event.start_offset + ) { + yield* parseTextGroup(source, pending_text, options); + pending_text = []; + } + + pending_text.push(event); + continue; + } + + if (pending_text.length > 0) { + yield* parseTextGroup(source, pending_text, options); + pending_text = []; + } + + yield event; + } + + if (pending_text.length > 0) { + yield* parseTextGroup(source, pending_text, options); + } +} + +/** + * Parse one merged run of adjacent block-parser text events. + * + * The block parser may emit several neighboring text events for one logical + * inline region. Inline constructs can span across those boundaries, so we + * first merge the run into one scan range and only then resolve inline syntax. + * + * Concrete example: + * + * ```text + * block stage hands us: + * text("Hello ") + * text("[[Mars|planet]]") + * text(" world") + * + * inline stage treats that as one scan range: + * "Hello [[Mars|planet]] world" + * ``` + * + * That merge step matters because the link syntax crosses the smaller text + * event boundaries. Scanning each piece in isolation would miss the full + * `[[Mars|planet]]` construct. + */ +function* parseTextGroup( + source: TextSource, + events: TextEvent[], + options: InlineEventOptions, +): Generator { + const first = events[0]; + const last = events[events.length - 1]; + + // Most merged text groups are still ordinary prose. When there is no inline + // opener anywhere in the group, preserve the merged text as-is and skip the + // extra work of building line-start tables and rescanning the whole range. + // + // Example safe fast path: + // + // input range: "Just plain text here" + // opener scan: finds no [[, {{, '', <, &, __, ~~~, or bare-url start + // result: emit one plain text event and stop + // + // This is safe because the block parser has already decided the exact source + // range for the text group. If there is no possible inline opener inside + // that range, the inline stage would only rebuild the same text event with + // newly computed positions. + if ( + findNextInlineSpecialStart(source, first.start_offset, last.end_offset) === + last.end_offset + ) { + yield textEventFromPoints( + first.start_offset, + last.end_offset, + first.position.start, + last.position.end, + ); + return; + } + + const ctx = buildTextGroupContext( + source, + first.start_offset, + last.end_offset, + first.position.start, + ); + ctx.diagnostics = options.diagnostics === true; + + yield* parseInlineRange(ctx, ctx.start_offset, ctx.end_offset, true); +} + +/** + * Parse one merged inline text range from left to right. + * + * The key invariant here is that `plain_start` always marks the beginning of + * the next still-unemitted plain-text run. When a matcher recognizes a real + * inline construct, the parser first flushes the plain text before it, then + * emits the construct events, then resumes scanning after the construct. + */ + +/** + * Parse one absolute source range. + * + * Plain text is emitted lazily only when a special construct is found or the + * range ends. + * + * Read this as a left-to-right scan with deferred text emission. The parser + * does not emit `text("H")`, `text("He")`, `text("Hel")`, and so on while it + * walks plain prose. It remembers where the current plain run started, keeps + * scanning, and only emits that plain range when it must split around real + * inline syntax. + * + * ```text + * source: "Hello [[Mars]] world" + * + * plain_start = 0 + * cursor scans forward until it reaches the `[[` at offset 6 + * + * flush text [0, 6) -> "Hello " + * emit wikilink events -> "[[Mars]]" + * resume at offset 14 + * + * after loop flush trailing text [14, end) -> " world" + * ``` + * + * This shape keeps ordinary text cheap. We do not allocate a text event for + * every character. We hold one pending plain range and only emit it when we + * have to split around a recognized construct. + */ +function* parseInlineRange( + ctx: TextGroupContext, + start_offset: number, + end_offset: number, + allow_bare_url: boolean, +): Generator { + let cursor = start_offset; + let plain_start = start_offset; + + while (cursor < end_offset) { + // Most bytes inside a merged text group are still ordinary text. Jumping + // directly to the next possible opener avoids paying `matchSpecial()` on + // every character of long prose-heavy runs. + // + // Example: + // "The red planet is [[Mars]]." + // ^^^^^^^^^^^^^^^^^^^ jump over this plain prose in one cheap scan + // ^ stop here because `[` could begin inline syntax + cursor = findNextInlineSpecialStart(ctx.source, cursor, end_offset); + if (cursor >= end_offset) break; + + const match = matchSpecial(ctx, cursor, end_offset, allow_bare_url, plain_start); + if (match === null) { + cursor++; + continue; + } + + const match_start = match.start_offset ?? cursor; + + if (plain_start < match_start) { + yield emitText(ctx, plain_start, match_start); + } + + for (const event of match.events) { + yield event; + } + + cursor = match.end_offset; + plain_start = cursor; + } + + if (plain_start < end_offset) { + yield emitText(ctx, plain_start, end_offset); + } +} + +/** + * Skip forward to the next character that could plausibly open inline syntax. + */ +function findNextInlineSpecialStart( + source: TextSource, + start_offset: number, + end_offset: number, +): number { + let cursor = start_offset; + + while (cursor < end_offset) { + const code = source.charCodeAt(cursor); + if (code < 128 && INLINE_SPECIAL_START[code] === 1) { + return cursor; + } + cursor++; + } + + return end_offset; +} + +/** + * Try to recognize one inline construct at `cursor`. + * + * The order inside each dispatch branch matters. For example, comment syntax + * must be checked before generic tag parsing because ``. */ +function matchComment( + ctx: TextGroupContext, + cursor: number, + end_offset: number, +): SpecialMatch | null { + if (!hasLiteral(ctx.source, cursor, end_offset, '', cursor + 4, end_offset); + if (close_start === -1) return null; + const end = close_start + 3; + + return { + end_offset: end, + events: wrapLeaf(ctx, cursor, end, 'comment', { + value: ctx.source.slice(cursor + 4, close_start), + }), + }; +} + +/** Match triple-brace argument syntax such as `{{{name|default}}}`. */ +function matchArgument( + ctx: TextGroupContext, + cursor: number, + end_offset: number, +): SpecialMatch | null { + if (!hasLiteral(ctx.source, cursor, end_offset, '{{{')) return null; + + const close_end = findBalanced(ctx.source, cursor, end_offset, '{{{', '}}}'); + if (close_end === -1) return null; + + const inner_start = cursor + 3; + const inner_end = close_end - 3; + const pipe = firstTopLevelSeparator(ctx.source, inner_start, inner_end, CC_PIPE); + const name_range = trimRange(ctx.source, inner_start, pipe === -1 ? inner_end : pipe); + const name = ctx.source.slice(name_range.start, name_range.end); + if (name.length === 0) return null; + + const props = pipe === -1 + ? { name } + : { + name, + default: ctx.source.slice(...rangeTuple(trimRange(ctx.source, pipe + 1, inner_end))), + }; + + return { + end_offset: close_end, + events: wrapLeaf(ctx, cursor, close_end, 'argument', props), + }; +} + +/** + * Match template and parser-function syntax. + * + * This helper resolves the outer `{{...}}` range first, then finds only the + * top-level `|` separators so nested templates or wikilinks do not split the + * outer argument list accidentally. + */ +function matchTemplate( + ctx: TextGroupContext, + cursor: number, + end_offset: number, +): SpecialMatch | null { + if (!hasLiteral(ctx.source, cursor, end_offset, '{{') || hasLiteral(ctx.source, cursor, end_offset, '{{{')) { + return null; + } + + const close_end = findBalanced(ctx.source, cursor, end_offset, '{{', '}}'); + if (close_end === -1) return null; + + const inner_start = cursor + 2; + const inner_end = close_end - 2; + const first_separator = firstTopLevelSeparator(ctx.source, inner_start, inner_end, CC_PIPE); + const separators = first_separator === -1 + ? EMPTY_SEPARATOR_LIST + : [ + first_separator, + ...topLevelSeparators(ctx.source, first_separator + 1, inner_end, CC_PIPE), + ]; + const head_end = first_separator === -1 ? inner_end : first_separator; + const name_range = trimRange(ctx.source, inner_start, head_end); + const name = ctx.source.slice(name_range.start, name_range.end); + if (name.length === 0) return null; + + const node_type = name.startsWith('#') ? 'parser-function' : 'template'; + const outer_start = pointAt(ctx, cursor); + const outer_end = pointAt(ctx, close_end); + const events: WikitextEvent[] = [enterEventFromPoints(node_type, { name }, outer_start, outer_end)]; + + for (let index = 0; index < separators.length; index++) { + const arg_start = separators[index] + 1; + const arg_end = index + 1 < separators.length ? separators[index + 1] : inner_end; + appendTemplateArgument(events, ctx, arg_start, arg_end); + } + + events.push(exitEventFromPoints(node_type, outer_start, outer_end)); + return { end_offset: close_end, events }; +} + +/** + * Parse one template argument inside an already matched template body. + * + * The argument name is structural metadata, so we trim it. The value is parsed + * as another inline range so nested links, templates, and entities still show + * up inside argument payloads. + */ +function appendTemplateArgument( + events: WikitextEvent[], + ctx: TextGroupContext, + start_offset: number, + end_offset: number, +): void { + const eq = topLevelEquals(ctx.source, start_offset, end_offset); + const props = eq === -1 + ? {} + : { + name: ctx.source.slice( + ...rangeTuple(trimRange(ctx.source, start_offset, eq)), + ), + }; + const value_start = eq === -1 ? start_offset : eq + 1; + const arg_start = pointAt(ctx, start_offset); + const arg_end = pointAt(ctx, end_offset); + + events.push(enterEventFromPoints('template-argument', props, arg_start, arg_end)); + appendInlineRange(events, ctx, value_start, end_offset, true); + events.push(exitEventFromPoints('template-argument', arg_start, arg_end)); +} + +/** + * Match double-bracket link syntax. + * + * Namespace dispatch happens here because the same raw `[[...]]` wrapper can + * mean a normal wikilink, category assignment, or image/file link depending on + * the trimmed target text. + */ +function matchWikilink( + ctx: TextGroupContext, + cursor: number, + end_offset: number, +): SpecialMatch | null { + if (!hasLiteral(ctx.source, cursor, end_offset, '[[')) return null; + + const close_end = findBalanced(ctx.source, cursor, end_offset, '[[', ']]'); + if (close_end === -1) return null; + + const inner_start = cursor + 2; + const inner_end = close_end - 2; + const first_separator = firstTopLevelSeparator(ctx.source, inner_start, inner_end, CC_PIPE); + const separators = first_separator === -1 + ? EMPTY_SEPARATOR_LIST + : [ + first_separator, + ...topLevelSeparators(ctx.source, first_separator + 1, inner_end, CC_PIPE), + ]; + const target_end = first_separator === -1 ? inner_end : first_separator; + let target_range = trimRange(ctx.source, inner_start, target_end); + if (target_range.start === target_range.end) return null; + + let leading_colon = false; + if (ctx.source.charCodeAt(target_range.start) === CC_COLON) { + leading_colon = true; + target_range = trimRange(ctx.source, target_range.start + 1, target_range.end); + } + + const target = ctx.source.slice(target_range.start, target_range.end); + const target_lower = target.toLowerCase(); + if (!leading_colon && target_lower.startsWith('category:')) { + const sort_key = separators.length === 0 + ? undefined + : ctx.source.slice( + ...rangeTuple(trimRange(ctx.source, separators[0] + 1, inner_end)), + ); + const props = sort_key === undefined || sort_key.length === 0 + ? { target } + : { target, sort_key }; + return { + end_offset: close_end, + events: wrapLeaf(ctx, cursor, close_end, 'category-link', props), + }; + } + + const node_type = !leading_colon && (target_lower.startsWith('file:') || target_lower.startsWith('image:')) + ? 'image-link' + : 'wikilink'; + const outer_start = pointAt(ctx, cursor); + const outer_end = pointAt(ctx, close_end); + const events: WikitextEvent[] = [enterEventFromPoints(node_type, { target }, outer_start, outer_end)]; + + if (separators.length > 0) { + appendInlineRange(events, ctx, separators[0] + 1, inner_end, false); + } + + events.push(exitEventFromPoints(node_type, outer_start, outer_end)); + return { end_offset: close_end, events }; +} + +/** + * Match bracketed external-link syntax such as `[https://example.com label]`. + * + * The URL must begin immediately after `[` for this form to match. If it does + * not, later recovery may still find a bare URL inside the brackets. + */ +function matchExternalLink( + ctx: TextGroupContext, + cursor: number, + end_offset: number, +): SpecialMatch | null { + if (ctx.source.charCodeAt(cursor) !== CC_OPEN_BRACKET || hasLiteral(ctx.source, cursor, end_offset, '[[')) { + return null; + } + + const url_end = scanUrl(ctx.source, cursor + 1, end_offset, 'explicit'); + if (url_end === cursor + 1) return null; + + const close = indexOfChar(ctx.source, CC_CLOSE_BRACKET, url_end, end_offset); + if (close === -1) return null; + + const url = ctx.source.slice(cursor + 1, url_end); + const outer_end = close + 1; + const outer_start = pointAt(ctx, cursor); + const outer_end_point = pointAt(ctx, outer_end); + const events: WikitextEvent[] = [enterEventFromPoints('external-link', { url }, outer_start, outer_end_point)]; + + let label_start = url_end; + while (label_start < close) { + const code = ctx.source.charCodeAt(label_start); + if (code !== CC_SPACE && code !== CC_TAB) break; + label_start++; + } + + if (label_start < close) { + appendInlineRange(events, ctx, label_start, close, false); + } + + events.push(exitEventFromPoints('external-link', outer_start, outer_end_point)); + return { end_offset: outer_end, events }; +} + +/** Match bare URI-like links in plain text once a scheme separator is reached. */ +function matchBareUrl( + ctx: TextGroupContext, + cursor: number, + end_offset: number, + plain_start: number, +): SpecialMatch | null { + const url_start = findBareUrlStart(ctx.source, cursor, plain_start); + if (url_start === -1 || !isBareUrlStartBoundary(ctx.source, url_start)) { + return null; + } + + const url_end = scanUrl(ctx.source, url_start, end_offset, 'bare'); + if (url_end === url_start) return null; + const url = ctx.source.slice(url_start, url_end); + const start_point = pointAt(ctx, url_start); + const end_point = pointAt(ctx, url_end); + return { + start_offset: url_start, + end_offset: url_end, + events: [ + enterEventFromPoints('external-link', { url }, start_point, end_point), + exitEventFromPoints('external-link', start_point, end_point), + ], + }; +} + +/** + * Match apostrophe-based emphasis runs. + * + * Four apostrophes are the awkward case. MediaWiki-style recovery typically + * treats that as one literal apostrophe plus a bold run, so we do the same by + * emitting the first apostrophe as text and then matching bold from the next + * character. + */ +function matchEmphasis( + ctx: TextGroupContext, + cursor: number, + end_offset: number, +): SpecialMatch | null { + if (ctx.source.charCodeAt(cursor) !== CC_APOSTROPHE) return null; + const run = repeatedCharRun(ctx.source, cursor, end_offset, CC_APOSTROPHE); + if (run < 2) return null; + + if (run === 4) { + const bold = matchDelimitedInline(ctx, cursor + 1, end_offset, 3, 'bold'); + if (bold === null) return null; + return { + end_offset: bold.end_offset, + events: [emitText(ctx, cursor, cursor + 1), ...bold.events], + }; + } + + if (run >= 5) { + return matchDelimitedInline(ctx, cursor, end_offset, 5, 'bold-italic'); + } + if (run === 3) { + return matchDelimitedInline(ctx, cursor, end_offset, 3, 'bold'); + } + return matchDelimitedInline(ctx, cursor, end_offset, 2, 'italic'); +} + +/** + * Parse the contents of one delimited emphasis range. + * + * Emphasis never crosses a physical line break in this implementation. If no + * closing marker is found before the line ends, recovery closes the node at the + * line boundary instead of scanning the whole remaining text group. + */ +function matchDelimitedInline( + ctx: TextGroupContext, + cursor: number, + end_offset: number, + marker_length: number, + node_type: string, +): SpecialMatch | null { + const content_start = cursor + marker_length; + const line_end = findLineEnd(ctx.source, content_start, end_offset); + const close_start = findApostropheClose(ctx.source, content_start, line_end, marker_length); + const content_end = close_start === -1 ? line_end : close_start; + const close_end = close_start === -1 ? line_end : close_start + marker_length; + return { + end_offset: close_end, + events: createWrappedInlineEvents( + ctx, + cursor, + close_end, + node_type, + {}, + content_start, + content_end, + true, + ), + }; +} + +/** + * Match HTML-like tags and the special inline tags built on that syntax. + * + * `br`, `nowiki`, and `ref` get custom node types because downstream consumers + * are likely to care about them directly. Other tags are normalized into the + * generic `html-tag` node with preserved attributes. + */ +function matchTagLike( + ctx: TextGroupContext, + cursor: number, + end_offset: number, +): SpecialMatch | null { + if (ctx.source.charCodeAt(cursor) !== CC_LT) return null; + if (hasLiteral(ctx.source, cursor, end_offset, '` do not interfere with tag + * recovery. + */ +function findMatchingCloseTag( + source: TextSource, + open_tag: TagOpen, + start_offset: number, + end_offset: number, +): TagBoundary | null { + // Generic tag matching can get expensive on malformed input because the naive + // approach checks every byte as a possible tag boundary. We skip directly to + // the next `<` because only that character can begin a relevant open tag, + // close tag, or comment opener. + let cursor = start_offset; + let depth = 0; + + while (cursor < end_offset) { + while (cursor < end_offset && source.charCodeAt(cursor) !== CC_LT) cursor++; + if (cursor >= end_offset) break; + + if (hasLiteral(source, cursor, end_offset, '', cursor + 4, end_offset); + cursor = close === -1 ? end_offset : close + 3; + continue; + } + + const close_tag = parseClosingTag(source, cursor, end_offset); + if (close_tag !== null && close_tag.tag_name_lower === open_tag.tag_name_lower) { + if (depth === 0) { + return { start_offset: cursor, end_offset: close_tag.end_offset }; + } + depth--; + cursor = close_tag.end_offset; + continue; + } + + const nested_open = parseTagOpen(source, cursor, end_offset); + if ( + nested_open !== null && + nested_open.kind === 'parsed' && + nested_open.tag_name_lower === open_tag.tag_name_lower + ) { + if (!nested_open.self_closing) depth++; + cursor = nested_open.end_offset; + continue; + } + + cursor++; + } + + return null; +} + +/** + * Parse loose HTML-style attributes from the raw attribute segment. + * + * Attribute parsing here is recovery-oriented. Unknown attribute names are + * preserved, missing values become empty strings, and malformed fragments are + * skipped rather than throwing. + */ +function parseAttributes( + source: TextSource, + start_offset: number, + end_offset: number, +): Readonly> | undefined { + let cursor = start_offset; + let result: Record | undefined; + + while (cursor < end_offset) { + while (cursor < end_offset) { + const code = source.charCodeAt(cursor); + if (code !== CC_SPACE && code !== CC_TAB && code !== CC_LF && code !== CC_CR) break; + cursor++; + } + if (cursor >= end_offset) break; + if (!isAttrNameChar(source.charCodeAt(cursor))) { + cursor++; + continue; + } + + const name_start = cursor; + cursor++; + while (cursor < end_offset && isAttrNameChar(source.charCodeAt(cursor))) cursor++; + const name = source.slice(name_start, cursor); + + while (cursor < end_offset) { + const code = source.charCodeAt(cursor); + if (code !== CC_SPACE && code !== CC_TAB && code !== CC_LF && code !== CC_CR) break; + cursor++; + } + + let value = ''; + if (cursor < end_offset && source.charCodeAt(cursor) === CC_EQUALS) { + cursor++; + while (cursor < end_offset) { + const code = source.charCodeAt(cursor); + if (code !== CC_SPACE && code !== CC_TAB && code !== CC_LF && code !== CC_CR) break; + cursor++; + } + + if (cursor < end_offset) { + const quote = source.charCodeAt(cursor); + if (quote === CC_DOUBLE_QUOTE || quote === CC_SINGLE_QUOTE) { + cursor++; + const value_start = cursor; + while (cursor < end_offset && source.charCodeAt(cursor) !== quote) cursor++; + value = source.slice(value_start, cursor); + if (cursor < end_offset) cursor++; + } else { + const value_start = cursor; + while (cursor < end_offset) { + const code = source.charCodeAt(cursor); + if (code === CC_SPACE || code === CC_TAB || code === CC_LF || code === CC_CR) break; + cursor++; + } + value = source.slice(value_start, cursor); + } + } + } + + result = result ?? {}; + result[name] = value; + } + + return result; +} + +/** Build a zero-width diagnostic position at one absolute source offset. */ +function zeroWidthPosition(ctx: TextGroupContext, offset: number): Position { + return createPosition(ctx, offset, offset); +} + +/** Report that a plausible HTML-like opener never reached its closing `>`. */ +function unterminatedTagOpenerError( + ctx: TextGroupContext, + offset: number, + tag_name: string, +): WikitextEvent { + return errorEvent( + `Unterminated <${tag_name}> opener before end of inline range.`, + zeroWidthPosition(ctx, offset), + { + severity: 'warning', + code: DiagnosticCode.INLINE_TAG_UNTERMINATED_OPENER, + recoverable: true, + source: 'inline', + details: { tag_name }, + }, + ); +} + +/** Report that a parsed opener never found its matching close tag. */ +function missingCloseTagError( + ctx: TextGroupContext, + offset: number, + tag_name: string, +): WikitextEvent { + return errorEvent( + `Missing closing before end of inline range.`, + zeroWidthPosition(ctx, offset), + { + severity: 'warning', + code: DiagnosticCode.INLINE_TAG_MISSING_CLOSE, + recoverable: true, + source: 'inline', + details: { tag_name }, + }, + ); +} + +/** Reduce raw tag attributes to the public reference-node props we expose. */ +function referenceProps( + attributes?: Readonly>, +): Readonly> { + if (attributes === undefined) return {}; + const props: Record = {}; + if (attributes.name !== undefined) props.name = attributes.name; + if (attributes.group !== undefined) props.group = attributes.group; + return props; +} + +/** Check whether `literal` appears exactly at `offset`. */ +function hasLiteral( + source: TextSource, + offset: number, + end_offset: number, + literal: string, +): boolean { + if (offset + literal.length > end_offset) return false; + for (let index = 0; index < literal.length; index++) { + if (source.charCodeAt(offset + index) !== literal.charCodeAt(index)) return false; + } + return true; +} + +/** Find the next occurrence of a literal string within a source range. */ +function indexOfLiteral( + source: TextSource, + literal: string, + start_offset: number, + end_offset: number, +): number { + // This is a low-level scan helper used by comment and tag recovery. The fast + // first-character guard keeps it cheap on long regions that contain very few + // actual candidates for the requested literal. + const first = literal.charCodeAt(0); + + for (let cursor = start_offset; cursor + literal.length <= end_offset; cursor++) { + if (source.charCodeAt(cursor) === first && hasLiteral(source, cursor, end_offset, literal)) { + return cursor; + } + } + return -1; +} + +/** Find the next occurrence of one character code within a source range. */ +function indexOfChar( + source: TextSource, + code: number, + start_offset: number, + end_offset: number, +): number { + for (let cursor = start_offset; cursor < end_offset; cursor++) { + if (source.charCodeAt(cursor) === code) return cursor; + } + return -1; +} + +/** Count how many times the same character repeats from `offset`. */ +function repeatedCharRun( + source: TextSource, + offset: number, + end_offset: number, + code: number, +): number { + let run = 0; + while (offset + run < end_offset && source.charCodeAt(offset + run) === code) run++; + return run; +} + +/** + * Scan a bare or bracketed external-link URL prefix. + * + * This stays deliberately lightweight. It accepts either a generic + * `scheme://...` prefix or a small allowlist of colon-only schemes such as + * `mailto:` and `data:`. The scan then applies simple boundary rules and a + * trim pass instead of instantiating heavier URL objects in the hot path. + */ +type UrlScanMode = 'bare' | 'explicit'; + +type UriPrefix = { + scheme_end: number; + payload_start: number; + has_authority: boolean; +}; + +function scanUrl( + source: TextSource, + start_offset: number, + end_offset: number, + mode: UrlScanMode, +): number { + const prefix = scanUriPrefix(source, start_offset, end_offset); + if (prefix === null) return start_offset; + + const { payload_start } = prefix; + + let cursor = payload_start; + if (cursor >= end_offset) return start_offset; + + const first_payload_code = source.charCodeAt(cursor); + if (isUrlStopCode(first_payload_code)) return start_offset; + + let open_paren_count = 0; + let open_square_count = 0; + let open_curly_count = 0; + + while (cursor < end_offset) { + const code = source.charCodeAt(cursor); + + if (code === CC_OPEN_PAREN) { + open_paren_count++; + cursor++; + continue; + } + + if (code === CC_CLOSE_PAREN) { + if (open_paren_count === 0) break; + open_paren_count--; + cursor++; + continue; + } + + if (code === CC_OPEN_BRACKET) { + open_square_count++; + cursor++; + continue; + } + + if (code === CC_CLOSE_BRACKET) { + if (open_square_count === 0) break; + open_square_count--; + cursor++; + continue; + } + + if (code === CC_OPEN_BRACE) { + open_curly_count++; + cursor++; + continue; + } + + if (code === CC_CLOSE_BRACE) { + if (open_curly_count === 0) break; + open_curly_count--; + cursor++; + continue; + } + + if (isUrlStopCode(code)) { + break; + } + + cursor++; + } + + const trimmed = trimBareUrlTrailingPunctuation(source, start_offset, cursor); + if (mode === 'bare' && !isBareAutolinkCandidate(source, start_offset, prefix, trimmed)) { + return start_offset; + } + + return trimmed > payload_start ? trimmed : start_offset; +} + +function findBareUrlStart(source: TextSource, colon_offset: number, min_offset: number): number { + let cursor = colon_offset - 1; + + while (cursor >= min_offset && isUriSchemeChar(source.charCodeAt(cursor))) { + cursor--; + } + + const start_offset = cursor + 1; + const scheme_length = colon_offset - start_offset; + if (scheme_length < 2) return -1; + if (start_offset < min_offset || !isAsciiLetter(source.charCodeAt(start_offset))) { + return -1; + } + + return start_offset; +} + +function scanUriPrefix(source: TextSource, start_offset: number, end_offset: number): UriPrefix | null { + if (start_offset >= end_offset || !isAsciiLetter(source.charCodeAt(start_offset))) { + return null; + } + + let cursor = start_offset + 1; + while (cursor < end_offset && isUriSchemeChar(source.charCodeAt(cursor))) { + cursor++; + } + + if (cursor >= end_offset || source.charCodeAt(cursor) !== CC_COLON) { + return null; + } + + const scheme_end = cursor; + if (scheme_end - start_offset < 2) { + return null; + } + + const after_colon = cursor + 1; + if (after_colon >= end_offset) { + return null; + } + + if ( + after_colon + 1 < end_offset && + source.charCodeAt(after_colon) === CC_SLASH && + source.charCodeAt(after_colon + 1) === CC_SLASH + ) { + return { + scheme_end, + payload_start: after_colon + 2, + has_authority: true, + }; + } + + return { + scheme_end, + payload_start: after_colon, + has_authority: false, + }; +} + +function isBareUrlStartBoundary(source: TextSource, offset: number): boolean { + if (offset <= 0) return true; + + const previous = source.charCodeAt(offset - 1); + return !isAsciiAlphanumeric(previous) && previous !== CC_UNDERSCORE; +} + +function isUrlStopCode(code: number): boolean { + return code === CC_SPACE || + code === CC_TAB || + code === CC_LF || + code === CC_CR || + code === CC_LT || + code === CC_GT || + code === CC_DOUBLE_QUOTE || + code === CC_SINGLE_QUOTE; +} + +function isUriSchemeChar(code: number): boolean { + return isAsciiAlphanumeric(code) || code === CC_PLUS || code === CC_DASH || code === CC_PERIOD; +} + +function isBareAutolinkCandidate( + source: TextSource, + start_offset: number, + prefix: UriPrefix, + end_offset: number, +): boolean { + if (!isBareAutolinkSchemePlausible(source, start_offset, prefix.scheme_end)) { + return false; + } + + if (prefix.has_authority) { + return true; + } + + return hasBareOpaqueUriEvidence(source, prefix.payload_start, end_offset); +} + +function isBareAutolinkSchemePlausible(source: TextSource, start_offset: number, end_offset: number): boolean { + let has_separator = false; + + for (let cursor = start_offset; cursor < end_offset; cursor++) { + const code = source.charCodeAt(cursor); + if (code === CC_PLUS || code === CC_DASH || code === CC_PERIOD) { + has_separator = true; + break; + } + } + + if (has_separator) { + return true; + } + + return end_offset - start_offset <= 7; +} + +function hasBareOpaqueUriEvidence(source: TextSource, start_offset: number, end_offset: number): boolean { + let saw_strong_signal = false; + let structural_signal_count = 0; + let colon_count = 0; + let saw_digit = false; + + for (let cursor = start_offset; cursor < end_offset; cursor++) { + const code = source.charCodeAt(cursor); + + if (isAsciiDigit(code)) { + saw_digit = true; + continue; + } + + if (isStrongOpaqueUriSignal(code)) { + saw_strong_signal = true; + continue; + } + + if (code === CC_COLON) { + colon_count++; + structural_signal_count++; + continue; + } + + if (code === CC_EQUALS || code === CC_SEMICOLON || code === CC_COMMA || code === CC_AMP) { + structural_signal_count++; + continue; + } + + if (code === CC_PLUS && cursor === start_offset && cursor + 1 < end_offset) { + if (isAsciiDigit(source.charCodeAt(cursor + 1))) { + return true; + } + structural_signal_count++; + } + } + + if (saw_strong_signal) { + return true; + } + + if (colon_count > 0 && saw_digit) { + return true; + } + + return structural_signal_count >= 2; +} + +function isStrongOpaqueUriSignal(code: number): boolean { + return code === CC_AT || + code === CC_SLASH || + code === CC_HASH || + code === CC_QUESTION || + code === CC_PERCENT; +} + +function trimBareUrlTrailingPunctuation( + source: TextSource, + start_offset: number, + end_offset: number, +): number { + let cursor = end_offset; + + while (cursor > start_offset) { + const code = source.charCodeAt(cursor - 1); + + if ( + code === CC_PERIOD || + code === CC_COMMA || + code === CC_SEMICOLON || + code === CC_COLON || + code === CC_BANG || + code === CC_QUESTION + ) { + cursor--; + continue; + } + + if (code === CC_CLOSE_PAREN && hasUnmatchedTrailingCloseParen(source, start_offset, cursor)) { + cursor--; + continue; + } + + break; + } + + return cursor; +} + +function hasUnmatchedTrailingCloseParen( + source: TextSource, + start_offset: number, + end_offset: number, +): boolean { + let open_count = 0; + let close_count = 0; + + for (let cursor = start_offset; cursor < end_offset; cursor++) { + const code = source.charCodeAt(cursor); + if (code === CC_OPEN_PAREN) open_count++; + if (code === CC_CLOSE_PAREN) close_count++; + } + + return close_count > open_count; +} + +/** Whether a code point is an ASCII letter. */ +function isAsciiLetter(code: number): boolean { + return (code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a); +} + +/** Whether a code point is an ASCII digit. */ +function isAsciiDigit(code: number): boolean { + return code >= 0x30 && code <= 0x39; +} + +/** Whether a code point is ASCII alphanumeric. */ +function isAsciiAlphanumeric(code: number): boolean { + return isAsciiLetter(code) || isAsciiDigit(code); +} + +/** Whether a code point is a hexadecimal digit. */ +function isHexDigit(code: number): boolean { + return isAsciiDigit(code) || + (code >= 0x41 && code <= 0x46) || + (code >= 0x61 && code <= 0x66); +} + +function isTagNameStart(code: number): boolean { + return isAsciiLetter(code); +} + +function isTagNameChar(code: number): boolean { + return isAsciiAlphanumeric(code) || code === CC_DASH || code === CC_COLON; +} + +function isAttrNameChar(code: number): boolean { + return isAsciiAlphanumeric(code) || code === CC_DASH || code === CC_COLON || code === CC_UNDERSCORE; +} \ No newline at end of file diff --git a/experiments/event-shape-study/lazy-position/code/mod.ts b/experiments/event-shape-study/lazy-position/code/mod.ts new file mode 100644 index 0000000..88f5f6d --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/mod.ts @@ -0,0 +1,62 @@ +/** + * Public entry point for `@okikio/wikitext`. + * + * This package is built as an event-stream-first wikitext parser. The raw + * token stream is the cheapest layer. The event stream adds structure. The + * tree model gives callers a normal nested object graph when they want one. + * + * This file simply re-exports the public surface so callers can import from one + * place instead of remembering each source file. + * + * Today that public surface includes: + * + * - `TextSource` for the parser's input shape + * - `Token` and `TokenType` for raw tokenizer output + * - `WikitextEvent` and related helpers for the event stream + * - `WikistNode` types, type guards, and builders for the tree model + * - `tokenize()` for raw scanning + * - `blockEvents()` for block-level structure + * - `inlineEvents()` for inline event enrichment + * - `tokens()`, `outlineEvents()`, `events()`, `parse()`, + * `parseWithDiagnostics()`, `parseStrictWithDiagnostics()`, and + * `parseWithRecovery()` for + * orchestration + * - `analyze()` and `materialize()` for the findings-first lane that + * separates parser facts from tree materialization policy + * - `buildTree()`, `buildTreeWithDiagnostics()`, `buildTreeStrict()`, and + * `buildTreeWithRecovery()` for AST materialization from an event stream plus + * source + * - `TreeMaterializationPolicy`, `DiagnosticCode`, and related result types + * for stable parser-owned vocabularies and tree/diagnostic result shapes + * - `visit()`, `filter()`, `resolveTreePath()`, `resolveDiagnosticAnchor()`, + * and `locateDiagnostic()` helpers for common tree and event queries + * - `createSession()` for cached repeated access to one source input, + * including `session.parseWithDiagnostics()`, + * `session.parseStrictWithDiagnostics()`, + * `session.parseWithRecovery()`, `session.analyze()`, and + * `session.materialize()` + * + * As more features land, this entry point is where they will be re-exported. + * + * @example Importing the current public API + * ```ts + * import type { TextSource, Token, WikitextEvent, WikistNode } from '@okikio/wikitext'; + * import { TokenType, tokenize, blockEvents } from '@okikio/wikitext'; + * ``` + * + * @module + */ + +// Re-export the public surface exactly as defined in each module. +// There is no wrapper layer here. +export * from './text_source.ts'; +export * from './token.ts'; +export * from './events.ts'; +export * from './ast.ts'; +export * from './tokenizer.ts'; +export * from './block_parser.ts'; +export * from './inline_parser.ts'; +export * from './tree_builder.ts'; +export * from './parse.ts'; +export * from './filter.ts'; +export * from './session.ts'; diff --git a/experiments/event-shape-study/lazy-position/code/parse.ts b/experiments/event-shape-study/lazy-position/code/parse.ts new file mode 100644 index 0000000..fbcb310 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/parse.ts @@ -0,0 +1,604 @@ +/** + * Public orchestration helpers for the parser pipeline. + * + * The lower-level modules already expose each pipeline stage separately: + * tokenizer, block parser, and inline parser. This file adds the sync pull API + * that most callers want when they do not need to wire stages together by + * hand. + * + * The functions intentionally mirror the pipeline layers: + * + * ```text + * tokens(source) -> raw tokenizer output + * outlineEvents(source) -> block structure only + * events(source) -> block + inline event stream + * parse(source) -> default tree + * parseWithDiagnostics(source) -> default tree + diagnostics + * parseStrictWithDiagnostics(source) -> conservative tree + diagnostics + * parseWithRecovery(source) -> default tree + recovered + diagnostics + * analyze(source) -> replayable findings (events + diagnostics + + * recovery list) + * materialize(findings) -> tree + diagnostics for one policy choice + * ``` + * + * The key split is now diagnostic emission first, then materialization policy. + * If a caller does not want diagnostics, the block and inline stages should + * not emit diagnostic events for that lane. + * + * The event stream itself stays policy-neutral. The default tree family and + * the conservative tree are both materializations of the same parser findings. + * + * @example Walking the full event stream + * ```ts + * import { events } from './parse.ts'; + * + * for (const event of events("A [[Page|link]]")) { + * console.log(event.kind); + * } + * ``` + * + * @module + */ + +import type { Token } from './token.ts'; +import type { WikitextEvent } from './events.ts'; +import type { TextSource } from './text_source.ts'; +import type { WikistNodeType, WikistRoot } from './ast.ts'; +import type { + ParseDiagnostic, + ParseDiagnosticAnchor, + ParseDiagnosticsResult, + ParseResult, + TreeMaterializationPolicy, +} from './tree_builder.ts'; +import type { Position } from './events.ts'; +import { DiagnosticCode, type KnownDiagnosticCode } from './events.ts'; + +import { tokenize } from './tokenizer.ts'; +import { blockEvents } from './block_parser.ts'; +import { inlineEvents } from './inline_parser.ts'; +import { + buildTree, + buildTreeStrict, + buildTreeWithDiagnostics, + buildTreeWithRecovery, + TreeMaterializationPolicy as TreeMaterializationPolicyMap, +} from './tree_builder.ts'; + +/** + * Public switches for event-stream production. + * + * The main cost choice here is whether parser diagnostics should be emitted at + * all. If `diagnostics` is omitted or `false`, the block and inline + * stages stay on the cheapest event lane and do not emit `error` events. + */ +export interface EventOptions { + /** Whether block and inline stages should emit diagnostic events. */ + readonly diagnostics?: boolean; +} + +/** + * Internal event-pipeline switches used by the tree wrappers in this module. + * + * The parser exposes one shared event pipeline and several tree lanes layered + * on top of it. + * + * ```text + * parse() -> default tree, no diagnostics + * parseWithDiagnostics() -> default tree + diagnostics + * parseStrictWithDiagnostics() + * -> conservative tree + diagnostics + * parseWithRecovery() -> default tree + diagnostics + recovered summary + * ``` + * + * These options are the plumbing that keeps those lanes honest. Without them, + * the public API would expose different result shapes while still paying for + * the same underlying work. + * + * @internal + */ +export interface EventPipelineOptions { + /** Whether block and inline stages should emit parser diagnostics. */ + readonly diagnostics?: boolean; +} + +/** + * Yield the raw token stream for one source input. + * + * This is the cheapest parser entry point. It is useful for search, grep-like + * tooling, and low-level diagnostics that do not need structural nesting. + */ +export function tokens(source: TextSource): Generator { + return tokenize(source); +} + +/** + * Yield block-level events only. + * + * Inline content remains plain text ranges. This is the cheap structural mode + * for outlines, table-of-contents extraction, and other block-focused tools. + */ +export function outlineEvents( + source: TextSource, + options: EventOptions = {}, +): Generator { + return outlineEventsWithOptions(source, options); +} + +/** + * Yield the full event stream for one source input. + * + * This is the default event-level API. It runs the tokenizer, block parser, + * and inline enrichment in order. + */ +export function events( + source: TextSource, + options: EventOptions = {}, +): Generator { + return eventsWithOptions(source, { + diagnostics: options.diagnostics, + }); +} + +/** + * @internal + * Build the block-only event stream for one diagnostics choice. + */ +export function outlineEventsWithOptions( + source: TextSource, + options: EventPipelineOptions, +): Generator { + return blockEvents(source, tokenize(source), { + diagnostics: options.diagnostics, + }); +} + +/** + * Build the full event stream for one diagnostics choice. + * + * This helper keeps the public wrappers small and makes the block/inline split + * explicit: first get block structure, then enrich it with inline markup. + * Materialization policy is intentionally not part of this step. + * + * @internal + */ +export function eventsWithOptions( + source: TextSource, + options: EventPipelineOptions, +): Generator { + return eventsFromOutline( + source, + outlineEventsWithOptions(source, options), + options, + ); +} + +/** + * Rebuild the full event stream from a previously computed outline stream. + * + * This helper lets higher-level wrappers such as `Session` reuse a cached + * outline stream instead of rerunning the block parser just to reach the full + * event stream. + * + * @internal + */ +export function eventsFromOutline( + source: TextSource, + outline: Iterable, + options: EventPipelineOptions, +): Generator { + return inlineEvents(source, outline, { + diagnostics: options.diagnostics, + }); +} + +/** + * Parse source text into a wikist tree. + * + * This is the convenience API for callers that want a full AST and do not need + * to inspect the intermediate event stream themselves. + * + * It is also the cheapest tree-building lane. It does not request diagnostics + * from the block or inline stages, and it keeps the default tolerant + * HTML-like tree shape when malformed input is encountered. + * + * If the caller also needs diagnostics or explicit recovery metadata, use + * {@linkcode parseWithDiagnostics} or {@linkcode parseWithRecovery} instead. + */ +export function parse(source: TextSource): WikistRoot { + return buildTree(eventsWithOptions(source, { + diagnostics: false, + }), { source }); +} + +/** + * Parse source text into the default wikist tree and keep diagnostics. + * + * This is the diagnostics-first entry point. It preserves the same default + * HTML-like tree shape as {@linkcode parse}, but also returns the diagnostics + * that describe malformed input and parser continuation points. + */ +export function parseWithDiagnostics(source: TextSource): ParseDiagnosticsResult { + return buildTreeWithDiagnostics(eventsWithOptions(source, { + diagnostics: true, + }), { source }); +} + +/** + * Parse source text into a conservative tree and keep diagnostics. + * + * This is the source-strict materialization lane. It uses the same parser + * findings as {@linkcode parseWithDiagnostics}, but it collapses recovery-heavy + * wrappers back to plain text during tree materialization when the source never + * clearly committed to them. + */ +export function parseStrictWithDiagnostics(source: TextSource): ParseDiagnosticsResult { + return buildTreeStrict(eventsWithOptions(source, { + diagnostics: true, + }), { source }); +} + +/** + * Parse source text into a wikist tree and report whether recovery happened. + * + * This is the explicit recovery-aware entry point. It returns the same + * default tree as {@linkcode parse}, plus a `recovered` flag and the + * diagnostics that explain what the parser had to do on the caller's behalf. + * + * Read the result like two coordinated lanes: + * + * ```text + * source + * ├─► parse() -> tree only + * ├─► parseWithDiagnostics() -> tree + diagnostics + * ├─► parseStrictWithDiagnostics() + * │ -> conservative tree + diagnostics + * └─► parseWithRecovery() -> tree + recovered + diagnostics + * ``` + * + * The important distinction from `parseWithDiagnostics()` is not just the + * extra boolean. This lane adds an explicit summary field for consumers that + * want the parser's tolerant default behavior to stay visible in control flow. + * + * The diagnostics include a narrow `anchor` so downstream tools can resolve + * the nearest node around the recovery point. + * + * Today those diagnostics mostly come from block-parser findings and + * tree-builder continuation steps. `parse()` intentionally drops them, + * `parseWithDiagnostics()` preserves them with the default tree, and + * `parseWithRecovery()` adds the explicit `recovered` summary. + * + * That anchor is intentionally tree-only today. Edit-stable anchor semantics + * belong to later session/edit tracking work and are not part of this public + * API yet. + */ +export function parseWithRecovery(source: TextSource): ParseResult { + return buildTreeWithRecovery(eventsWithOptions(source, { + diagnostics: true, + }), { source }); +} + +// --------------------------------------------------------------------------- +// Findings-first lane: analyze() + materialize() +// --------------------------------------------------------------------------- +// +// The tree-first wrappers above bake one materialization policy into each +// result. That is fine for most consumers, but some tools want to see the +// parser's findings first and then decide how to turn them into a tree (or +// decide whether to build a tree at all). +// +// ```text +// analyze(source) collect events + diagnostics + recovery list +// │ +// ├─► materialize(findings) default-html-like tree +// └─► materialize(findings, { policy }) pick a policy per call +// ``` +// +// Findings are replayable on purpose. A caller can ask the same findings +// object for more than one materialization without reparsing the source. + +/** + * Structured recovery classes the parser currently knows how to describe. + * + * This vocabulary is intentionally small. Each kind names one decision the + * parser had to make while continuing through malformed input, so later + * tooling can inspect recovery without matching on long human-readable + * messages. + * + * The kinds are: + * + * - `missing-close`: an inline opener was complete but its matching close + * never arrived before the enclosing text range ended. + * - `unterminated-opener`: an inline opener started but never reached its + * closing `>`. + * - `unclosed-table`: a block-level table opened but never closed. + * - `mismatched-exit`: an `exit` event referenced a node that was not the + * innermost open frame, so the tree builder auto-closed one or more inner + * frames before honoring it. + * - `orphan-exit`: an `exit` event referenced no currently open frame. + * - `eof-autoclose`: the event stream ended while one or more frames were + * still open. + */ +export type ParseRecoveryKind = + | 'missing-close' + | 'unterminated-opener' + | 'unclosed-table' + | 'mismatched-exit' + | 'orphan-exit' + | 'eof-autoclose'; + +/** + * One structural recovery decision the parser made while analyzing the source. + * + * A `ParseRecovery` is the narrower, taxonomy-oriented cousin of + * {@linkcode ParseDiagnostic}. Diagnostics carry a human-readable message and + * are useful for logs and editor hints. Recovery entries are the replayable + * decisions those diagnostics describe: what kind of malformed region was + * encountered, where it lives, and which materialization policies can change + * how it ends up in a final tree. + * + * `policies` is the list of package-owned materialization policies that + * produce a distinct outcome for this recovery. When both public policies + * would render the region the same way (for example, an unterminated opener + * stays as text under either policy), only `DEFAULT_HTML_LIKE` is listed. + */ +export interface ParseRecovery { + /** Classifier for this recovery. */ + readonly kind: ParseRecoveryKind; + /** Underlying diagnostic code that triggered the recovery. */ + readonly code: KnownDiagnosticCode | string; + /** Source position where the recovery was recorded. */ + readonly position: Position; + /** Tree-path anchor resolved against the default materialization. */ + readonly anchor: ParseDiagnosticAnchor; + /** Node type involved in the recovery, when the parser knows it. */ + readonly node_type?: WikistNodeType; + /** Package-owned materialization policies that can change the final shape. */ + readonly policies: readonly TreeMaterializationPolicy[]; +} + +/** + * Options for {@linkcode analyze}. + * + * Recovery-list construction is cheap, but it is still opt-out for callers + * that only want events and diagnostics. + */ +export interface AnalyzeOptions { + /** + * Whether to include the derived `recovery` array on the returned findings. + * + * Defaults to `true`. Set to `false` when only events and diagnostics are + * needed. + */ + readonly recovery?: boolean; +} + +/** + * Replayable parser findings for one source input. + * + * `ParseFindings` is the public shape the findings-first lane returns. It is + * intentionally narrow: + * + * - `events` is a fully collected array, so downstream tools can replay the + * stream more than once without reparsing the source + * - `diagnostics` are preserved in the shape downstream tools already + * understand, including tree-anchor metadata + * - `recovery` lists the structural decisions the parser had to make, when + * {@linkcode AnalyzeOptions.recovery} is not turned off + * + * The findings object does not include a tree on purpose. A caller chooses + * when (and whether) to materialize one by passing the findings to + * {@linkcode materialize}. + */ +export interface ParseFindings { + /** Original source text backing the findings. */ + readonly source: TextSource; + /** Collected event stream, ready to replay. */ + readonly events: readonly WikitextEvent[]; + /** Diagnostics discovered while analyzing the source. */ + readonly diagnostics: readonly ParseDiagnostic[]; + /** Structural recovery decisions, when requested. */ + readonly recovery?: readonly ParseRecovery[]; +} + +/** + * Options for {@linkcode materialize}. + * + * The policy selection here mirrors the wrappers exposed by + * {@linkcode parseWithDiagnostics} and {@linkcode parseStrictWithDiagnostics}, + * but the caller stays in control of when materialization happens. + */ +export interface MaterializeOptions { + /** + * Tree-shaping policy for this materialization. + * + * Defaults to `TreeMaterializationPolicy.DEFAULT_HTML_LIKE`. + */ + readonly policy?: TreeMaterializationPolicy; +} + +/** + * Result of one {@linkcode materialize} call. + * + * This is the same shape as {@linkcode ParseDiagnosticsResult}. It is given a + * dedicated name here so the findings-first lane reads cleanly: findings go + * in, a tree plus diagnostics come out. + */ +export interface ParseOutput { + /** Materialized wikist tree for the requested policy. */ + readonly tree: WikistRoot; + /** Diagnostics produced by this materialization. */ + readonly diagnostics: readonly ParseDiagnostic[]; +} + +/** + * Analyze source text into replayable parser findings. + * + * This is the findings-first lane. It runs the event pipeline with + * diagnostics enabled, collects the events into an array so they can be + * replayed, and summarizes the parser's recovery decisions. + * + * Use this when a caller wants to inspect what the parser found before + * deciding whether (or how) to materialize a tree. A common pattern is to + * analyze once and materialize several times with different policies: + * + * ```ts + * const findings = analyze(source); + * + * if (findings.recovery?.length) { + * // Show diagnostics, or collapse recovery-heavy regions. + * const strict = materialize(findings, { + * policy: TreeMaterializationPolicy.SOURCE_STRICT, + * }); + * } + * + * const tolerant = materialize(findings); + * ``` + * + * Diagnostics in the returned findings are computed against the default + * tolerant tree shape, so their `anchor` paths resolve against a + * `DEFAULT_HTML_LIKE` materialization. Calling {@linkcode materialize} with + * `SOURCE_STRICT` returns its own diagnostics with anchors retargeted to the + * conservative tree. + */ +export function analyze( + source: TextSource, + options: AnalyzeOptions = {}, +): ParseFindings { + const events = Array.from(eventsWithOptions(source, { diagnostics: true })); + const { diagnostics } = buildTreeWithDiagnostics(events, { source }); + + if (options.recovery === false) { + return { + source, + events, + diagnostics, + }; + } + + return { + source, + events, + diagnostics, + recovery: recoveriesFromDiagnostics(diagnostics), + }; +} + +/** + * Materialize a wikist tree from previously analyzed findings. + * + * This is the consumer side of the findings-first lane. It takes an existing + * {@linkcode ParseFindings} object and builds a tree under the requested + * materialization policy. The findings can be replayed more than once, which + * means a caller can materialize the same parse under different policies + * without repeating tokenizer or block-parser work. + * + * When the policy is omitted, the default tolerant HTML-like materialization + * is used, so this call produces the same tree as {@linkcode parseWithDiagnostics} + * would for the same source. + */ +export function materialize( + findings: ParseFindings, + options: MaterializeOptions = {}, +): ParseOutput { + const policy = options.policy ?? TreeMaterializationPolicyMap.DEFAULT_HTML_LIKE; + + if (policy === TreeMaterializationPolicyMap.SOURCE_STRICT) { + return buildTreeStrict(findings.events, { source: findings.source }); + } + + return buildTreeWithDiagnostics(findings.events, { source: findings.source }); +} + +/** + * Derive a structured recovery list from a set of diagnostics. + * + * Only diagnostics that describe one of the parser's known structural + * recoveries map to a {@linkcode ParseRecovery} entry. Other diagnostics + * (including non-recoverable ones or future codes without a recovery + * classification) are skipped so the recovery vocabulary stays narrow. + * + * @internal Exposed so `Session` can derive recovery entries from its + * already-cached diagnostics without a redundant tree walk. + */ +export function recoveriesFromDiagnostics( + diagnostics: readonly ParseDiagnostic[], +): readonly ParseRecovery[] { + const recoveries: ParseRecovery[] = []; + + for (const diagnostic of diagnostics) { + const kind = recoveryKindForCode(diagnostic.code); + if (kind === undefined) continue; + + const entry: ParseRecovery = { + kind, + code: diagnostic.code ?? '', + position: diagnostic.position, + anchor: diagnostic.anchor, + policies: recoveryPoliciesForCode(diagnostic.code), + ...(diagnostic.anchor.node_type !== 'root' + ? { node_type: diagnostic.anchor.node_type } + : {}), + }; + + recoveries.push(entry); + } + + return recoveries; +} + +/** + * Map a diagnostic code to its structural recovery classifier. + * + * Returns `undefined` for diagnostics that do not describe a tree-shape + * decision (for example, future non-structural diagnostic codes), so the + * recovery list only contains entries that tools can act on. + */ +function recoveryKindForCode( + code: KnownDiagnosticCode | string | undefined, +): ParseRecoveryKind | undefined { + switch (code) { + case DiagnosticCode.INLINE_TAG_MISSING_CLOSE: + return 'missing-close'; + case DiagnosticCode.INLINE_TAG_UNTERMINATED_OPENER: + return 'unterminated-opener'; + case DiagnosticCode.UNCLOSED_TABLE: + return 'unclosed-table'; + case DiagnosticCode.TREE_MISMATCHED_EXIT: + return 'mismatched-exit'; + case DiagnosticCode.TREE_ORPHAN_EXIT: + return 'orphan-exit'; + case DiagnosticCode.TREE_EOF_AUTOCLOSE: + return 'eof-autoclose'; + default: + return undefined; + } +} + +/** + * List the materialization policies that can change the final shape for one + * diagnostic code. + * + * For recovery-heavy wrappers that source-strict materialization collapses + * back to text (for example, `INLINE_TAG_MISSING_CLOSE` or `UNCLOSED_TABLE`), + * both public policies are listed because switching policy produces a + * different tree. For codes where both policies produce the same outcome, + * only the default policy is listed to make it obvious that policy choice is + * not meaningful here. + */ +function recoveryPoliciesForCode( + code: KnownDiagnosticCode | string | undefined, +): readonly TreeMaterializationPolicy[] { + switch (code) { + case DiagnosticCode.INLINE_TAG_MISSING_CLOSE: + case DiagnosticCode.UNCLOSED_TABLE: + case DiagnosticCode.TREE_MISMATCHED_EXIT: + case DiagnosticCode.TREE_EOF_AUTOCLOSE: + return [ + TreeMaterializationPolicyMap.DEFAULT_HTML_LIKE, + TreeMaterializationPolicyMap.SOURCE_STRICT, + ]; + default: + return [TreeMaterializationPolicyMap.DEFAULT_HTML_LIKE]; + } +} \ No newline at end of file diff --git a/experiments/event-shape-study/lazy-position/code/session.ts b/experiments/event-shape-study/lazy-position/code/session.ts new file mode 100644 index 0000000..c7b47e4 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/session.ts @@ -0,0 +1,487 @@ +/** + * Basic stateful wrapper around the stateless parser pipeline. + * + * Most parser entry points stay stateless on purpose. They are easy to reason + * about and easy to test. Some callers, especially editors and repeated-query + * tooling, still want one object they can hold onto and ask for the outline, + * full events, or full tree without recomputing each layer every time. + * + * Phase 5 keeps this wrapper intentionally small: + * + * ```text + * createSession(source) + * ├─► session.outline() -> cached block events + * ├─► session.events() -> cached full events + * ├─► session.parse() -> cached default tree + * ├─► session.parseWithDiagnostics() -> cached default tree + diagnostics + * ├─► session.parseStrictWithDiagnostics() + * │ -> cached conservative tree + diagnostics + * └─► session.parseWithRecovery() -> cached default tree + recovery summary + * ``` + * + * Streaming writes and incremental edits belong to later phases. This file is + * only the basic cached wrapper over the existing sync pipeline. + * + * @module + */ + +import type { TextSource } from './text_source.ts'; +import type { WikitextEvent } from './events.ts'; +import type { WikistRoot } from './ast.ts'; +import type { ParseDiagnosticsResult, ParseResult } from './tree_builder.ts'; +import type { + AnalyzeOptions, + MaterializeOptions, + ParseFindings, + ParseOutput, +} from './parse.ts'; + +import { blockEvents } from './block_parser.ts'; +import { tokenize } from './tokenizer.ts'; +import { buildTree, buildTreeStrict, buildTreeWithDiagnostics, buildTreeWithRecovery } from './tree_builder.ts'; +import { eventsFromOutline, recoveriesFromDiagnostics } from './parse.ts'; + +/** + * Public switches for cached event-stream access. + * + * Sessions keep separate caches for diagnostics-off and diagnostics-on event + * lanes. That lets callers stay on the cheapest stream path unless they + * explicitly opt into diagnostics. + */ +export interface SessionStreamOptions { + /** Whether the cached event lane should preserve parser diagnostics. */ + readonly diagnostics?: boolean; +} + +/** + * Cache-lane selector for the session wrapper. + * + * Sessions are not just memoizing one monolithic parse result. They keep + * separate caches for the cheap tree-only lane and the diagnostics-enabled + * lanes so callers do not accidentally pay for diagnostics they never asked + * for. + * + * @internal + */ +export interface SessionEventOptions { + /** Whether this lane wants event-level parser diagnostics preserved. */ + readonly diagnostics: boolean; +} + +/** + * Basic stateful parser session. + * + * This interface is intentionally small. It is a cache wrapper around the + * existing sync pipeline, not a long-lived mutable document model yet. + * + * A useful way to read it is as one cached pipeline with several result lanes: + * + * - `outline()` caches block structure + * - `events()` reuses the outline cache and adds inline structure + * - `parse()` reuses the full event cache and materializes the default tree + * - `parseWithDiagnostics()` preserves diagnostics alongside that same tree + * - `parseStrictWithDiagnostics()` materializes the conservative source-strict tree + * - `parseWithRecovery()` keeps the default tree and adds an explicit boolean + * summary on top of its diagnostics + * + * The important design rule is that these are not separate parsers. They are + * different materializations and summary shapes built from the same cached + * outline and event work. + */ +export interface Session { + /** Original source text backing this session. */ + readonly source: TextSource; + + /** + * Return the cached block-only event stream. + * + * Call this when you only need document structure such as headings, lists, + * tables, or paragraphs. It is the cheapest structured cache in the session. + */ + outline(options?: SessionStreamOptions): Generator; + + /** + * Return the cached full event stream. + * + * This adds inline markup on top of the cached outline stage. Repeated calls + * should not rerun block parsing for the same source. + */ + events(options?: SessionStreamOptions): Generator; + + /** + * Return the cached parsed tree. + * + * This is the tree-only lane. If the caller also needs parser diagnostics or + * explicit recovery metadata, use {@linkcode parseWithDiagnostics} or + * {@linkcode parseWithRecovery}. + */ + parse(): WikistRoot; + + /** + * Return the cached parsed tree plus diagnostics. + * + * This is the diagnostics-first lane. It preserves diagnostics alongside the + * same default tree shape returned by {@linkcode parse}. + */ + parseWithDiagnostics(): ParseDiagnosticsResult; + + /** + * Return the cached conservative tree-plus-diagnostics result. + * + * This lane uses the source-strict materialization policy. Recovery-heavy + * wrappers are more likely to collapse back to plain text when the source did + * not clearly commit to them. + */ + parseStrictWithDiagnostics(): ParseDiagnosticsResult; + + /** + * Return the cached parsed tree plus explicit recovery metadata. + * + * This is the recovery-aware lane for consumers that want the parser's + * tolerant default behavior to stay explicit in their own control flow. + */ + parseWithRecovery(): ParseResult; + + /** + * Return the cached findings-first result. + * + * This is the same shape as top-level {@linkcode analyze}, but the session + * remembers the parsed events so repeated calls do not reparse. When + * `options.recovery` is `false`, the cached recovery list is dropped from + * the returned findings so callers only pay for the metadata they ask for. + */ + analyze(options?: AnalyzeOptions): ParseFindings; + + /** + * Materialize a tree from cached findings. + * + * This is the session-friendly equivalent of top-level + * {@linkcode materialize}. The session reuses whichever tree cache already + * exists for the requested policy, so calling this repeatedly with the same + * policy does not rebuild the tree. + */ + materialize(options?: MaterializeOptions): ParseOutput; +} + +/** + * Concrete session implementation for one immutable source input. + * + * The caches are layered, but lane-aware rather than fully shared: + * + * ```text + * diagnostics outline + diagnostics events -> diagnostics, conservative, and recovery results + * cheap tree-only or reusable default events cache -> parse() + * ``` + * + * That shape matters because it keeps `parse()` cheap when the caller does not + * want diagnostics, while still reusing the more expensive diagnostics-aware + * caches if some other consumer path already paid for them. + * + * Read the cache graph like this: + * + * - default outline and default events back the cheapest no-diagnostics lane + * - diagnostics-enabled events back `parseWithDiagnostics()`, + * `parseWithRecovery()`, and `parseStrictWithDiagnostics()` + * - tree-level caches reuse whichever tree lane already exists so one caller + * does not repay the same materialization cost twice + * + * @internal + */ +export class BasicSession implements Session { + readonly source: TextSource; + #outline_cache?: WikitextEvent[]; + #diagnostic_outline_cache?: WikitextEvent[]; + #event_cache?: WikitextEvent[]; + #diagnostic_event_cache?: WikitextEvent[]; + #tree_cache?: WikistRoot; + #diagnostics_cache?: ParseDiagnosticsResult; + #conservative_cache?: ParseDiagnosticsResult; + #recovery_cache?: ParseResult; + #findings_cache?: ParseFindings; + + constructor(source: TextSource) { + this.source = source; + } + + *outline(options: SessionStreamOptions = {}): Generator { + yield* this.getOutlineCacheWithOptions({ + diagnostics: options.diagnostics === true, + }); + } + + *events(options: SessionStreamOptions = {}): Generator { + yield* this.getEventsCache({ + diagnostics: options.diagnostics === true, + }); + } + + /** + * Materialize the cached tree-only result. + * + * If recovery was requested first, this reuses that already-materialized + * default tree directly. Otherwise it builds from the cheap no-diagnostics + * event lane. + * + * That split is deliberate. `parse()` is the "give me a usable tree and keep + * overhead down" API, so it should not silently populate the more expensive + * diagnostics-enabled caches unless some other consumer path already did that + * work. + */ + parse(): WikistRoot { + if (this.#tree_cache === undefined) { + if (this.#recovery_cache !== undefined) { + this.#tree_cache = this.#recovery_cache.tree; + } else if (this.#diagnostics_cache !== undefined) { + this.#tree_cache = this.#diagnostics_cache.tree; + } else { + this.#tree_cache = buildTree(this.getEventsCache({ + diagnostics: false, + }), { source: this.source }); + } + } + + return this.#tree_cache; + } + + /** + * Materialize the cached tree-plus-diagnostics result. + * + * This lane caches separately because callers may ask for diagnostics before + * any other tree result, but it preserves the same default tree shape as + * {@linkcode parse}. + */ + parseWithDiagnostics(): ParseDiagnosticsResult { + if (this.#diagnostics_cache === undefined) { + if (this.#recovery_cache !== undefined) { + this.#diagnostics_cache = { + tree: this.#recovery_cache.tree, + diagnostics: this.#recovery_cache.diagnostics, + }; + + return this.#diagnostics_cache; + } + + const result = buildTreeWithDiagnostics(this.getEventsCache({ + diagnostics: true, + }), { + source: this.source, + }); + + if (this.#tree_cache !== undefined) { + this.#diagnostics_cache = { + tree: this.#tree_cache, + diagnostics: result.diagnostics, + }; + } else { + this.#diagnostics_cache = result; + } + } + + return this.#diagnostics_cache; + } + + /** + * Materialize the cached conservative tree-plus-diagnostics result. + * + * This lane uses the same diagnostics-enabled event findings as + * {@linkcode parseWithDiagnostics}. Only the final tree materialization is + * more conservative. + */ + parseStrictWithDiagnostics(): ParseDiagnosticsResult { + if (this.#conservative_cache === undefined) { + this.#conservative_cache = buildTreeStrict(this.getEventsCache({ + diagnostics: true, + }), { + source: this.source, + }); + } + + return this.#conservative_cache; + } + + /** + * Materialize the cached tree-plus-recovery result. + * + * This lane shares the same default recovered tree as + * {@linkcode parseWithDiagnostics}. Its only extra field is the `recovered` + * summary boolean. That means it can reuse the diagnostics cache directly + * when the caller already asked for diagnostics first. + */ + parseWithRecovery(): ParseResult { + if (this.#recovery_cache === undefined) { + if (this.#diagnostics_cache !== undefined) { + this.#recovery_cache = { + tree: this.#diagnostics_cache.tree, + diagnostics: this.#diagnostics_cache.diagnostics, + recovered: this.#diagnostics_cache.diagnostics.length > 0, + }; + this.#tree_cache = this.#diagnostics_cache.tree; + + return this.#recovery_cache; + } + + const result = buildTreeWithRecovery(this.getEventsCache({ + diagnostics: true, + }), { + source: this.source, + }); + + if (this.#tree_cache !== undefined) { + this.#recovery_cache = { + tree: this.#tree_cache, + recovered: result.recovered, + diagnostics: result.diagnostics, + }; + } else { + this.#recovery_cache = result; + this.#tree_cache = result.tree; + } + } + + return this.#recovery_cache; + } + + /** + * Return the cached findings-first result. + * + * The findings are built from the cached diagnostics-enabled event lane and + * the cached diagnostics tree, so repeated calls do not reparse the source + * or recompute diagnostics. When `options.recovery` is `false`, the recovery + * list is stripped from the returned findings on each call. + */ + analyze(options: AnalyzeOptions = {}): ParseFindings { + if (this.#findings_cache === undefined) { + const diagnostics_result = this.parseWithDiagnostics(); + const events = this.getEventsCache({ diagnostics: true }); + this.#findings_cache = { + source: this.source, + events, + diagnostics: diagnostics_result.diagnostics, + recovery: recoveriesFromDiagnostics(diagnostics_result.diagnostics), + }; + } + + if (options.recovery === false) { + return { + source: this.#findings_cache.source, + events: this.#findings_cache.events, + diagnostics: this.#findings_cache.diagnostics, + }; + } + + return this.#findings_cache; + } + + /** + * Materialize a tree from the session's cached findings. + * + * Each policy has its own cache lane. Calling this repeatedly with the same + * policy is therefore a cache lookup, not a fresh tree build. Switching + * policies only pays for the extra materialization, not for tokenize or + * event-stream work. + */ + materialize(options: MaterializeOptions = {}): ParseOutput { + if (options.policy === 'source-strict') { + const conservative = this.parseStrictWithDiagnostics(); + return { + tree: conservative.tree, + diagnostics: conservative.diagnostics, + }; + } + + const diagnostics_result = this.parseWithDiagnostics(); + return { + tree: diagnostics_result.tree, + diagnostics: diagnostics_result.diagnostics, + }; + } + + /** + * Return the appropriate outline cache for one session lane. + * + * Read the branching rule like this: + * + * ```text + * diagnostics lane requested? + * yes -> use or build the diagnostics-enabled outline cache + * no -> prefer the cheap outline cache, but reuse the diagnostics cache if + * it already exists because that work has already been paid for + * ``` + */ + private getOutlineCacheWithOptions(options: SessionEventOptions): WikitextEvent[] { + if (options.diagnostics) { + if (this.#diagnostic_outline_cache === undefined) { + this.#diagnostic_outline_cache = Array.from(blockEvents(this.source, tokenize(this.source), { + diagnostics: true, + })); + } + + return this.#diagnostic_outline_cache; + } + + if (this.#outline_cache !== undefined) { + return this.#outline_cache; + } + + if (this.#diagnostic_outline_cache !== undefined) { + return this.#diagnostic_outline_cache; + } + + this.#outline_cache = Array.from( + blockEvents(this.source, tokenize(this.source), { + diagnostics: false, + }), + ); + + return this.#outline_cache; + } + + /** + * Return the appropriate full-event cache for one session lane. + * + * The same reuse rule as `getOutlineCacheWithOptions()` applies here. The + * session preserves the cheap diagnostics-off lane when possible, but it does + * not avoid reusing a more expensive cache once that cache already exists. + * Materialization policy is intentionally not part of this cache. + */ + private getEventsCache(options: SessionEventOptions): WikitextEvent[] { + if (options.diagnostics) { + if (this.#diagnostic_event_cache === undefined) { + this.#diagnostic_event_cache = Array.from(eventsFromOutline( + this.source, + this.getOutlineCacheWithOptions(options), + options, + )); + } + + return this.#diagnostic_event_cache; + } + + if (this.#event_cache !== undefined) { + return this.#event_cache; + } + + if (this.#diagnostic_event_cache !== undefined) { + return this.#diagnostic_event_cache; + } + + this.#event_cache = Array.from(eventsFromOutline( + this.source, + this.getOutlineCacheWithOptions(options), + options, + )); + + return this.#event_cache; + } +} + +/** + * Create a basic cached parser session. + * + * This is the entry point for repeated sync access to one immutable source. + * It is useful for tooling that wants to ask several questions about the same + * text without rebuilding every parser layer each time. + */ +export function createSession(source: TextSource): Session { + return new BasicSession(source); +} \ No newline at end of file diff --git a/experiments/event-shape-study/lazy-position/code/text_source.ts b/experiments/event-shape-study/lazy-position/code/text_source.ts new file mode 100644 index 0000000..92b5910 --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/text_source.ts @@ -0,0 +1,182 @@ +/** + * A small text interface for the whole parser pipeline. + * + * The parser reads source text one character at a time. In the common case, + * that source is just a normal JavaScript string. In an editor or live + * collaboration system, the source might live in a rope or CRDT buffer + * instead. This file gives the parser one small shape it can rely on, so the + * rest of the code does not care where the text came from. + * + * In practice that means all parser stages can work with the same input style: + * + * ``` + * plain string + * rope-backed editor buffer + * CRDT-backed document + * │ + * └── implements TextSource + * │ + * ▼ + * tokenizer -> events -> tree builder + * ``` + * + * A plain string already works with no wrapper because it already has the + * methods we need: `length`, `charCodeAt()`, and `slice()`. + * + * Why these three methods? + * + * - `length` tells the scanner when to stop. + * - `charCodeAt()` lets the tokenizer check one character at a time in its + * hottest loop without creating a new one-character string for every check. + * - `slice()` turns a stored range back into real text only when a later stage + * or consumer actually needs the text. + * + * That last point matters. Tokens and events mostly store start and end + * offsets, not copied strings. So instead of creating tiny strings all the + * time during scanning, the parser keeps cheap numeric ranges and resolves the + * real text on demand. + * + * ```ts + * const value = source.slice(token.start, token.end); + * ``` + * + * This keeps the hot path simpler and avoids extra allocation pressure while + * scanning large articles. + * + * @example Using a plain string directly + * ```ts + * import type { TextSource } from './text_source.ts'; + * + * const source: TextSource = '== Heading ==\nSome text.'; + * source.charCodeAt(0); // 61 + * source.slice(0, 13); // '== Heading ==' + * source.length; // 24 + * ``` + * + * @example Adapting a custom backing store + * ```ts + * import type { TextSource } from './text_source.ts'; + * + * class RopeSource implements TextSource { + * readonly length: number; + * + * constructor( + * private readonly rope: { + * charAt(i: number): string; + * toString(): string; + * length: number; + * }, + * ) { + * this.length = rope.length; + * } + * + * charCodeAt(index: number): number { + * return this.rope.charAt(index).charCodeAt(0); + * } + * + * slice(start: number, end: number): string { + * return this.rope.toString().slice(start, end); + * } + * } + * ``` + * + * @module + */ + +/** + * Minimal read-only text interface consumed by all parser pipeline stages. + * + * Any object that exposes `length`, `charCodeAt`, and `slice` with the same + * semantics as the built-in `String` prototype satisfies this interface. + * A plain `string` works out of the box: + * + * ```ts + * const src: TextSource = 'hello'; // ✓ no wrapper needed + * ``` + * + * `iterSlices` is an optional optimization hook for chunked consumers + * (e.g., streaming serializers that want to avoid concatenating the entire + * source into a single string). + */ +export interface TextSource { + /** Total length in UTF-16 code units. */ + readonly length: number; + + /** + * Return the UTF-16 character code at the given offset. + * + * Must behave identically to `String.prototype.charCodeAt`: return + * `NaN` for out-of-range indices. + * + * This is the single hottest method in the parser. The tokenizer's + * inner loop calls it on every character position. We use `charCodeAt` + * (not `charAt`) because comparing numeric codes avoids allocating a + * one-character string per comparison, which matters at scan speeds of + * millions of characters per second. + * + * @param index - Zero-based UTF-16 code unit offset. + */ + charCodeAt(index: number): number; + + /** + * Return the substring from `start` (inclusive) to `end` (exclusive), + * measured in UTF-16 code units. + * + * Must behave identically to `String.prototype.slice` for non-negative + * indices within bounds. + * + * @param start - Inclusive start offset. + * @param end - Exclusive end offset. + */ + slice(start: number, end: number): string; + + /** + * Optional: iterate sub-slices of the range `[start, end)` without + * concatenating into a single string first. Useful for chunked + * serialization or streaming output where the backing store is + * segmented (e.g., rope nodes, CRDT runs). + * + * When absent, consumers fall back to `slice(start, end)`. + * + * @param start - Inclusive start offset. + * @param end - Exclusive end offset. + */ + iterSlices?(start: number, end: number): Iterable; +} + +/** + * Resolve a range from a {@linkcode TextSource} into a plain string. + * + * Throughout the parser, tokens and events carry offset ranges (start/end + * integers) rather than extracted strings. This is a deliberate performance + * choice: it avoids allocating a new string for every token and sidesteps + * V8's sliced-string retention risk, where a small `.slice()` can pin the + * entire parent string in memory. + * + * When a consumer actually needs the text (e.g., to display a node's content + * or to build a template name), it calls `slice(source, start, end)`. This + * single call site keeps the string-resolution pattern consistent. + * + * @example Resolving a token range to its string value + * ```ts + * import { slice } from './text_source.ts'; + * + * const src = '== Heading =='; + * slice(src, 3, 10); // 'Heading' + * ``` + * + * @example Resolving a zero-length range + * ```ts + * import { slice } from './text_source.ts'; + * + * slice('hello', 2, 2); // '' + * ``` + * + * @param source - The text source to read from. + * @param start - Inclusive start offset (UTF-16 code units). + * @param end - Exclusive end offset (UTF-16 code units). + * @returns The resolved substring. + */ +export function slice(source: TextSource, start: number, end: number): string { + return source.slice(start, end); +} diff --git a/experiments/event-shape-study/lazy-position/code/token.ts b/experiments/event-shape-study/lazy-position/code/token.ts new file mode 100644 index 0000000..b91d7ba --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/token.ts @@ -0,0 +1,344 @@ +/** + * Token vocabulary and token shape for the raw scanner layer. + * + * This file defines the smallest structural units the tokenizer can emit. + * A token is not a parsed wiki node. It is a labeled span of source text. + * Later stages decide what those spans mean in context. + * + * The tokenizer walks a `TextSource` and emits `Token` objects whose `type` + * says what kind of character sequence was recognized and whose `start` and + * `end` fields point back into the original source. Consumers recover text + * only when they actually need it. + * + * That design keeps the hot path simple: + * + * - the scanner can work with offsets instead of allocating a string for + * every token + * - downstream code can slice lazily + * - small token views do not accidentally keep large source strings alive + * + * The key invariant is simple: token ranges tile the input from start to end. + * There are no gaps and no overlaps. Adjacent tokens meet exactly at their + * shared boundary. + * + * For the input `"== Hi =="`, the stream can be visualized like this: + * + * ``` + * source: = = H i = = + * index : 0 1 2 3 4 5 6 7 8 + * + * range : [0,2) [2,3) [3,5) [5,6) [6,8) [8,8) + * token : ? WHITESPACE TEXT WHITESPACE ? EOF + * ``` + * + * The exact token kind for the `==` runs depends on the tokenizer's heading + * rules. If the scanner distinguishes opening and closing heading markers, + * those positions would be `HEADING_MARKER` and `HEADING_MARKER_CLOSE`. + * If it does not, they would be `EQUALS`. + * + * This file only defines the vocabulary and the shared token contract. + * Block structure, inline structure, and semantic classification happen in + * later stages of the pipeline. + * + * @module + */ + +/** + * Constant map of token kinds emitted by the tokenizer. + * + * Each value names one class of source span the scanner can recognize. + * Some token kinds are purely lexical, such as `TEXT`, `NEWLINE`, and + * `WHITESPACE`. Others mark delimiter runs such as `[[`, `{{`, `{|`, + * or apostrophe runs used later for bold and italic parsing. + * + * `TokenType` is a plain object instead of a TypeScript `enum`, then frozen at + * runtime. + * + * That keeps the runtime shape simple and standard JavaScript friendly while + * still giving TypeScript a literal-string union for narrowing and exhaustive + * switching. + * + * In practice that means: + * + * - debugger output stays readable because token types are strings + * - logs show meaningful names instead of numeric enum members + * - bundlers do not need to preserve enum machinery + * + * Example shape: + * + * ```ts + * const tok = { type: TokenType.TEXT, start: 5, end: 9 }; + * ``` + * + * The token does not store its own string value. Consumers recover text from + * the source with `slice(source, tok.start, tok.end)`. + * + * Freezing is intentional here because token kinds are parser-owned + * vocabulary, not an extension registry. If a downstream tool wants extra + * labels, it should define its own adapter-level strings instead of mutating + * the tokenizer's shared constant table. + * + * Keep one boundary in mind while reading these names: they describe what the + * scanner saw, not the full meaning of the construct. For example, `[[` may + * later become a wikilink, a category link, or a file link depending on the + * surrounding parse rules. + */ +/** Public map shape for the stable tokenizer token vocabulary. */ +export type TokenTypeMap = Readonly<{ + TEXT: 'TEXT'; + NEWLINE: 'NEWLINE'; + WHITESPACE: 'WHITESPACE'; + HEADING_MARKER: 'HEADING_MARKER'; + HEADING_MARKER_CLOSE: 'HEADING_MARKER_CLOSE'; + BULLET: 'BULLET'; + HASH: 'HASH'; + COLON: 'COLON'; + SEMICOLON: 'SEMICOLON'; + THEMATIC_BREAK: 'THEMATIC_BREAK'; + TABLE_OPEN: 'TABLE_OPEN'; + TABLE_CLOSE: 'TABLE_CLOSE'; + TABLE_ROW: 'TABLE_ROW'; + TABLE_CAPTION: 'TABLE_CAPTION'; + PIPE: 'PIPE'; + DOUBLE_PIPE: 'DOUBLE_PIPE'; + TABLE_HEADER_CELL: 'TABLE_HEADER_CELL'; + DOUBLE_BANG: 'DOUBLE_BANG'; + APOSTROPHE_RUN: 'APOSTROPHE_RUN'; + LINK_OPEN: 'LINK_OPEN'; + LINK_CLOSE: 'LINK_CLOSE'; + EXT_LINK_OPEN: 'EXT_LINK_OPEN'; + EXT_LINK_CLOSE: 'EXT_LINK_CLOSE'; + TEMPLATE_OPEN: 'TEMPLATE_OPEN'; + TEMPLATE_CLOSE: 'TEMPLATE_CLOSE'; + ARGUMENT_OPEN: 'ARGUMENT_OPEN'; + ARGUMENT_CLOSE: 'ARGUMENT_CLOSE'; + TAG_OPEN: 'TAG_OPEN'; + TAG_CLOSE: 'TAG_CLOSE'; + CLOSING_TAG_OPEN: 'CLOSING_TAG_OPEN'; + SELF_CLOSING_TAG_END: 'SELF_CLOSING_TAG_END'; + COMMENT_OPEN: 'COMMENT_OPEN'; + COMMENT_CLOSE: 'COMMENT_CLOSE'; + HTML_ENTITY: 'HTML_ENTITY'; + SIGNATURE: 'SIGNATURE'; + BEHAVIOR_SWITCH: 'BEHAVIOR_SWITCH'; + PREFORMATTED_MARKER: 'PREFORMATTED_MARKER'; + EQUALS: 'EQUALS'; + EOF: 'EOF'; +}>; + +const TOKEN_TYPE_VALUES: TokenTypeMap = { + // -- Text and whitespace -- + + /** Literal text content (no special wiki meaning at this position). */ + TEXT: 'TEXT', + /** Newline sequence: `\n`, `\r\n`, or bare `\r`. */ + NEWLINE: 'NEWLINE', + /** One or more space or tab characters. */ + WHITESPACE: 'WHITESPACE', + + // -- Heading -- + + /** One or more `=` characters recognized as a heading opener. */ + HEADING_MARKER: 'HEADING_MARKER', + /** One or more `=` characters recognized as a heading closer. */ + HEADING_MARKER_CLOSE: 'HEADING_MARKER_CLOSE', + + // -- Lists -- + + /** `*` at line start (bullet list marker). */ + BULLET: 'BULLET', + /** `#` at line start (ordered list marker). */ + HASH: 'HASH', + /** `:` at line start (definition description / indent). */ + COLON: 'COLON', + /** `;` at line start (definition term). */ + SEMICOLON: 'SEMICOLON', + + // -- Thematic break -- + + /** Four or more `-` at line start (`----`). */ + THEMATIC_BREAK: 'THEMATIC_BREAK', + + // -- Table -- + + /** `{|` at line start (table open). */ + TABLE_OPEN: 'TABLE_OPEN', + /** `|}` at line start (table close). */ + TABLE_CLOSE: 'TABLE_CLOSE', + /** `|-` at line start (table row separator). */ + TABLE_ROW: 'TABLE_ROW', + /** `|+` at line start (table caption). */ + TABLE_CAPTION: 'TABLE_CAPTION', + /** `|` (table cell delimiter or separator). */ + PIPE: 'PIPE', + /** `||` (inline table cell separator). */ + DOUBLE_PIPE: 'DOUBLE_PIPE', + /** `!` at line start (table header cell). */ + TABLE_HEADER_CELL: 'TABLE_HEADER_CELL', + /** `!!` (inline table header cell separator). */ + DOUBLE_BANG: 'DOUBLE_BANG', + + // -- Bold / Italic -- + + /** + * Consecutive `'` characters (2 or more). The token's length encodes + * + * + * Consecutive apostrophes, usually length 2 or greater. + * + * The tokenizer preserves the raw run length and leaves interpretation to the + * inline parser. That later stage decides whether the run participates in + * italic, bold, bold+italic, or should stay literal under recovery rules. + * + * Standard usage is: 2 = italic, 3 = bold, 5 = bold+italic, etc. + */ + APOSTROPHE_RUN: 'APOSTROPHE_RUN', + + // -- Links -- + + /** `[[` delimiter. Later parsing decides whether this is a wikilink, file link, or category link. */ + LINK_OPEN: 'LINK_OPEN', + /** `]]` delimiter for double-bracket links. */ + LINK_CLOSE: 'LINK_CLOSE', + /** `[` delimiter for bracketed external-link syntax. */ + EXT_LINK_OPEN: 'EXT_LINK_OPEN', + /** `]` delimiter for bracketed external-link syntax. */ + EXT_LINK_CLOSE: 'EXT_LINK_CLOSE', + + // -- Templates / arguments -- + + /** `{{` delimiter used by templates and parser-function-like constructs. */ + TEMPLATE_OPEN: 'TEMPLATE_OPEN', + /** `}}` closing delimiter for double-brace constructs. */ + TEMPLATE_CLOSE: 'TEMPLATE_CLOSE', + /** `{{{` opening delimiter for triple-brace argument syntax. */ + ARGUMENT_OPEN: 'ARGUMENT_OPEN', + /** `}}}` closing delimiter for triple-brace argument syntax. */ + ARGUMENT_CLOSE: 'ARGUMENT_CLOSE', + + // -- HTML / extension tags -- + + /** `<` that opens an HTML or extension tag. */ + TAG_OPEN: 'TAG_OPEN', + /** `>` that closes a tag opening. */ + TAG_CLOSE: 'TAG_CLOSE', + /** `` (self-closing tag end). */ + SELF_CLOSING_TAG_END: 'SELF_CLOSING_TAG_END', + /** `` (comment close). */ + COMMENT_CLOSE: 'COMMENT_CLOSE', + + // -- HTML entity -- + /** A complete HTML character reference such as `&`, `{`, or ``. */ + HTML_ENTITY: 'HTML_ENTITY', + + // -- Special constructs -- + + /** `~~~`, `~~~~`, or `~~~~~` (signature). */ + SIGNATURE: 'SIGNATURE', + /** `__TOC__`, `__NOTOC__`, etc. (behavior switch). */ + BEHAVIOR_SWITCH: 'BEHAVIOR_SWITCH', + /** Leading space at line start (preformatted line). */ + PREFORMATTED_MARKER: 'PREFORMATTED_MARKER', + + // -- Equals (non-heading context) -- + + /** `=` characters not classified as heading markers at this scanner position. */ + EQUALS: 'EQUALS', + + // -- End of input -- + + /** Signals end of the token stream. */ + EOF: 'EOF', +} as const; + +/** + * Stable token-type names emitted by the tokenizer. + * + * These names describe the raw token shapes the scanner recognized, not the + * final meaning a later parser stage may assign to them. + */ +export const TokenType: TokenTypeMap = Object.freeze(TOKEN_TYPE_VALUES); + +/** + * Union of all token type string literals. + * + * Derived from `TokenType`, so `Token["type"]` narrows cleanly in equality + * checks and `switch` statements. + */ +export type TokenType = typeof TokenType[keyof typeof TokenType]; + +/** + * Membership lookup for fast runtime validation of token type strings. + * + * `isToken()` only needs to answer one question: is this string one of the + * fixed token-type keys? A null-prototype object matches that use case better + * than a `Set`, keeps the vocabulary data explicit, strips inherited + * prototype properties such as `toString`, and still avoids repeated array + * allocation and linear scans over `Object.values(TokenType)`. + */ +const TOKEN_TYPE_LOOKUP: Partial> = Object.assign( + Object.create(null), + Object.fromEntries( + Object.values(TokenType).map((token_type) => [token_type, true] as const), + ), +); + +/** + * A single scanner token. + * + * A token identifies a span of source text and labels it with a token kind. + * It does not store a copied string value. Consumers recover text from the + * original `{@link TextSource}` when needed. + * + * This keeps the scanner cheap and makes token streams safe to hold onto even + * when the input is large. + * + * Another important invariant follows from the tokenizer contract: token + * ranges tile the input. For non-EOF tokens, the concatenation of all + * `slice(source, token.start, token.end)` values reconstructs the original + * source exactly. + */ +export interface Token { + /** Token kind from the shared `TokenType` vocabulary. */ + readonly type: TokenType; + + /** Inclusive UTF-16 start offset into the source. */ + readonly start: number; + + /** Exclusive UTF-16 end offset into the source. */ + readonly end: number; +} + +/** + * Returns `true` when a value has the runtime shape of a `Token`. + * + * This is mainly useful at system boundaries where static typing cannot help, + * such as JSON input, message passing, or mixed collections. + * + * The check is structural: + * + * - object and not `null` + * - `type` is a known token type string + * - `start` and `end` are numbers + * + * It does not validate semantic invariants such as `start <= end` or whether + * the range is valid for a particular source. + */ +export function isToken(value: unknown): value is Token { + // Reject primitives and null early — they can't be tokens. + if (typeof value !== 'object' || value === null) return false; + const obj = value as Record; + return ( + typeof obj.type === 'string' && + typeof obj.start === 'number' && + typeof obj.end === 'number' && + // O(1) lookup against the precomputed token-type vocabulary. + // `Object.hasOwn(...)` keeps the check on this table's own keys instead of + // matching something inherited through the normal object prototype chain. + Object.hasOwn(TOKEN_TYPE_LOOKUP, obj.type) + ); +} diff --git a/experiments/event-shape-study/lazy-position/code/tokenizer.ts b/experiments/event-shape-study/lazy-position/code/tokenizer.ts new file mode 100644 index 0000000..5b7676c --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/tokenizer.ts @@ -0,0 +1,1128 @@ +/** + * Generator-based tokenizer for raw wikitext source. + * + * This is the first real parsing stage. Its job is simple: walk through the + * source from left to right and mark the important character runs it sees. + * Think of it as turning one long source string into labeled slices such as: + * + * - plain text + * - newline + * - `[[` + * - `{{` + * - `|` + * - heading marker runs like `==` + * + * It does not decide the full meaning of those pieces yet. For example, + * spotting `[[` is not the same as deciding whether the final construct is a + * normal wikilink, a category link, or a file link. This stage only marks the + * raw source shape. Parser literature often calls this the lexical stage, but + * the practical meaning here is just "recognize the text patterns first, then + * let later stages decide what they mean together". + * + * The tokenizer runs in one pass and yields tokens lazily: + * + * ``` + * TextSource -> tokenize() -> Token stream + * ``` + * + * The hottest operation in this whole file is `source.charCodeAt(i)`. The + * tokenizer calls it over and over while scanning. Using numeric character + * codes lets the hot loop compare small numbers instead of creating temporary + * one-character strings. + * + * The scanner also keeps one small piece of context: whether it is at the + * start of a line. Wikitext uses the same characters differently depending on + * where they appear. For example: + * + * - `=` at the start of a line can begin a heading + * - `*` at the start of a line can begin a bullet list item + * - the same characters in the middle of normal text often mean something else + * or just stay text + * + * A large chunk of the input is usually ordinary prose. So the fast path is + * not the special markup. The fast path is "keep absorbing plain text until we + * hit something that could start markup". + * + * Every code unit in the source belongs to exactly one token range. In plain + * English, the token ranges cover the whole input with no holes and no overlap. + * If one token ends at offset 12, the next one starts at 12. + * + * For `Hello [[world]]`, the stream tiles the input like this: + * + * ``` + * source: H e l l o [ [ w o r l d ] ] + * index : 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 + * token : [0,5) TEXT + * [5,6) WHITESPACE + * [6,8) LINK_OPEN + * [8,13) TEXT + * [13,15) LINK_CLOSE + * ``` + * + * The tokenizer never throws. Even malformed input still produces a full token + * stream ending in EOF. + * + * @example Tokenizing a simple heading + * ```ts + * import { tokenize } from './tokenizer.ts'; + * + * const tokens = Array.from(tokenize('== Hi ==')); + * ``` + * + * @example Tokenizing a wikilink + * ```ts + * import { tokenize } from './tokenizer.ts'; + * + * const tokens = Array.from(tokenize('[[Page|label]]')); + * // LINK_OPEN, TEXT("Page"), PIPE, TEXT("label"), LINK_CLOSE, EOF + * ``` + * + * @module + */ + +import type { TextSource } from './text_source.ts'; +import type { Token } from './token.ts'; +import { TokenType } from './token.ts'; + +// --------------------------------------------------------------------------- +// Character code constants +// +// The tokenizer's inner loop compares numeric character codes, not strings. +// These constants name each code point the scanner cares about. Using named +// constants instead of inline hex literals makes the switch/if chains +// readable without sacrificing performance (V8 inlines them). +// --------------------------------------------------------------------------- + +/** @internal */ const CC_LF = 0x0a; // '\n' +/** @internal */ const CC_CR = 0x0d; // '\r' +/** @internal */ const CC_SPACE = 0x20; // ' ' +/** @internal */ const CC_TAB = 0x09; // '\t' +/** @internal */ const CC_BANG = 0x21; // '!' +/** @internal */ const CC_HASH = 0x23; // '#' +/** @internal */ const CC_AMP = 0x26; // '&' +/** @internal */ const CC_APOSTROPHE = 0x27; // "'" +/** @internal */ const CC_ASTERISK = 0x2a; // '*' +/** @internal */ const CC_DASH = 0x2d; // '-' +/** @internal */ const CC_COLON = 0x3a; // ':' +/** @internal */ const CC_SEMICOLON = 0x3b; // ';' +/** @internal */ const CC_LT = 0x3c; // '<' +/** @internal */ const CC_EQUALS = 0x3d; // '=' +/** @internal */ const CC_GT = 0x3e; // '>' +/** @internal */ const CC_OPEN_BRACKET = 0x5b; // '[' +/** @internal */ const CC_CLOSE_BRACKET = 0x5d; // ']' +/** @internal */ const CC_UNDERSCORE = 0x5f; // '_' +/** @internal */ const CC_OPEN_BRACE = 0x7b; // '{' +/** @internal */ const CC_PIPE = 0x7c; // '|' +/** @internal */ const CC_CLOSE_BRACE = 0x7d; // '}' +/** @internal */ const CC_TILDE = 0x7e; // '~' +/** @internal */ const CC_SLASH = 0x2f; // '/' + +// --------------------------------------------------------------------------- +// Delimiter lookup table +// +// A precomputed 128-entry table where entry `c` is 1 if character code `c` +// could start a wikitext delimiter, 0 otherwise. Used by the TEXT +// accumulation loop to decide when to stop consuming plain text. +// +// This replaces a 23-case switch statement. The advantage: +// - Single array access instead of a jump table +// - Characters >= 128 (all non-ASCII: CJK, emoji, RTL) skip the lookup +// entirely via a single comparison (`c < 128`), making prose-heavy +// articles faster to scan. +// --------------------------------------------------------------------------- + +/** + * Return whether a character code is a wikitext delimiter that can start + * a recognized token. Used both as a readable predicate and to populate + * the {@linkcode DELIMITER} lookup table. + */ +function isDelimiterChar(c: number): boolean { + switch (c) { + case CC_LF: + case CC_CR: + case CC_SPACE: + case CC_TAB: + case CC_BANG: + case CC_HASH: + case CC_AMP: + case CC_APOSTROPHE: + case CC_ASTERISK: + case CC_DASH: + case CC_COLON: + case CC_SEMICOLON: + case CC_LT: + case CC_EQUALS: + case CC_GT: + case CC_OPEN_BRACKET: + case CC_CLOSE_BRACKET: + case CC_UNDERSCORE: + case CC_OPEN_BRACE: + case CC_PIPE: + case CC_CLOSE_BRACE: + case CC_TILDE: + case CC_SLASH: + return true; + default: + return false; + } +} + +const DELIMITER = Uint8Array.from({ length: 128 }, (_, c) => + isDelimiterChar(c) ? 1 : 0, +); + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +/** + * Create a token object. Centralizes token construction so the shape is + * consistent and monomorphic (V8 produces a single hidden class). + */ +function tok(type: TokenType, start: number, end: number): Token { + return { type, start, end }; +} + +/** + * Check whether a character code is an ASCII letter (a-z, A-Z). + */ +function isAsciiLetter(code: number): boolean { + return (code >= 0x41 && code <= 0x5a) || (code >= 0x61 && code <= 0x7a); +} + +/** + * Check whether a character code is an ASCII digit (0-9). + */ +function isAsciiDigit(code: number): boolean { + return code >= 0x30 && code <= 0x39; +} + +/** + * Check whether a character code is an ASCII alphanumeric character. + */ +function isAsciiAlphanumeric(code: number): boolean { + return isAsciiLetter(code) || isAsciiDigit(code); +} + + + +// --------------------------------------------------------------------------- +// Tokenizer +// --------------------------------------------------------------------------- + +/** + * Scan a {@linkcode TextSource} and yield one {@linkcode Token} per recognized + * syntactic unit. + * + * The generator performs a single left-to-right pass over the source, + * tracking one boolean (`lineStart`) alongside the scan position `i`. + * `lineStart` is `true` at position 0 and after every newline. This flag + * determines how ambiguous characters are classified: `=` at line start + * is a heading marker, but mid-line it is a plain equals sign (used in + * template argument `name=value` syntax). + * + * The algorithm has two phases per character. Both exist to keep the + * common case fast while still recognizing all wikitext delimiters: + * + * 1. **Fast text gate**: if the character is non-ASCII *or* is not in the + * DELIMITER lookup table, fall into a tight inner loop that absorbs + * consecutive non-delimiter characters into a single TEXT token. + * This handles the majority of input (plain prose) in bulk. + * + * 2. **Delimiter dispatch**: for the ~23 delimiter characters, a `switch` + * statement selects the appropriate token or multi-character sequence. + * Each case consumes one logical unit (e.g., `` close. If found, it yields three tokens: + // COMMENT_OPEN, TEXT (the comment content), COMMENT_CLOSE. + // If `-->` is never found (unclosed comment), everything + // after `visible" + // COMMENT_OPEN [0,4), TEXT [4,11) " hidden ", + // COMMENT_CLOSE [11,14), TEXT [14,21) "visible" + // + // Example: "` arrived. + // + // 2. ``). + // → CLOSING_TAG_OPEN [i,i+2) + // + // 3. `<` followed by a letter opens an HTML/extension tag + // (e.g., ``, `
`). Only the `<` itself is emitted; + // the tag name becomes TEXT tokens for the block parser. + // → TAG_OPEN [i,i+1) + // + // This split may look surprising if you expect a single "whole tag" + // token. The tokenizer stays lexical here: it only marks the raw tag + // boundary shape. Later stages decide whether the following text forms + // a recognized HTML-like construct in context. + // + // 4. Bare `<` (not followed by `!--`, `/`, or letter) is text. + // Example: "3 < 5" → the '<' is TEXT. + case CC_LT: { + // + if (i + 3 < len && + source.charCodeAt(i + 1) === CC_BANG && + source.charCodeAt(i + 2) === CC_DASH && + source.charCodeAt(i + 3) === CC_DASH) { + yield tok(TokenType.COMMENT_OPEN, i, i + 4); + i += 4; + const contentStart = i; + let found = false; + while (i < len) { + if (source.charCodeAt(i) === CC_DASH && + i + 2 < len && + source.charCodeAt(i + 1) === CC_DASH && + source.charCodeAt(i + 2) === CC_GT) { + if (i > contentStart) { + yield tok(TokenType.TEXT, contentStart, i); + } + yield tok(TokenType.COMMENT_CLOSE, i, i + 3); + i += 3; + found = true; + break; + } + i++; + } + // Recovery keeps the stream tiled when the comment never closes. The + // caller can still see where comment syntax started, and the trailing + // text is not lost. + if (!found && i > contentStart) { + yield tok(TokenType.TEXT, contentStart, i); + } + lineStart = false; + continue; + } + // ` in `
` or `
`. Always yields + // TAG_CLOSE regardless of context; the block parser pairs it + // with the earlier TAG_OPEN. + case CC_GT: { + yield tok(TokenType.TAG_CLOSE, i, i + 1); + i += 1; + lineStart = false; + continue; + } + + // --- Slash: self-closing tag end '/>' --- + // + // `/>` ends a self-closing tag like `
` or ``. + // A bare `/` not followed by `>` is plain text. + case CC_SLASH: { + if (i + 1 < len && source.charCodeAt(i + 1) === CC_GT) { + yield tok(TokenType.SELF_CLOSING_TAG_END, i, i + 2); + i += 2; + lineStart = false; + continue; + } + yield tok(TokenType.TEXT, i, i + 1); + i += 1; + lineStart = false; + continue; + } + + // --- Ampersand: HTML character entities --- + // + // HTML entities let wikitext include characters that would otherwise + // be interpreted as markup (e.g., `<` for `<`, `&` for `&`). + // The tokenizer recognizes well-formed entities so the AST can + // represent them as HtmlEntity nodes rather than raw text. + // + // Three entity forms: + // &name; → named entity (e.g., & < ") + // &#digits; → decimal entity (e.g., {) + // &#xhex; → hex entity (e.g., 💩) + // + // The algorithm probes ahead from `&`: + // Step 1: is next char `#`? → numeric path + // Step 1a: is char after `#` an `x`/`X`? → hex path + // absorb hex digits, require trailing `;` + // Step 1b: otherwise → decimal path + // absorb decimal digits, require trailing `;` + // Step 2: is next char a letter? → named entity path + // absorb alphanumeric chars, require trailing `;` + // Step 3: if none matched → bare `&` is TEXT + // + // If the probe finds the right pattern but no `;`, the `&` falls + // through to TEXT. This means `¬aentity` stays as text. + // + // Example: "&" → HTML_ENTITY [0,5) + // Example: "{" → HTML_ENTITY [0,6) + // Example: "" → HTML_ENTITY [0,6) + // Example: "&oops" → TEXT [0,1) (no semicolon) + case CC_AMP: { + const start = i; + let j = i + 1; + if (j < len) { + const next = source.charCodeAt(j); + // Numeric entity + if (next === CC_HASH) { + j++; + if (j < len && (source.charCodeAt(j) === 0x78 || source.charCodeAt(j) === 0x58)) { + // &#x hex + j++; + const hexStart = j; + while (j < len && isHexDigit(source.charCodeAt(j))) j++; + if (j > hexStart && j < len && source.charCodeAt(j) === CC_SEMICOLON) { + j++; + yield tok(TokenType.HTML_ENTITY, start, j); + i = j; + lineStart = false; + continue; + } + } else { + // &#decimal + const decStart = j; + while (j < len && isAsciiDigit(source.charCodeAt(j))) j++; + if (j > decStart && j < len && source.charCodeAt(j) === CC_SEMICOLON) { + j++; + yield tok(TokenType.HTML_ENTITY, start, j); + i = j; + lineStart = false; + continue; + } + } + } + // Named entity + else if (isAsciiLetter(next)) { + j++; + while (j < len && isAsciiAlphanumeric(source.charCodeAt(j))) j++; + if (j < len && source.charCodeAt(j) === CC_SEMICOLON) { + j++; + yield tok(TokenType.HTML_ENTITY, start, j); + i = j; + lineStart = false; + continue; + } + } + } + yield tok(TokenType.TEXT, i, i + 1); + i += 1; + lineStart = false; + continue; + } + + // --- Open brace: table, template, and argument openers --- + // + // Braces introduce the most deeply nested structures in wikitext: + // tables, templates, and template arguments. The tokenizer must + // distinguish all three because they nest differently and the + // block/inline parsers handle each one as a distinct construct. + // + // Three multi-character sequences start with `{`: + // + // 1. `{|` at line start opens a table. + // Example: "{| class='wikitable'" → TABLE_OPEN [0,2) + // + // 2. `{{{` opens a template argument (triple-brace parameter). + // Example: "{{{name|default}}}" → ARGUMENT_OPEN [0,3) + // Checked before `{{` because `{{{` starts with `{{`. + // + // 3. `{{` opens a template or parser function. + // Example: "{{Infobox|...}}" → TEMPLATE_OPEN [0,2) + // + // A lone `{` is plain text. + case CC_OPEN_BRACE: { + if (lineStart && i + 1 < len && source.charCodeAt(i + 1) === CC_PIPE) { + yield tok(TokenType.TABLE_OPEN, i, i + 2); + i += 2; + lineStart = false; + continue; + } + if (i + 2 < len && + source.charCodeAt(i + 1) === CC_OPEN_BRACE && + source.charCodeAt(i + 2) === CC_OPEN_BRACE) { + yield tok(TokenType.ARGUMENT_OPEN, i, i + 3); + i += 3; + lineStart = false; + continue; + } + if (i + 1 < len && source.charCodeAt(i + 1) === CC_OPEN_BRACE) { + yield tok(TokenType.TEMPLATE_OPEN, i, i + 2); + i += 2; + lineStart = false; + continue; + } + yield tok(TokenType.TEXT, i, i + 1); + i += 1; + lineStart = false; + continue; + } + + // --- Close brace: argument and template closers --- + // + // `}}}` closes a template argument. `}}` closes a template. + // `}}}` is checked first because it starts with `}}`. + // A lone `}` is plain text. + // + // Example: "{{{1|fallback}}}" → ... ARGUMENT_CLOSE [14,17) + // Example: "{{T}}" → ... TEMPLATE_CLOSE [3,5) + case CC_CLOSE_BRACE: { + if (i + 2 < len && + source.charCodeAt(i + 1) === CC_CLOSE_BRACE && + source.charCodeAt(i + 2) === CC_CLOSE_BRACE) { + yield tok(TokenType.ARGUMENT_CLOSE, i, i + 3); + i += 3; + lineStart = false; + continue; + } + if (i + 1 < len && source.charCodeAt(i + 1) === CC_CLOSE_BRACE) { + yield tok(TokenType.TEMPLATE_CLOSE, i, i + 2); + i += 2; + lineStart = false; + continue; + } + yield tok(TokenType.TEXT, i, i + 1); + i += 1; + lineStart = false; + continue; + } + + // --- Open bracket: wikilinks and external links --- + // + // `[[` opens a wikilink (internal link, image, or category tag). + // `[` alone opens an external link. + // + // Example: "[[Main Page|display]]" → LINK_OPEN [0,2) + // Example: "[https://example.com text]" → EXT_LINK_OPEN [0,1) + case CC_OPEN_BRACKET: { + if (i + 1 < len && source.charCodeAt(i + 1) === CC_OPEN_BRACKET) { + yield tok(TokenType.LINK_OPEN, i, i + 2); + i += 2; + } else { + // A single `[` only means "maybe external link" at this stage. The + // inline parser validates the URL shape later. + yield tok(TokenType.EXT_LINK_OPEN, i, i + 1); + i += 1; + } + lineStart = false; + continue; + } + + // --- Close bracket: wikilinks and external links --- + // + // `]]` closes a wikilink. `]` alone closes an external link. + case CC_CLOSE_BRACKET: { + if (i + 1 < len && source.charCodeAt(i + 1) === CC_CLOSE_BRACKET) { + yield tok(TokenType.LINK_CLOSE, i, i + 2); + i += 2; + } else { + yield tok(TokenType.EXT_LINK_CLOSE, i, i + 1); + i += 1; + } + lineStart = false; + continue; + } + + // --- Pipe: table delimiters and cell separators --- + // + // The pipe `|` is the most context-dependent delimiter in wikitext. + // Inside tables it controls row/cell/caption structure; inside + // templates and links it separates arguments and display text. + // The tokenizer emits distinct token types so that the block + // parser can tell these roles apart. + // + // At line start (inside a table): + // `|}` closes the table → TABLE_CLOSE [i,i+2) + // `|-` starts a new table row → TABLE_ROW [i,i+2) + // `|+` starts a table caption → TABLE_CAPTION [i,i+2) + // `|` starts a table data cell → PIPE [i,i+1) + // + // Mid-line: + // `||` separates inline cells → DOUBLE_PIPE [i,i+2) + // `|` separates template args, + // link display text, etc. → PIPE [i,i+1) + // + // Example: "{|\n|-\n| cell1 || cell2\n|}" + // line 1: "{|" → TABLE_OPEN + // line 2: "|-" → TABLE_ROW + // line 3: "|" → PIPE, " cell1 " → tokens, "||" → DOUBLE_PIPE + // line 4: "|}" → TABLE_CLOSE + case CC_PIPE: { + if (lineStart) { + if (i + 1 < len && source.charCodeAt(i + 1) === CC_CLOSE_BRACE) { + yield tok(TokenType.TABLE_CLOSE, i, i + 2); + i += 2; + lineStart = false; + continue; + } + if (i + 1 < len && source.charCodeAt(i + 1) === CC_DASH) { + yield tok(TokenType.TABLE_ROW, i, i + 2); + i += 2; + lineStart = false; + continue; + } + if (i + 1 < len && source.charCodeAt(i + 1) === 0x2b) { + yield tok(TokenType.TABLE_CAPTION, i, i + 2); + i += 2; + lineStart = false; + continue; + } + // Bare `|` at line start can begin a table data row, so it remains a + // structural token even before the block parser has confirmed that we + // are really inside a table. + yield tok(TokenType.PIPE, i, i + 1); + i += 1; + lineStart = false; + continue; + } + if (i + 1 < len && source.charCodeAt(i + 1) === CC_PIPE) { + yield tok(TokenType.DOUBLE_PIPE, i, i + 2); + i += 2; + } else { + yield tok(TokenType.PIPE, i, i + 1); + i += 1; + } + lineStart = false; + continue; + } + + // --- Apostrophe: bold and italic markers --- + // + // Bold/italic is the most common inline formatting in wikitext. + // The tokenizer counts consecutive apostrophes and emits one + // APOSTROPHE_RUN token. The inline parser later resolves whether + // a given run opens/closes bold, italic, or both, using + // MediaWiki's disambiguation algorithm. + // + // Consecutive apostrophes encode formatting: + // '' (2) = italic toggle + // ''' (3) = bold toggle + // '''' (4) = effectively bold + one literal ' + // ''''' (5) = bold+italic toggle + // + // The tokenizer absorbs the run and emits APOSTROPHE_RUN for + // runs of 2 or more. A single apostrophe is TEXT (it's a normal + // punctuation character). The inline parser later determines the + // exact bold/italic nesting using MediaWiki's disambiguation + // algorithm. + // + // Example: "it's '''bold''' text" + // pos 2: single ' → TEXT [2,3) + // pos 4: ''' → APOSTROPHE_RUN [4,7) + // pos 11: ''' → APOSTROPHE_RUN [11,14) + case CC_APOSTROPHE: { + const start = i; + while (i < len && source.charCodeAt(i) === CC_APOSTROPHE) i++; + // Single apostrophes are overwhelmingly ordinary punctuation. Treating + // only longer runs as structural keeps prose cheap and easier to debug. + if (i - start >= 2) { + yield tok(TokenType.APOSTROPHE_RUN, start, i); + } else { + yield tok(TokenType.TEXT, start, i); + } + lineStart = false; + continue; + } + + // --- Tilde: signature markers --- + // + // Tilde runs of exactly 3, 4, or 5 are signature markers (expanded + // by MediaWiki's pre-save transform to user/timestamp text): + // ~~~ (3) = username + // ~~~~ (4) = username + timestamp + // ~~~~~ (5) = timestamp only + // + // Runs of 1-2 or 6+ tildes are plain text. The tokenizer absorbs + // the full run, checks the length, and decides. + // + // Example: "Signed: ~~~~" + // pos 8: absorb 4 tildes → SIGNATURE [8,12) + // + // Example: "~~~~~~ not a sig" + // pos 0: absorb 6 tildes → TEXT [0,6) + case CC_TILDE: { + const start = i; + while (i < len && source.charCodeAt(i) === CC_TILDE) i++; + const runLen = i - start; + if (runLen >= 3 && runLen <= 5) { + yield tok(TokenType.SIGNATURE, start, i); + } else { + yield tok(TokenType.TEXT, start, i); + } + lineStart = false; + continue; + } + + // --- Underscore: behavior switches __WORD__ --- + // + // Behavior switches are double-underscore keywords that control + // page-level rendering in MediaWiki (e.g., __TOC__, __NOTOC__, + // __NOEDITSECTION__). + // + // The tokenizer recognizes the structural pattern `__LETTERS__` + // (two underscores, one or more ASCII letters, two underscores) + // and always emits BEHAVIOR_SWITCH. It does NOT check against a + // known word list. Whether the word is valid for a given + // MediaWiki installation is a consumer/profile concern. + // + // This matters because MediaWiki extensions can register new + // behavior switches at runtime. A source parser cannot know the + // full set without configuration. + // + // Algorithm: + // Step 1: check for two consecutive underscores __ + // Step 2: scan forward absorbing ASCII letters (a-z, A-Z) + // Step 3: require at least one letter (j > i+2) + // Step 4: check for closing __ at position j + // Step 5: if all matched → BEHAVIOR_SWITCH; otherwise TEXT + // + // Example: "__TOC__" + // pos 0: __ detected, scan letters T,O,C → j=5 + // pos 5: __ found at j → BEHAVIOR_SWITCH [0,7) + // + // Example: "__123__" (digits, not letters) + // pos 0: __ detected, scan finds '1' (not a letter) → j=2 + // j == i+2 (no letters absorbed) → TEXT [0,2) + // + // Example: "____" (no letters between) + // pos 0: __ detected, j=2, scan finds '_' (not a letter) + // j == i+2 → TEXT [0,2), then TEXT [2,4) + // + // Example: "__CUSTOM__" (unknown but valid pattern) + // → BEHAVIOR_SWITCH [0,10) (tokenizer is structural) + case CC_UNDERSCORE: { + if (i + 1 < len && source.charCodeAt(i + 1) === CC_UNDERSCORE) { + const start = i; + let j = i + 2; + while (j < len && isAsciiLetter(source.charCodeAt(j))) j++; + if (j > i + 2 && + j + 1 < len && + source.charCodeAt(j) === CC_UNDERSCORE && + source.charCodeAt(j + 1) === CC_UNDERSCORE) { + yield tok(TokenType.BEHAVIOR_SWITCH, start, j + 2); + i = j + 2; + lineStart = false; + continue; + } + yield tok(TokenType.TEXT, i, i + 2); + i += 2; + lineStart = false; + continue; + } + yield tok(TokenType.TEXT, i, i + 1); + i += 1; + lineStart = false; + continue; + } + + // --- Fallback: any delimiter char not handled above --- + default: { + yield tok(TokenType.TEXT, i, i + 1); + i += 1; + lineStart = false; + continue; + } + } + } + + // Final EOF token: signals end of stream. + yield tok(TokenType.EOF, len, len); +} + + + +/** + * Check whether a character code is a hexadecimal digit (0-9, a-f, A-F). + */ +function isHexDigit(code: number): boolean { + return ( + (code >= 0x30 && code <= 0x39) || // 0-9 + (code >= 0x41 && code <= 0x46) || // A-F + (code >= 0x61 && code <= 0x66) // a-f + ); +} diff --git a/experiments/event-shape-study/lazy-position/code/tree_builder.ts b/experiments/event-shape-study/lazy-position/code/tree_builder.ts new file mode 100644 index 0000000..f46211f --- /dev/null +++ b/experiments/event-shape-study/lazy-position/code/tree_builder.ts @@ -0,0 +1,999 @@ +/** + * Build a wikist tree from the parser's event stream. + * + * The parser's core pipeline is events-first, not tree-first. That keeps the + * hot path cheaper for consumers that only need a stream. The tree builder is + * the stage that materializes those events into nested Wikist nodes when a + * caller does want an object graph it can walk later. + * + * One detail matters here: text events are range-first. They carry source + * offsets, not copied strings. That is great for the parser pipeline, but a + * `Text` node needs a real `value` string. For that reason, `buildTree()` + * takes the original `source` alongside the event iterable. + * + * Read the conversion like this: + * + * ```text + * event stream + * ├─► enter(node) -> push frame + * ├─► text(range) -> slice source, append Text child + * └─► exit(node) -> pop frame, attach finished node to parent + * ``` + * + * The tree builder keeps tree shape, diagnostics, and recovery metadata as + * separate result lanes. + * `buildTree()` returns the default tolerant tree. `buildTreeWithDiagnostics()` + * keeps that same tree shape while preserving diagnostics. + * `buildTreeStrict()` is the conservative materialization lane that collapses + * recovery-heavy wrappers back to plain text when the source never clearly + * committed to them. `buildTreeWithRecovery()` adds an explicit `recovered` + * summary on top of the default diagnostics lane. + * + * @example Building a tree from the full event pipeline + * ```ts + * import { buildTree } from './tree_builder.ts'; + * import { events } from './parse.ts'; + * + * const source = "== Title ==\n\nA [[Page|link]]."; + * const tree = buildTree(events(source), { source }); + * ``` + * + * @module + */ + +import type { + DiagnosticSeverity, + ErrorEvent, + KnownDiagnosticCode, + Position, + WikitextEvent, +} from './events.ts'; +import type { TextSource } from './text_source.ts'; +import type { Point } from './events.ts'; +import type { WikistNode, WikistNodeType, WikistRoot } from './ast.ts'; +import { DiagnosticCode } from './events.ts'; +import { slice } from './text_source.ts'; + +/** + * Options for {@linkcode buildTree}. + * + * The event iterable alone is not enough to materialize literal node values, + * because text events only carry source offsets. The original source is the + * extra input that lets the tree builder turn those ranges back into strings. + */ +export interface BuildTreeOptions { + /** + * Original source text for resolving text-event ranges into node values. + */ + readonly source: TextSource; +} + +/** + * Accumulator frame for the final document root. + * + * This is separate from `NodeFrame` because incoming `enter('root')` and + * `exit('root')` events are treated as boundary markers, not as a child node + * that should be pushed and popped on the main node stack. + */ +interface RootFrame { + /** Discriminant for stack narrowing. */ + readonly kind: 'root'; + /** Top-level children collected while walking the stream. */ + readonly children: WikistNode[]; + /** Start point from `enter('root')` when present in the event stream. */ + start_point?: Point; + /** End point from `exit('root')` when present in the event stream. */ + end_point?: Point; +} + +/** + * Stack frame for one non-root node currently being materialized. + * + * Think of this as the builder's in-progress node record: we capture opening + * metadata on enter, append children while nested events arrive, and finalize + * the node when a matching or recovery-triggered exit is seen. + */ +interface NodeFrame { + /** Discriminant for stack narrowing. */ + readonly kind: 'node'; + /** Node type opened by the corresponding enter event. */ + readonly node_type: Exclude; + /** Props captured from the enter event and forwarded at finalize time. */ + readonly props: Readonly>; + /** Opening point captured from the enter event. */ + readonly start: Point; + /** + * Fallback end point when the stream ends before a matching exit arrives. + */ + readonly default_end: Point; + /** Whether this node type accepts child nodes in the wikist model. */ + readonly accepts_children: boolean; + /** Child nodes accumulated while this frame stays open on the stack. */ + readonly children: WikistNode[]; + /** Whether this frame should materialize back into plain source text. */ + recover_as_text?: boolean; +} + +/** Active builder stack item used during event-to-tree conversion. */ +type TreeFrame = RootFrame | NodeFrame; + +/** + * Stable materialization-policy names for consumers that want to refer to the + * parser's public tree-shaping policies without hard-coding string literals. + */ +/** Public map shape for the parser's stable tree-materialization policies. */ +export type TreeMaterializationPolicyMap = Readonly<{ + DEFAULT_HTML_LIKE: 'default-html-like'; + SOURCE_STRICT: 'source-strict'; +}>; + +const TREE_MATERIALIZATION_POLICY_VALUES: TreeMaterializationPolicyMap = { + /** Keep the parser's default tolerant HTML-like materialization. */ + DEFAULT_HTML_LIKE: 'default-html-like', + /** Collapse recovery-heavy wrappers back to plain source-backed text. */ + SOURCE_STRICT: 'source-strict', +} as const; + +/** + * Stable materialization-policy names for the parser's public tree-shaping + * policies. + */ +export const TreeMaterializationPolicy: TreeMaterializationPolicyMap = Object.freeze( + TREE_MATERIALIZATION_POLICY_VALUES, +); + +/** Public names for the tree materialization policies exposed by this module. */ +export type TreeMaterializationPolicy = + typeof TreeMaterializationPolicy[keyof typeof TreeMaterializationPolicy]; + +const PARENT_NODE_TYPE_LOOKUP: Partial> = Object.assign( + Object.create(null), + { + root: true, + heading: true, + paragraph: true, + preformatted: true, + list: true, + 'list-item': true, + 'definition-list': true, + 'definition-term': true, + 'definition-description': true, + table: true, + 'table-caption': true, + 'table-row': true, + 'table-cell': true, + bold: true, + italic: true, + 'bold-italic': true, + wikilink: true, + 'external-link': true, + 'image-link': true, + template: true, + 'template-argument': true, + 'parser-function': true, + 'html-tag': true, + redirect: true, + gallery: true, + reference: true, + }, +); + +const LITERAL_VALUE_NODE_LOOKUP: Partial> = Object.assign( + Object.create(null), + { + 'html-entity': true, + nowiki: true, + comment: true, + }, +); + +/** + * Parse-time diagnostic enriched with tree-location metadata. + * + * The parser already reports recovery points as `error` events. The problem + * for tree-only consumers is that those events disappear once `buildTree()` + * materializes the AST. + * + * `anchor` fixes that gap. Today the anchor is intentionally narrow: it stores + * one root-relative tree path to the nearest node active when the diagnostic + * was recorded. + * + * ```text + * root + * ├─ paragraph path [0] + * │ └─ bold path [0, 0] + * └─ table path [1] + * ``` + * + * Tools can walk `tree.children[path[0]].children[path[1]]...` to recover the + * closest concrete node around the recovery point. + * + * The public API stops there on purpose. Session-backed edit-stable anchors, + * slot identities, and other long-lived anchor semantics depend on later edit + * tracking work, so they stay out of `ParseDiagnostic` for now. + */ +export interface ParseDiagnosticAnchor { + /** Current anchor kind for diagnostics resolved against one final tree. */ + readonly kind: 'tree-path'; + /** Child-index path from the root to the nearest active node. */ + readonly path: readonly number[]; + /** Node type at `path`, or `'root'` when only the document is known. */ + readonly node_type: WikistNodeType; +} + +/** + * Recovery diagnostic preserved alongside the parsed tree. + * + * `anchor` is the location contract callers should use. It is tree-oriented + * today, which means it resolves against the final materialized tree only. It + * does not promise edit stability across later session changes yet. + */ +export interface ParseDiagnostic { + /** Human-readable description of what was recovered from. */ + readonly message: string; + /** Severity copied from the original diagnostic event when available. */ + readonly severity?: DiagnosticSeverity; + /** + * Stable machine-readable code for filtering and telemetry. + * + * Match on this field when building editor hints, quick fixes, or metrics. + * The human-readable `message` is still useful for logs and UI, but the code + * is the stable contract. + */ + readonly code?: KnownDiagnosticCode | string; + /** Whether recovery continued with a deterministic fallback. */ + readonly recoverable?: boolean; + /** Parser stage that emitted the diagnostic. */ + readonly source?: 'tokenizer' | 'block' | 'inline' | 'tree'; + /** Optional structured metadata payload. */ + readonly details?: Readonly>; + /** Source position where the diagnostic was detected. */ + readonly position: Position; + /** Narrow location anchor for resolving the nearest node in the final tree. */ + readonly anchor: ParseDiagnosticAnchor; +} + +/** + * Tree plus recovery diagnostics. + * + * The exact tree shape depends on which materialization policy produced it. + * `buildTreeWithDiagnostics()` returns the default HTML-like tree, while + * `buildTreeStrict()` returns the conservative source-strict tree. + */ +export interface ParseDiagnosticsResult { + /** Materialized wikist tree. */ + readonly tree: WikistRoot; + /** Diagnostics collected while consuming the event stream. */ + readonly diagnostics: readonly ParseDiagnostic[]; +} + +/** + * Recovery-aware tree result. + * + * This is the default diagnostics lane plus a boolean summary so a caller can + * branch on recovery explicitly without rechecking the diagnostics array + * length itself. + */ +export interface ParseResult extends ParseDiagnosticsResult { + /** Whether any recovery diagnostics were recorded while producing this tree. */ + readonly recovered: boolean; +} + +/** + * Materialize a wikist tree from an event iterable. + * + * The event stream produced by `events()` already contains `enter('root')` + * and `exit('root')`. The tree builder treats those as document boundary + * markers, not as a nested root child node. + * + * ```text + * incoming events (simplified) + * enter(root) + * enter(paragraph) + * text(...) + * exit(paragraph) + * exit(root) + * + * resulting tree + * root + * └─ paragraph + * └─ text + * ``` + * + * Recovery model for malformed streams: + * + * - `token` and `error` events are ignored for AST shape. + * - a mismatched exit closes frames until the matching node is found. + * - EOF with open frames auto-closes those frames using their last known end. + * + * This function never throws on malformed event streams. If the stream ends + * with still-open nodes, it closes them using their last known end position so + * callers still get a usable tree. + */ +export function buildTree( + events: Iterable, + options: BuildTreeOptions, +): WikistRoot { + return materializeTree(events, options, TreeMaterializationPolicy.DEFAULT_HTML_LIKE).tree; +} + +/** + * Materialize the default tolerant wikist tree and keep diagnostics alongside it. + * + * This is the diagnostics-first tree-building path for callers that want the + * same HTML-like default tree shape as {@linkcode buildTree}, plus the + * diagnostics that explain where malformed input was detected. + * + * The returned diagnostics include both event-layer findings preserved from + * earlier parser stages and tree-builder-local findings such as mismatched + * exits or EOF auto-closes. + */ +export function buildTreeWithDiagnostics( + events: Iterable, + options: BuildTreeOptions, +): ParseDiagnosticsResult { + return materializeDiagnosticsTree(events, options, TreeMaterializationPolicy.DEFAULT_HTML_LIKE); +} + +/** + * Materialize a conservative wikist tree and keep diagnostics alongside it. + * + * Use this when a caller wants diagnostics, but does not want recovery-heavy + * wrappers to survive in the final tree unless the source clearly committed to + * them. + */ +export function buildTreeStrict( + events: Iterable, + options: BuildTreeOptions, +): ParseDiagnosticsResult { + return materializeDiagnosticsTree(events, options, TreeMaterializationPolicy.SOURCE_STRICT); +} + +/** + * Materialize a wikist tree and make recovery explicit in the result shape. + * + * This uses the same diagnostics-preserving tree walk as + * {@linkcode buildTreeWithDiagnostics}, but it also reports whether any + * recovery happened while producing that tree. + */ +export function buildTreeWithRecovery( + events: Iterable, + options: BuildTreeOptions, +): ParseResult { + return materializeTree(events, options, TreeMaterializationPolicy.DEFAULT_HTML_LIKE, []); +} + +/** + * Create one in-progress node frame from an enter event. + * + * This captures the opening metadata once so later exit handling only needs to + * decide where the node ends and how children should be attached. + */ +function createFrame(event: Extract): NodeFrame { + return { + kind: 'node', + node_type: event.node_type as Exclude, + props: event.props, + start: event.position.start, + default_end: event.position.end, + accepts_children: acceptsChildren(event.node_type as WikistNodeType), + children: [], + recover_as_text: false, + }; +} + +/** + * Close frames until `node_type` is found or the stack reaches the root frame. + * + * This is the main malformed-stream recovery hook. If exits arrive out of + * order, we still produce a usable tree by finalizing intermediate frames at + * the reported end point. + */ +function closeFrame( + stack: TreeFrame[], + node_type: string, + end: Point, + source: TextSource, + materialization_policy: TreeMaterializationPolicy, + diagnostics?: ParseDiagnostic[], +): void { + while (stack.length > 1) { + const top = stack[stack.length - 1]; + if (top === undefined || top.kind === 'root') return; + + if (top.node_type !== node_type && diagnostics !== undefined) { + if (materialization_policy === TreeMaterializationPolicy.SOURCE_STRICT) { + top.recover_as_text = true; + } + diagnostics.push(mismatchedExitDiagnostic(stack, end, node_type, top.node_type)); + } + + const frame = stack.pop(); + if (frame === undefined || frame.kind === 'root') return; + + appendChild(stack, finalizeFrame(frame, end, source)); + if (frame.node_type === node_type) return; + } + + if (diagnostics !== undefined) { + diagnostics.push(orphanExitDiagnostic(stack, end, node_type)); + } +} + +/** + * Attach one finished child node to the nearest frame that accepts children. + * + * The walk is from top of stack toward root, so nested nodes are attached to + * the closest still-open parent first. + */ +function appendChild(stack: TreeFrame[], node: WikistNode): void { + for (let index = stack.length - 1; index >= 0; index--) { + const frame = stack[index]; + if (frame.kind === 'root' || frame.accepts_children) { + frame.children.push(node); + return; + } + } +} + +/** + * Convert an in-progress frame into a concrete wikist node. + * + * Parent-like node types receive `children`; literal node types receive a + * string `value`; remaining node types are materialized as void-style objects + * with props and position only. + */ +function finalizeFrame(frame: NodeFrame, end: Point, source: TextSource): WikistNode { + const position: Position = { + start: frame.start, + end, + }; + + if (frame.recover_as_text) { + return { + type: 'text', + value: slice(source, frame.start.offset, end.offset), + position, + }; + } + + if (frame.accepts_children) { + return Object.assign( + { + type: frame.node_type, + children: frame.children, + position, + }, + frame.props, + ) as WikistNode; + } + + if (isLiteralValueNode(frame.node_type)) { + return { + type: frame.node_type, + value: readStringProp(frame.props, 'value'), + position, + } as WikistNode; + } + + return Object.assign( + { + type: frame.node_type, + position, + }, + frame.props, + ) as WikistNode; +} + +/** + * Finalize the top-level root node. + * + * Priority order for root position: + * + * 1. explicit `enter('root')` / `exit('root')` points from the stream + * 2. fallback to first/last child positions when explicit root boundaries are + * absent + * 3. omit `position` for empty roots with no explicit boundaries + */ +function finalizeRoot(root: RootFrame): WikistRoot { + if (root.start_point !== undefined && root.end_point !== undefined) { + return { + type: 'root', + children: root.children, + position: { + start: root.start_point, + end: root.end_point, + }, + }; + } + + if (root.children.length === 0) { + return { type: 'root', children: [] }; + } + + const first = root.children[0]; + const last = root.children[root.children.length - 1]; + + if (first.position === undefined || last.position === undefined) { + return { type: 'root', children: root.children }; + } + + return { + type: 'root', + children: root.children, + position: { + start: first.position.start, + end: last.position.end, + }, + }; +} + +/** Read a string property from enter-event props with a safe empty fallback. */ +function readStringProp( + props: Readonly>, + key: string, +): string { + const value = props[key]; + return typeof value === 'string' ? value : ''; +} + +/** + * Build the tree once and optionally capture diagnostics during the same walk. + * + * Keeping the shared traversal here ensures `buildTree()`, + * `buildTreeWithDiagnostics()`, `buildTreeStrict()`, and + * `buildTreeWithRecovery()` stay structurally identical apart from their final + * materialization policy. The diagnostics path only pays extra work when a + * collector array is provided. + */ +function materializeTree( + events: Iterable, + options: BuildTreeOptions, + materialization_policy: TreeMaterializationPolicy, + diagnostics?: ParseDiagnostic[], +): ParseResult { + const root: RootFrame = { kind: 'root', children: [] }; + const stack: TreeFrame[] = [root]; + + for (const event of events) { + switch (event.kind) { + case 'enter': + if (event.node_type === 'root') { + root.start_point = event.position.start; + root.end_point = event.position.end; + break; + } + stack.push(createFrame(event)); + break; + + case 'exit': + if (event.node_type === 'root') { + root.end_point = event.position.end; + break; + } + closeFrame( + stack, + event.node_type, + event.position.end, + options.source, + materialization_policy, + diagnostics, + ); + break; + + case 'text': + appendChild(stack, { + type: 'text', + value: slice(options.source, event.start_offset, event.end_offset), + position: event.position, + }); + break; + + case 'error': + if (diagnostics !== undefined) { + if (materialization_policy === TreeMaterializationPolicy.SOURCE_STRICT) { + markCurrentFrameForSourceStrictText(stack, event.code); + } + diagnostics.push(parseDiagnosticFromEvent(event, stack)); + } + break; + + case 'token': + break; + } + } + + while (stack.length > 1) { + const top = stack[stack.length - 1]; + if (top === undefined || top.kind === 'root') break; + + if (diagnostics !== undefined) { + if (materialization_policy === TreeMaterializationPolicy.SOURCE_STRICT) { + top.recover_as_text = true; + } + diagnostics.push(eofAutocloseDiagnostic(stack, top.default_end, top.node_type)); + } + + const frame = stack.pop(); + if (frame === undefined || frame.kind === 'root') break; + appendChild(stack, finalizeFrame(frame, frame.default_end, options.source)); + } + + return { + tree: finalizeRoot(root), + recovered: (diagnostics?.length ?? 0) > 0, + diagnostics: diagnostics ?? [], + }; +} + +function materializeDiagnosticsTree( + events: Iterable, + options: BuildTreeOptions, + materialization_policy: TreeMaterializationPolicy, +): ParseDiagnosticsResult { + const result = materializeTree(events, options, materialization_policy, []); + + if (materialization_policy !== TreeMaterializationPolicy.SOURCE_STRICT) { + return { + tree: result.tree, + diagnostics: result.diagnostics, + }; + } + + const stripped_paths = strippedDiagnosticPaths(result.diagnostics); + + return stripped_paths.length === 0 + ? { + tree: result.tree, + diagnostics: result.diagnostics, + } + : { + tree: result.tree, + diagnostics: retargetDiagnostics(result.diagnostics, stripped_paths), + }; +} + +function strippedDiagnosticPaths( + diagnostics: readonly ParseDiagnostic[], +): readonly (readonly number[])[] { + const raw_paths = diagnostics + .flatMap((diagnostic) => { + if (diagnostic.anchor.kind !== 'tree-path') return []; + if (diagnostic.anchor.path.length === 0) return []; + if (!shouldStripRecoveredNode(diagnostic.code)) return []; + return [Array.from(diagnostic.anchor.path)]; + }) + .sort(compareTreePaths); + + const filtered_paths: number[][] = []; + for (const path of raw_paths) { + if (filtered_paths.some((selected_path) => isTreePathPrefix(selected_path, path))) { + continue; + } + filtered_paths.push(path); + } + + return filtered_paths; +} + +function shouldStripRecoveredNode(code?: KnownDiagnosticCode | string): boolean { + switch (code) { + case DiagnosticCode.UNCLOSED_TABLE: + case DiagnosticCode.INLINE_TAG_MISSING_CLOSE: + case DiagnosticCode.TREE_MISMATCHED_EXIT: + case DiagnosticCode.TREE_EOF_AUTOCLOSE: + return true; + + default: + return false; + } +} + +function compareTreePaths(left: readonly number[], right: readonly number[]): number { + if (left.length !== right.length) { + return left.length - right.length; + } + + for (let index = 0; index < left.length; index++) { + const difference = left[index] - right[index]; + if (difference !== 0) return difference; + } + + return 0; +} + +function isTreePathPrefix(prefix: readonly number[], path: readonly number[]): boolean { + if (prefix.length > path.length) return false; + + for (let index = 0; index < prefix.length; index++) { + if (prefix[index] !== path[index]) return false; + } + + return true; +} + +function retargetDiagnostics( + diagnostics: readonly ParseDiagnostic[], + stripped_paths: readonly (readonly number[])[], +): readonly ParseDiagnostic[] { + let changed = false; + const next_diagnostics = diagnostics.map((diagnostic) => { + const next_anchor = retargetDiagnosticAnchor(diagnostic.anchor, stripped_paths); + if (next_anchor === diagnostic.anchor) { + return diagnostic; + } + + changed = true; + return Object.assign({}, diagnostic, { anchor: next_anchor }); + }); + + return changed ? next_diagnostics : diagnostics; +} + +function retargetDiagnosticAnchor( + anchor: ParseDiagnosticAnchor, + stripped_paths: readonly (readonly number[])[], +): ParseDiagnosticAnchor { + if (anchor.kind !== 'tree-path') { + return anchor; + } + + const replacement_path = nearestStrippedAncestorPath(anchor.path, stripped_paths); + if (replacement_path === undefined) { + return anchor; + } + + return { + kind: 'tree-path', + path: Array.from(replacement_path), + node_type: 'text', + }; +} + +function nearestStrippedAncestorPath( + path: readonly number[], + stripped_paths: readonly (readonly number[])[], +): readonly number[] | undefined { + let nearest_path: readonly number[] | undefined; + + for (const stripped_path of stripped_paths) { + if (!isTreePathPrefix(stripped_path, path)) { + continue; + } + + if (nearest_path === undefined || stripped_path.length > nearest_path.length) { + nearest_path = stripped_path; + } + } + + return nearest_path; +} + +function markCurrentFrameForSourceStrictText( + stack: TreeFrame[], + code?: KnownDiagnosticCode | string, +): void { + if (!shouldStripRecoveredNode(code)) { + return; + } + + const top = stack[stack.length - 1]; + if (top !== undefined && top.kind === 'node') { + top.recover_as_text = true; + } +} + +/** + * Classify node types that own child nodes. + * + * A null-prototype lookup table is a better fit here than `Set` because this + * is a fixed string vocabulary, not a dynamic runtime collection. The lookup + * data stays in one auditable place, `Object.create(null)` removes inherited + * prototype keys, and `Object.hasOwn(...)` keeps the membership check on the + * table's own entries instead of walking the prototype chain. + */ +function acceptsChildren(node_type: WikistNodeType): boolean { + return Object.hasOwn(PARENT_NODE_TYPE_LOOKUP, node_type); +} + +/** + * Return whether a node stores its payload in a `value` field. + * + * These nodes are still represented in the tree, but they do not have child + * nodes. Their content is copied from enter-event props during finalization. + */ +function isLiteralValueNode(node_type: WikistNodeType): boolean { + return Object.hasOwn(LITERAL_VALUE_NODE_LOOKUP, node_type); +} + +/** + * Turn an event-layer `error` event into a tree-oriented diagnostic. + * + * This path preserves parser-stage diagnostics exactly as they were emitted, + * then adds tree-local location metadata. A block or inline parser can report + * the recovery in its own words, and the tree builder adds a narrow tree + * anchor so the consumer can still find the relevant region after tree + * materialization without exposing future edit-stable anchor semantics early. + */ +function parseDiagnosticFromEvent( + event: ErrorEvent, + stack: TreeFrame[], +): ParseDiagnostic { + const anchor = currentDiagnosticAnchor(stack); + + return { + message: event.message, + severity: event.severity, + code: event.code, + recoverable: event.recoverable, + source: event.source, + details: event.details, + position: event.position, + anchor, + }; +} + +/** + * Report that the tree builder had to auto-close an inner node before it could + * honor the requested exit event. + * + * Example malformed event order: + * + * ```text + * enter(paragraph) + * enter(bold) + * exit(paragraph) + * ``` + * + * The builder closes `bold` first, then closes `paragraph`, so the final tree + * stays well-formed. + * + * Consumers might respond by: + * + * - surfacing a warning near the recovered node + * - offering a fix that restores the missing closer for the inner node + * - ignoring it in tolerant rendering where the recovered shape is enough + */ +function mismatchedExitDiagnostic( + stack: TreeFrame[], + point: Point, + expected_node_type: string, + recovered_node_type: string, +): ParseDiagnostic { + return treeRecoveryDiagnostic( + stack, + point, + `Auto-closed ${recovered_node_type} while recovering from exit(${expected_node_type}).`, + DiagnosticCode.TREE_MISMATCHED_EXIT, + { + expected_node_type, + recovered_node_type, + }, + ); +} + +/** + * Report that the tree builder saw an exit event that no longer matched any + * open frame. + * + * Example malformed event order: + * + * ```text + * exit(italic) + * ``` + * + * at a point where the stack already returned to the root. + * + * Consumers can treat this as a structural warning, log it for parser + * debugging, or ignore it if only the recovered tree matters. + */ +function orphanExitDiagnostic( + stack: TreeFrame[], + point: Point, + expected_node_type: string, +): ParseDiagnostic { + return treeRecoveryDiagnostic( + stack, + point, + `Dropped unmatched exit(${expected_node_type}) at the root boundary.`, + DiagnosticCode.TREE_ORPHAN_EXIT, + { expected_node_type }, + ); +} + +/** + * Report that the event stream ended while a node was still open. + * + * Example malformed event order: + * + * ```text + * enter(paragraph) + * text(...) + * EOF + * ``` + * + * The builder closes the node at its last known end point so the final tree is + * still usable. + * + * Consumers may show a warning, offer a fix for the missing closer, or simply + * continue with the recovered tree when best-effort output is acceptable. + */ +function eofAutocloseDiagnostic( + stack: TreeFrame[], + point: Point, + recovered_node_type: string, +): ParseDiagnostic { + return treeRecoveryDiagnostic( + stack, + point, + `Auto-closed ${recovered_node_type} at end of event stream.`, + DiagnosticCode.TREE_EOF_AUTOCLOSE, + { recovered_node_type }, + ); +} + +/** + * Build a tree-stage recovery diagnostic at a zero-width point. + * + * This helper is intentionally small: the detailed, situation-specific TSDoc + * lives on the wrapper helpers for each recovery code so maintainers can read + * the exact failure mode where it is emitted. + */ +function treeRecoveryDiagnostic( + stack: TreeFrame[], + point: Point, + message: string, + code: KnownDiagnosticCode, + details: Readonly>, +): ParseDiagnostic { + const anchor = currentDiagnosticAnchor(stack); + const position = pointPosition(point); + + return { + message, + severity: 'warning', + code, + recoverable: true, + source: 'tree', + details, + position, + anchor, + }; +} + +/** + * Resolve the current narrow diagnostic anchor from the builder stack. + * + * Open frames are not attached to their parents until they close, so the path + * uses the current child counts as the future insertion index for each open + * frame. That gives downstream tools a stable route to the closest node once + * the final tree has been materialized. + */ +function currentDiagnosticAnchor( + stack: TreeFrame[], +): ParseDiagnosticAnchor { + const path: number[] = []; + + if (stack.length === 1) { + return { kind: 'tree-path', path, node_type: 'root' }; + } + + for (let index = 1; index < stack.length; index++) { + const parent = stack[index - 1]; + if (parent.kind === 'root' || parent.accepts_children) { + path.push(parent.children.length); + } + } + + const top = stack[stack.length - 1]; + return { + kind: 'tree-path', + path, + node_type: top.kind === 'root' ? 'root' : top.node_type, + }; +} + +/** Build a zero-width position at one point. */ +function pointPosition(point: Point): Position { + return { start: point, end: point }; +} \ No newline at end of file