diff --git a/_test_utils/perf_fixtures.ts b/_test_utils/perf_fixtures.ts new file mode 100644 index 0000000..669bd64 --- /dev/null +++ b/_test_utils/perf_fixtures.ts @@ -0,0 +1,523 @@ +import type { WikitextEvent } from '../events.ts'; + +import { + blockEvents, + buildTree, + createSession, + events, + inlineEvents, + outlineEvents, + parse, + tokenize, +} from '../mod.ts'; +import { UNICODE_TEXT_FIXTURES } from './unicode_fixtures.ts'; + +const CC_LF = 0x0a; +const CC_CR = 0x0d; +const CC_TAB = 0x09; +const CC_SPACE = 0x20; +const CC_BANG = 0x21; +const CC_HASH = 0x23; +const CC_AMP = 0x26; +const CC_APOSTROPHE = 0x27; +const CC_ASTERISK = 0x2a; +const CC_DASH = 0x2d; +const CC_SLASH = 0x2f; +const CC_COLON = 0x3a; +const CC_SEMICOLON = 0x3b; +const CC_LT = 0x3c; +const CC_EQUALS = 0x3d; +const CC_GT = 0x3e; +const CC_OPEN_BRACKET = 0x5b; +const CC_CLOSE_BRACKET = 0x5d; +const CC_UNDERSCORE = 0x5f; +const CC_OPEN_BRACE = 0x7b; +const CC_PIPE = 0x7c; +const CC_CLOSE_BRACE = 0x7d; +const CC_TILDE = 0x7e; + +export type RangeState = { + get(name: string): number; +}; + +export function repeatBlock(unit: string, repeat: number): string { + return unit.repeat(repeat); +} + +export function repeatToMinimumSize(unit: string, minimum_size: number): string { + const repeat = Math.ceil(minimum_size / unit.length); + return unit.repeat(repeat); +} + +export function cycleInputs(inputs: readonly T[]): () => T { + let index = 0; + + return () => { + const input = inputs[index]; + index = (index + 1) % inputs.length; + return input; + }; +} + +const PLAIN_PARAGRAPH_UNITS = [ + [ + 'Observational astronomy records atmospheric scattering, continental weathering, and navigational corrections across multiple expeditions.', + 'Field notes preserve calibration details, seasonal drift, and cross-check remarks so later readers can reconstruct the original measurement context.', + ].join(' '), + [ + 'Archival restoration teams compare transcription variants, publication histories, and editorial interventions before they publish a stable reference text.', + 'That workflow emphasizes provenance, reproducibility, and careful language around uncertainty instead of collapsing every disagreement into one canonical sentence.', + ].join(' '), + [ + 'Long-form technical prose often mixes descriptive paragraphs, cautious qualifications, and domain-specific terminology while still remaining ordinary text to the tokenizer.', + 'This fixture aims to model that kind of paragraph payload rather than a short pangram repeated until the benchmark reaches its target size.', + ].join(' '), +] as const; + +const PLAIN_WORD_BOUNDARY_STRESS_UNIT = [ + 'a a a a a a a a a a a a a a a a', + 'b b b b b b b b b b b b b b b b', + 'c c c c c c c c c c c c c c c c', +].join(' '); + +export function repeatParagraphText(unit: string, minimum_size: number): string { + return repeatToMinimumSize(`${unit}\n\n`, minimum_size); +} + +export function repeatWordBoundaryStressText(minimum_size: number): string { + return repeatToMinimumSize(`${PLAIN_WORD_BOUNDARY_STRESS_UNIT}\n`, minimum_size); +} + +export const TOKENIZER_SCAN_DELIMITER = Uint8Array.from({ length: 128 }, (_, code) => { + switch (code) { + case CC_LF: + case CC_CR: + case CC_TAB: + case CC_SPACE: + case CC_BANG: + case CC_HASH: + case CC_AMP: + case CC_APOSTROPHE: + case CC_ASTERISK: + case CC_DASH: + case CC_SLASH: + case CC_COLON: + case CC_SEMICOLON: + case CC_LT: + case CC_EQUALS: + case CC_GT: + case CC_OPEN_BRACKET: + case CC_CLOSE_BRACKET: + case CC_UNDERSCORE: + case CC_OPEN_BRACE: + case CC_PIPE: + case CC_CLOSE_BRACE: + case CC_TILDE: + return 1; + + default: + return 0; + } +}); + +export const PLAIN_TEXT_INPUTS = [ + repeatParagraphText(PLAIN_PARAGRAPH_UNITS[0], 9 * 1024), + repeatParagraphText(PLAIN_PARAGRAPH_UNITS[1], 9 * 1024), + repeatParagraphText(PLAIN_PARAGRAPH_UNITS[2], 9 * 1024), +] as const; + +export const PLAIN_TOKEN_DENSITY_STRESS_INPUTS = [ + repeatWordBoundaryStressText(9 * 1024), + repeatToMinimumSize(`${PLAIN_WORD_BOUNDARY_STRESS_UNIT}\t${PLAIN_WORD_BOUNDARY_STRESS_UNIT}\n`, 9 * 1024), +] as const; + +export const HEADING_TEXT_INPUTS = [ + '== Section ==\nParagraph text here.\n'.repeat(100), + '=== Nested ===\nAnother paragraph line.\n'.repeat(90), +] as const; + +export const TABLE_TEXT_INPUTS = [ + '{|\n! H1 !! H2\n|-\n| A || B\n|-\n| C || D\n|}\n'.repeat(50), + '{| class="wikitable"\n! Name !! Value\n|-\n| Alpha || 1\n|-\n| Beta || 2\n|}\n'.repeat(40), +] as const; + +export const LINK_TEXT_INPUTS = [ + "See [[Main Page|home]], '''bold''' and ''italic'' text.\n".repeat(100), + 'Visit [[Earth|planet]] with [https://example.com source] and & notes.\n'.repeat(80), +] as const; + +export const TEMPLATE_TEXT_INPUTS = [ + '{{Infobox|name={{{1}}}|value={{{2|default}}}}}\n'.repeat(100), + '{{Card|title={{PAGENAME}}|body={{{content|fallback}}}}}\n'.repeat(85), +] as const; + +export const MIXED_TEXT_INPUTS = [ + [ + '== Heading ==', + "'''Bold''' and ''italic'' and '''''both'''''.", + '* Bullet item', + '# Ordered item', + ': Indented', + '{|', + '! Header', + '|-', + '| [[Page|link]] || {{template|arg=val}}', + '|}', + '----', + '', + '& { 💩', + '~~~~ __TOC__', + '', + ].join('\n').repeat(50), + [ + '== Another Heading ==', + "A [[Main Page|home]] link with ''italic'' and '''bold'''.", + '; Term', + ': Definition', + '{|', + '! Name !! Count', + '|-', + '| {{Item|name=Alpha}} || 42', + '|}', + 'inline', + '<escaped> ~~ ~~ __TOC__', + '', + ].join('\n').repeat(48), +] as const; + +export const PATHOLOGICAL_TEXT_INPUTS = [ + [ + '[[[[{{{{