Something went wrong. Try again.
Original-language reader for Kant, Aristotle & more: select any passage to gloss, translate, or hear it read aloud, with exact citations.
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829import { localeFor } from "@/lib/model/locus"import type { Lang, Token } from "@/lib/schemas/model"
/** * Word tokenization over the clean `text` (DATA-ARCHITECTURE.md §1.3). * * `Intl.Segmenter` with word granularity handles polytonic Greek (combining * diacritics), Latin, and German uniformly. Only word-like segments become * tokens — whitespace and punctuation runs produce none, so a selection over * them normalizes to empty (§1.5). * * The same function runs at ingest (over the canonical text) and in the client * (to re-tokenize text patched by a correction overlay, §7), so a corrected * span snaps to word boundaries exactly as the original did. */export function tokenize(text: string, lang: Lang): Token[] { const segmenter = new Intl.Segmenter(localeFor(lang), { granularity: "word" }) const tokens: Token[] = [] for (const seg of segmenter.segment(text)) { if (!seg.isWordLike) continue tokens.push({ start: seg.index, end: seg.index + seg.segment.length, surface: seg.segment, }) } return tokens}