import { localeFor } from "@/lib/model/locus"
import type { Lang, Token } from "@/lib/schemas/model"

/**
 * Word tokenization over the clean `text` (DATA-ARCHITECTURE.md §1.3).
 *
 * `Intl.Segmenter` with word granularity handles polytonic Greek (combining
 * diacritics), Latin, and German uniformly. Only word-like segments become
 * tokens — whitespace and punctuation runs produce none, so a selection over
 * them normalizes to empty (§1.5).
 *
 * The same function runs at ingest (over the canonical text) and in the client
 * (to re-tokenize text patched by a correction overlay, §7), so a corrected
 * span snaps to word boundaries exactly as the original did.
 */
export function tokenize(text: string, lang: Lang): Token[] {
  const segmenter = new Intl.Segmenter(localeFor(lang), { granularity: "word" })
  const tokens: Token[] = []
  for (const seg of segmenter.segment(text)) {
    if (!seg.isWordLike) continue
    tokens.push({
      start: seg.index,
      end: seg.index + seg.segment.length,
      surface: seg.segment,
    })
  }
  return tokens
}