# /// script
# requires-python = ">=3.12"
# dependencies = ["tokenizers>=0.21"]
# ///
"""Generate src/unicode_table.zig by asking HF tokenizers' own BertNormalizer and
BertPreTokenizer about every scalar value, instead of re-deriving their rules.

usage: uv run scripts/gen_unicode.py > src/unicode_table.zig
"""

import tokenizers
from tokenizers.normalizers import BertNormalizer
from tokenizers.pre_tokenizers import BertPreTokenizer

norm = BertNormalizer(clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True)
pre = BertPreTokenizer()

SCALARS = [c for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF]


def ranges(cps):
    out = []
    for c in sorted(cps):
        if out and out[-1][1] + 1 == c:
            out[-1][1] = c
        else:
            out.append([c, c])
    return out


def classify(c):
    pieces = [p for p, _ in pre.pre_tokenize_str("a" + chr(c) + "b")]
    if pieces == ["a", "b"]:
        return "space"
    if pieces == ["a", chr(c), "b"]:
        return "punct"
    assert pieces == ["a" + chr(c) + "b"], (hex(c), pieces)
    return "word"


mapped = {}
cjk, removed, to_space = [], [], []
for c in SCALARS:
    out = norm.normalize_str(chr(c))
    if out == f" {chr(c)} ":
        cjk.append(c)
    elif out == "":
        removed.append(c)
    elif out == " " and c != 0x20:
        to_space.append(c)
    elif out != chr(c):
        mapped[c] = out

space, punct = [], []
for c in SCALARS:
    kind = classify(c)
    if kind == "space":
        space.append(c)
    elif kind == "punct":
        punct.append(c)

blob = []
entries = []
for c in sorted(mapped):
    enc = mapped[c].encode("utf-8")
    entries.append((c, len(bytes(blob)), len(enc)))
    blob.extend(enc)


def zig_ranges(name, rs):
    lines = [f"pub const {name} = [_]Range{{"]
    lines += [f"    .{{ .lo = 0x{lo:X}, .hi = 0x{hi:X} }}," for lo, hi in rs]
    lines.append("};")
    return "\n".join(lines)


print(f"//! generated by scripts/gen_unicode.py from tokenizers {tokenizers.__version__}; do not edit.")
print("//! per-scalar BertNormalizer output (lowercase, NFD accent strip, clean_text) plus")
print("//! BertPreTokenizer's whitespace/punctuation classes and the CJK padding set.")
print()
print("pub const Range = struct { lo: u21, hi: u21 };")
print("pub const Mapping = struct { cp: u21, off: u32, len: u8 };")
print()
print(zig_ranges("cjk", ranges(cjk)))
print()
print(zig_ranges("removed", ranges(removed)))
print()
print(zig_ranges("to_space", ranges(to_space)))
print()
print(zig_ranges("space", ranges(space)))
print()
print(zig_ranges("punct", ranges(punct)))
print()
print("pub const mappings = [_]Mapping{")
for c, off, n in entries:
    print(f"    .{{ .cp = 0x{c:X}, .off = {off}, .len = {n} }},")
print("};")
print()
print('pub const blob: []const u8 = "' + "".join(f"\\x{b:02x}" for b in blob) + '";')