Something went wrong. Try again.
sentence embeddings in pure zig: bge-small with an HF-exact tokenizer and an SME matmul
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100# /// script# requires-python = ">=3.12"# dependencies = ["tokenizers>=0.21"]# ///"""Generate src/unicode_table.zig by asking HF tokenizers' own BertNormalizer andBertPreTokenizer about every scalar value, instead of re-deriving their rules.
usage: uv run scripts/gen_unicode.py > src/unicode_table.zig"""
import tokenizersfrom tokenizers.normalizers import BertNormalizerfrom tokenizers.pre_tokenizers import BertPreTokenizer
norm = BertNormalizer(clean_text=True, handle_chinese_chars=True, strip_accents=None, lowercase=True)pre = BertPreTokenizer()
SCALARS = [c for c in range(0x110000) if not 0xD800 <= c <= 0xDFFF]
def ranges(cps): out = [] for c in sorted(cps): if out and out[-1][1] + 1 == c: out[-1][1] = c else: out.append([c, c]) return out
def classify(c): pieces = [p for p, _ in pre.pre_tokenize_str("a" + chr(c) + "b")] if pieces == ["a", "b"]: return "space" if pieces == ["a", chr(c), "b"]: return "punct" assert pieces == ["a" + chr(c) + "b"], (hex(c), pieces) return "word"
mapped = {}cjk, removed, to_space = [], [], []for c in SCALARS: out = norm.normalize_str(chr(c)) if out == f" {chr(c)} ": cjk.append(c) elif out == "": removed.append(c) elif out == " " and c != 0x20: to_space.append(c) elif out != chr(c): mapped[c] = out
space, punct = [], []for c in SCALARS: kind = classify(c) if kind == "space": space.append(c) elif kind == "punct": punct.append(c)
blob = []entries = []for c in sorted(mapped): enc = mapped[c].encode("utf-8") entries.append((c, len(bytes(blob)), len(enc))) blob.extend(enc)
def zig_ranges(name, rs): lines = [f"pub const {name} = [_]Range{{"] lines += [f" .{{ .lo = 0x{lo:X}, .hi = 0x{hi:X} }}," for lo, hi in rs] lines.append("};") return "\n".join(lines)
print(f"//! generated by scripts/gen_unicode.py from tokenizers {tokenizers.__version__}; do not edit.")print("//! per-scalar BertNormalizer output (lowercase, NFD accent strip, clean_text) plus")print("//! BertPreTokenizer's whitespace/punctuation classes and the CJK padding set.")print()print("pub const Range = struct { lo: u21, hi: u21 };")print("pub const Mapping = struct { cp: u21, off: u32, len: u8 };")print()print(zig_ranges("cjk", ranges(cjk)))print()print(zig_ranges("removed", ranges(removed)))print()print(zig_ranges("to_space", ranges(to_space)))print()print(zig_ranges("space", ranges(space)))print()print(zig_ranges("punct", ranges(punct)))print()print("pub const mappings = [_]Mapping{")for c, off, n in entries: print(f" .{{ .cp = 0x{c:X}, .off = {off}, .len = {n} }},")print("};")print()print('pub const blob: []const u8 = "' + "".join(f"\\x{b:02x}" for b in blob) + '";')