Something went wrong. Try again.
Anonymize your writing style. Zig WASM engine detects authorship markers, fine-tuned LLM rewrites to remove them. Runs entirely in-browser. fantasma.qstorage.quilibrium.com
wasm privacy qwen zig
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553const std = @import("std");
/// BPE tokenizer for Qwen3.5 models./// Loads from HuggingFace tokenizer.json format./// Uses byte-level BPE (GPT-2 style): bytes are mapped to unicode chars before BPE.pub const Tokenizer = struct { /// Token ID → string mapping for decoding vocab: [][]const u8, vocab_size: u32, /// BPE merge rules: pair → merged token. Stored as a sorted list for lookup. merges: []Merge, merge_count: u32, /// String → Token ID mapping for encoding (sorted for binary search) token_to_id: []TokenEntry, token_count: u32, /// Special token IDs eos_id: u32, im_start_id: u32, im_end_id: u32, /// Allocator used for loading allocator: std.mem.Allocator, /// Backing memory for all strings string_pool: []u8,
const Merge = struct { first: u32, second: u32, result: u32, rank: u32, // lower = higher priority };
const TokenEntry = struct { text: []const u8, id: u32, };
pub fn deinit(self: *Tokenizer) void { self.allocator.free(self.vocab); self.allocator.free(self.merges); self.allocator.free(self.token_to_id); self.allocator.free(self.string_pool); }
/// Load tokenizer from compact binary format (.tkn). /// Much faster than JSON parsing — just flat array reads. pub fn loadFromBinary(allocator: std.mem.Allocator, data: []const u8) !Tokenizer { if (data.len < 32) return error.InvalidFormat;
// Parse header const magic = data[0..4]; if (!std.mem.eql(u8, magic, "TOKN")) return error.InvalidFormat;
const vocab_size = std.mem.readInt(u32, data[4..8], .little); const entry_count = std.mem.readInt(u32, data[8..12], .little); const merge_count = std.mem.readInt(u32, data[12..16], .little); const eos_id = std.mem.readInt(u32, data[16..20], .little); const im_start_id = std.mem.readInt(u32, data[20..24], .little); const im_end_id = std.mem.readInt(u32, data[24..28], .little); const pool_size = std.mem.readInt(u32, data[28..32], .little);
var offset: usize = 32;
// String pool — copy so it's owned by the tokenizer if (offset + pool_size > data.len) return error.InvalidFormat; const string_pool = try allocator.alloc(u8, pool_size); @memcpy(string_pool, data[offset .. offset + pool_size]); offset += pool_size;
// Vocab table: vocab_size × (offset:u32, length:u32) const vtable_size = @as(usize, vocab_size) * 8; if (offset + vtable_size > data.len) return error.InvalidFormat; const vtable = data[offset .. offset + vtable_size]; offset += vtable_size;
const vocab = try allocator.alloc([]const u8, vocab_size); for (0..vocab_size) |i| { const base = i * 8; const str_off = std.mem.readInt(u32, vtable[base..][0..4], .little); const str_len = std.mem.readInt(u32, vtable[base + 4 ..][0..4], .little); if (str_off == 0xFFFFFFFF) { vocab[i] = ""; } else { vocab[i] = string_pool[str_off .. str_off + str_len]; } }
// Token-to-id table: entry_count × (offset:u32, length:u32, id:u32) const ttable_size = @as(usize, entry_count) * 12; if (offset + ttable_size > data.len) return error.InvalidFormat; const ttable = data[offset .. offset + ttable_size]; offset += ttable_size;
const token_to_id = try allocator.alloc(TokenEntry, entry_count); for (0..entry_count) |i| { const base = i * 12; const str_off = std.mem.readInt(u32, ttable[base..][0..4], .little); const str_len = std.mem.readInt(u32, ttable[base + 4 ..][0..4], .little); const id = std.mem.readInt(u32, ttable[base + 8 ..][0..4], .little); token_to_id[i] = .{ .text = string_pool[str_off .. str_off + str_len], .id = id, }; }
// Merges table: merge_count × (first:u32, second:u32, result:u32, rank:u32) const mtable_size = @as(usize, merge_count) * 16; if (offset + mtable_size > data.len) return error.InvalidFormat; const mtable = data[offset .. offset + mtable_size];
const merges = try allocator.alloc(Merge, merge_count); for (0..merge_count) |i| { const base = i * 16; merges[i] = .{ .first = std.mem.readInt(u32, mtable[base..][0..4], .little), .second = std.mem.readInt(u32, mtable[base + 4 ..][0..4], .little), .result = std.mem.readInt(u32, mtable[base + 8 ..][0..4], .little), .rank = std.mem.readInt(u32, mtable[base + 12 ..][0..4], .little), }; }
return Tokenizer{ .vocab = vocab, .vocab_size = vocab_size, .merges = merges, .merge_count = merge_count, .token_to_id = token_to_id, .token_count = entry_count, .eos_id = eos_id, .im_start_id = im_start_id, .im_end_id = im_end_id, .allocator = allocator, .string_pool = string_pool, }; }
/// Load tokenizer from a tokenizer.json file using std.json. pub fn loadFromFile(allocator: std.mem.Allocator, path: []const u8) !Tokenizer { const file = try std.fs.cwd().openFile(path, .{}); defer file.close();
const file_size = try file.getEndPos(); const data = try allocator.alloc(u8, file_size); defer allocator.free(data); const bytes_read = try file.readAll(data); if (bytes_read != file_size) return error.IncompleteRead;
return loadFromJson(allocator, data[0..bytes_read]); }
pub fn loadFromJson(allocator: std.mem.Allocator, json_data: []const u8) !Tokenizer { // Parse using std.json var parsed = try std.json.parseFromSlice(std.json.Value, allocator, json_data, .{ .allocate = .alloc_always, .max_value_len = std.json.default_max_value_len, }); defer parsed.deinit();
const root = parsed.value;
// Extract model section const model = root.object.get("model") orelse return error.InvalidFormat; const vocab_obj = model.object.get("vocab") orelse return error.InvalidFormat; const merges_arr = model.object.get("merges") orelse return error.InvalidFormat;
// Count vocab entries const vocab_map = vocab_obj.object; const num_vocab_entries: u32 = @intCast(vocab_map.count());
// Find max token ID to size the vocab array var max_id: u32 = 0; var it = vocab_map.iterator(); while (it.next()) |entry| { const id: u32 = @intCast(entry.value_ptr.integer); if (id > max_id) max_id = id; }
// Also check added_tokens var num_added: u32 = 0; if (root.object.get("added_tokens")) |added| { for (added.array.items) |tok| { const id: u32 = @intCast(tok.object.get("id").?.integer); if (id > max_id) max_id = id; num_added += 1; } }
const total_vocab: u32 = max_id + 1;
// Allocate string pool - estimate size var pool_size: usize = 0; it = vocab_map.iterator(); while (it.next()) |entry| { pool_size += entry.key_ptr.len + 1; } if (root.object.get("added_tokens")) |added| { for (added.array.items) |tok| { const content = tok.object.get("content").?.string; pool_size += content.len + 1; } }
var string_pool = try allocator.alloc(u8, pool_size); var pool_pos: usize = 0;
// Allocate vocab and token_to_id arrays const vocab = try allocator.alloc([]const u8, total_vocab); for (vocab) |*v| v.* = "";
const total_entries = num_vocab_entries + num_added; const token_to_id = try allocator.alloc(TokenEntry, total_entries); var entry_count: u32 = 0;
// Fill vocab from model.vocab it = vocab_map.iterator(); while (it.next()) |entry| { const text = entry.key_ptr.*; const id: u32 = @intCast(entry.value_ptr.integer);
// Copy string to pool const str_start = pool_pos; @memcpy(string_pool[pool_pos .. pool_pos + text.len], text); pool_pos += text.len; const pooled = string_pool[str_start..pool_pos];
vocab[id] = pooled; token_to_id[entry_count] = .{ .text = pooled, .id = id }; entry_count += 1; }
// Add special/added tokens var eos_id: u32 = 248046; var im_start_id: u32 = 248045; var im_end_id: u32 = 248046;
if (root.object.get("added_tokens")) |added| { for (added.array.items) |tok| { const id: u32 = @intCast(tok.object.get("id").?.integer); const content = tok.object.get("content").?.string;
// Copy to pool const str_start = pool_pos; @memcpy(string_pool[pool_pos .. pool_pos + content.len], content); pool_pos += content.len; const pooled = string_pool[str_start..pool_pos];
if (id < total_vocab) { vocab[id] = pooled; } token_to_id[entry_count] = .{ .text = pooled, .id = id }; entry_count += 1;
// Detect special tokens if (std.mem.eql(u8, content, "<|im_start|>")) im_start_id = id; if (std.mem.eql(u8, content, "<|im_end|>")) im_end_id = id; if (std.mem.eql(u8, content, "<|endoftext|>")) eos_id = id; } }
// Sort token_to_id by text for binary search std.mem.sort(TokenEntry, token_to_id[0..entry_count], {}, struct { fn lessThan(_: void, a: TokenEntry, b: TokenEntry) bool { return std.mem.order(u8, a.text, b.text) == .lt; } }.lessThan);
// Parse merges const merges_items = merges_arr.array.items; const num_merges: u32 = @intCast(merges_items.len); const merges = try allocator.alloc(Merge, num_merges); var merge_count: u32 = 0;
for (merges_items, 0..) |merge_val, rank| { // Merges can be either "token1 token2" (string) or ["token1", "token2"] (array) var first_str: []const u8 = undefined; var second_str: []const u8 = undefined;
switch (merge_val) { .string => |merge_str| { const space_idx = std.mem.indexOf(u8, merge_str, " ") orelse continue; first_str = merge_str[0..space_idx]; second_str = merge_str[space_idx + 1 ..]; }, .array => |arr| { if (arr.items.len != 2) continue; first_str = arr.items[0].string; second_str = arr.items[1].string; }, else => continue, }
const first_id = lookupTokenId(token_to_id[0..entry_count], first_str) orelse continue; const second_id = lookupTokenId(token_to_id[0..entry_count], second_str) orelse continue;
// The merged result is the concatenation const merged_str_buf = [_]u8{0} ** 0; _ = merged_str_buf;
// Look up merged token var merged_buf: [512]u8 = undefined; if (first_str.len + second_str.len > merged_buf.len) continue; @memcpy(merged_buf[0..first_str.len], first_str); @memcpy(merged_buf[first_str.len .. first_str.len + second_str.len], second_str); const merged_key = merged_buf[0 .. first_str.len + second_str.len];
const result_id = lookupTokenId(token_to_id[0..entry_count], merged_key) orelse continue;
merges[merge_count] = .{ .first = first_id, .second = second_id, .result = result_id, .rank = @intCast(rank), }; merge_count += 1; }
return Tokenizer{ .vocab = vocab, .vocab_size = total_vocab, .merges = merges, .merge_count = merge_count, .token_to_id = token_to_id, .token_count = entry_count, .eos_id = eos_id, .im_start_id = im_start_id, .im_end_id = im_end_id, .allocator = allocator, .string_pool = string_pool, }; }
/// Encode text to token IDs using BPE. pub fn encode(self: *const Tokenizer, text: []const u8, allocator: std.mem.Allocator, out: []u32) !u32 { // Step 1: Convert text bytes to byte-level BPE characters var byte_tokens = std.ArrayListUnmanaged(u32){}; defer byte_tokens.deinit(allocator);
// Map each byte to its corresponding single-byte token for (text) |byte| { const ch = byteToBpeChar(byte); var buf: [4]u8 = undefined; const len = std.unicode.utf8Encode(ch, &buf) catch continue; const char_str = buf[0..len];
if (lookupTokenId(self.token_to_id[0..self.token_count], char_str)) |id| { try byte_tokens.append(allocator, id); } }
if (byte_tokens.items.len == 0) return 0;
// Step 2: Apply BPE merges greedily // Repeatedly find the highest-priority merge and apply it var tokens = byte_tokens.items; var len: u32 = @intCast(tokens.len);
var changed = true; while (changed) { changed = false; var best_rank: u32 = std.math.maxInt(u32); var best_pos: u32 = 0; var best_result: u32 = 0;
// Find the best merge var i: u32 = 0; while (i + 1 < len) : (i += 1) { if (self.findMerge(tokens[i], tokens[i + 1])) |merge| { if (merge.rank < best_rank) { best_rank = merge.rank; best_pos = i; best_result = merge.result; } } }
if (best_rank < std.math.maxInt(u32)) { // Apply merge: replace tokens[best_pos] and tokens[best_pos+1] with best_result tokens[best_pos] = best_result; // Shift remaining tokens left var j: u32 = best_pos + 1; while (j + 1 < len) : (j += 1) { tokens[j] = tokens[j + 1]; } len -= 1; changed = true; } }
// Copy to output const out_len = @min(len, @as(u32, @intCast(out.len))); @memcpy(out[0..out_len], tokens[0..out_len]); return out_len; }
/// Encode a ChatML-wrapped prompt. /// Format: <|im_start|>system\n{system}<|im_end|>\n<|im_start|>user\n{user}<|im_end|>\n<|im_start|>assistant\n pub fn encodeChatML( self: *const Tokenizer, system_prompt: ?[]const u8, user_prompt: []const u8, allocator: std.mem.Allocator, out: []u32, ) !u32 { var pos: u32 = 0;
if (system_prompt) |sys| { out[pos] = self.im_start_id; pos += 1; pos += try self.encode("system\n", allocator, out[pos..]); pos += try self.encode(sys, allocator, out[pos..]); out[pos] = self.im_end_id; pos += 1; pos += try self.encode("\n", allocator, out[pos..]); }
out[pos] = self.im_start_id; pos += 1; pos += try self.encode("user\n", allocator, out[pos..]); pos += try self.encode(user_prompt, allocator, out[pos..]); out[pos] = self.im_end_id; pos += 1; pos += try self.encode("\n", allocator, out[pos..]);
out[pos] = self.im_start_id; pos += 1; pos += try self.encode("assistant\n", allocator, out[pos..]);
return pos; }
/// Decode a single token ID to its text representation. pub fn decode(self: *const Tokenizer, token_id: u32) []const u8 { if (token_id >= self.vocab_size) return ""; return self.vocab[token_id]; }
/// Decode a token and convert byte-level BPE chars back to actual bytes. pub fn decodeToBytes(self: *const Tokenizer, token_id: u32, out: []u8) u32 { const text = self.decode(token_id); var pos: u32 = 0;
var i: usize = 0; while (i < text.len) { const cp_len = std.unicode.utf8ByteSequenceLength(text[i]) catch { if (pos < out.len) { out[pos] = text[i]; pos += 1; } i += 1; continue; }; if (i + cp_len > text.len) break; const cp = std.unicode.utf8Decode(text[i .. i + cp_len]) catch { if (pos < out.len) { out[pos] = text[i]; pos += 1; } i += 1; continue; };
if (bpeCharToByte(cp)) |byte| { if (pos < out.len) { out[pos] = byte; pos += 1; } } i += cp_len; }
return pos; }
fn findMerge(self: *const Tokenizer, first: u32, second: u32) ?*const Merge { // Linear scan (could use hash map for speed, but this is simple) for (self.merges[0..self.merge_count]) |*m| { if (m.first == first and m.second == second) return m; } return null; }};
fn lookupTokenId(entries: []const Tokenizer.TokenEntry, text: []const u8) ?u32 { // Binary search var lo: usize = 0; var hi: usize = entries.len; while (lo < hi) { const mid = lo + (hi - lo) / 2; const cmp = std.mem.order(u8, entries[mid].text, text); switch (cmp) { .lt => lo = mid + 1, .gt => hi = mid, .eq => return entries[mid].id, } } return null;}
// ============================================================// GPT-2 byte-level BPE character mapping// ============================================================
/// Map a byte value to the corresponding unicode codepoint used in GPT-2 BPE./// Printable ASCII bytes map to themselves; others map to a range starting at U+0100.fn byteToBpeChar(byte: u8) u21 { return byte_to_unicode_table[byte];}
/// Reverse mapping: unicode codepoint back to byte.fn bpeCharToByte(cp: u21) ?u8 { if (cp < 256) { // Check if this is a direct-mapped printable char if (byte_to_unicode_table[cp] == cp) return @intCast(cp); } // Search in the table for mapped values for (byte_to_unicode_table, 0..) |mapped, byte| { if (mapped == cp) return @intCast(byte); } return null;}
/// GPT-2 byte_to_unicode mapping table./// Printable chars (33-126, 161-172, 174-255) map to themselves./// Other bytes (0-32, 127-160, 173) map to U+0100 onwards.const byte_to_unicode_table: [256]u21 = blk: { var table: [256]u21 = undefined; var n: u21 = 0x100; // Start of remapped range
for (0..256) |i| { const b: u8 = @intCast(i); if ((b >= 33 and b <= 126) or (b >= 161 and b <= 172) or (b >= 174 and b <= 255)) { table[i] = b; } else { table[i] = n; n += 1; } } break :blk table;};
test "byte to bpe char roundtrip" { // Space (0x20 = 32) should map to Ġ (U+0120) const space_char = byteToBpeChar(' '); try std.testing.expectEqual(@as(u21, 0x0120), space_char); const back = bpeCharToByte(0x0120); try std.testing.expectEqual(@as(?u8, ' '), back);
// 'A' (65) should map to itself const a_char = byteToBpeChar('A'); try std.testing.expectEqual(@as(u21, 'A'), a_char); const a_back = bpeCharToByte('A'); try std.testing.expectEqual(@as(?u8, 'A'), a_back);}