Something went wrong. Try again.
A fork of https://github.com/crosspoint-reader/crosspoint-reader
Something went wrong. Try again.
45 kB · 1079 lines
Python
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080#!python3import zlibimport sysimport reimport mathimport argparsefrom collections import namedtuple
# Force UTF-8 stdout so that `python fontconvert.py … > foo.h` on Windows# (default cp1252) doesn't emit UTF-16 LE / replacement chars in the generated# header. Wrapped in a hasattr guard so it's a no-op on older Pythons.if hasattr(sys.stdout, 'reconfigure'): sys.stdout.reconfigure(encoding='utf-8')
# Originally from https://github.com/vroland/epdiy
parser = argparse.ArgumentParser(description="Generate a header file from a font to be used with epdiy.")parser.add_argument("name", action="store", help="name of the font.")parser.add_argument("size", type=int, help="font size to use.")parser.add_argument("fontstack", action="store", nargs='+', help="list of font files, ordered by descending priority.")parser.add_argument("--2bit", dest="is2Bit", action="store_true", help="generate 2-bit greyscale bitmap instead of 1-bit black and white.")parser.add_argument("--additional-intervals", dest="additional_intervals", action="append", help="Additional code point intervals to export as min,max. This argument can be repeated.")parser.add_argument("--compress", dest="compress", action="store_true", help="Compress glyph bitmaps using DEFLATE with group-based compression.")parser.add_argument("--zopfli", dest="zopfli", action="store_true", help="Use Zopfli for the DEFLATE backend instead of zlib. Produces standard raw-DEFLATE streams (decoded unchanged by the on-device uzlib inflater), typically a few percent smaller than zlib -9, at the cost of much slower compression. Requires --compress and the 'zopfli' package.")parser.add_argument("--force-autohint", dest="force_autohint", action="store_true", help="Force FreeType auto-hinter instead of native font hinting. Improves stem width consistency for fonts with weak or no native TrueType hints.")parser.add_argument("--pnum", dest="pnum", action="store_true", help="Use proportional numerals (pnum OpenType feature) instead of default tabular figures. Reduces visual gaps between digits in running prose.")args = parser.parse_args()
import freetypefrom fontTools.ttLib import TTFont
GlyphProps = namedtuple("GlyphProps", ["width", "height", "advance_x", "left", "top", "data_length", "data_offset", "code_point"])
font_stack = [freetype.Face(f) for f in args.fontstack]is2Bit = args.is2Bitsize = args.sizefont_name = args.nameload_flags = freetype.FT_LOAD_RENDERif args.force_autohint: load_flags |= freetype.FT_LOAD_FORCE_AUTOHINT
# inclusive unicode code point intervals# must not overlap and be in ascending orderintervals = [ ### Basic Latin ### # ASCII letters, digits, punctuation, control characters (0x0000, 0x007F), ### Latin-1 Supplement ### # Accented characters for Western European languages (0x0080, 0x00FF), ### Latin Extended-A ### # Eastern European and Baltic languages (0x0100, 0x017F), ### Latin Extended-B (Vietnamese subset only) ### # Only Ơ/ơ (U+01A0-01A1), Ư/ư (U+01AF-01B0) for Vietnamese (0x01A0, 0x01A1), (0x01AF, 0x01B0), ### Latin Extended-B (European subset only) ### # Croatian digraphs (DŽ/Lj/Nj), Pinyin caron variants, # European diacritical variants, Romanian (Ș/ș/Ț/ț) (0x01C4, 0x021F), ### Vietnamese Extended ### # All precomposed Vietnamese characters with tone marks # Ả Ấ Ầ Ẩ Ẫ Ậ Ắ Ằ Ẳ Ẵ Ặ Ẹ Ẻ Ẽ Ế Ề Ể Ễ Ệ Ỉ Ị Ọ Ỏ Ố Ồ Ổ Ỗ Ộ Ớ Ờ Ở Ỡ Ợ Ụ Ủ Ứ Ừ Ử Ữ Ự Ỳ Ỵ Ỷ Ỹ (0x1EA0, 0x1EF9), ### General Punctuation (core subset) ### # Smart quotes, en dash, em dash, ellipsis, NO-BREAK SPACE (0x2000, 0x206F), ### Basic Symbols From "Latin-1 + Misc" ### # dashes, quotes, prime marks (0x2010, 0x203A), # misc punctuation (0x2040, 0x205F), # common currency symbols (0x20A0, 0x20CF), ### Combining Diacritical Marks (minimal subset) ### # Needed for proper rendering of many extended Latin languages (0x0300, 0x036F), ### Greek & Coptic ### # Used in science, maths, philosophy, some academic texts # (0x0370, 0x03FF), ### Cyrillic ### # Russian, Ukrainian, Bulgarian, etc. (0x0400, 0x04FF), ### Math Symbols (common subset) ### # Superscripts and Subscripts (0x2070, 0x209F), # General math operators (0x2200, 0x22FF), # Arrows (0x2190, 0x21FF), ### CJK ### # Core Unified Ideographs # (0x4E00, 0x9FFF), # # Extension A # (0x3400, 0x4DBF), # # Extension B # (0x20000, 0x2A6DF), # # Extension C–F # (0x2A700, 0x2EBEF), # # Extension G # (0x30000, 0x3134F), # # Hiragana # (0x3040, 0x309F), # # Katakana # (0x30A0, 0x30FF), # # Katakana Phonetic Extensions # (0x31F0, 0x31FF), # # Halfwidth Katakana # (0xFF60, 0xFF9F), # # Hangul Syllables # (0xAC00, 0xD7AF), # # Hangul Jamo # (0x1100, 0x11FF), # # Hangul Compatibility Jamo # (0x3130, 0x318F), # # Hangul Jamo Extended-A # (0xA960, 0xA97F), # # Hangul Jamo Extended-B # (0xD7B0, 0xD7FF), # # CJK Radicals Supplement # (0x2E80, 0x2EFF), # # Kangxi Radicals # (0x2F00, 0x2FDF), # # CJK Symbols and Punctuation # (0x3000, 0x303F), # # CJK Compatibility Forms # (0xFE30, 0xFE4F), # # CJK Compatibility Ideographs # (0xF900, 0xFAFF), ### Alphabetic Presentation Forms (Latin ligatures) ### # ff, fi, fl, ffi, ffl, long-st, st (0xFB00, 0xFB06), ### Specials # Replacement Character (0xFFFD, 0xFFFD),]
add_ints = []if args.additional_intervals: add_ints = [tuple([int(n, base=0) for n in i.split(",")]) for i in args.additional_intervals]
def norm_floor(val): return int(math.floor(val / (1 << 6)))
def norm_ceil(val): return int(math.ceil(val / (1 << 6)))
# Fixed-point (fp4) output conventions (must match EpdFontData.h / fp4 namespace):## advanceX 12.4 unsigned fixed-point (uint16_t).# 12 integer bits, 4 fractional bits = 1/16-pixel resolution.# Encoded from FreeType's 16.16 linearHoriAdvance.## kernMatrix 4.4 signed fixed-point (int8_t).# 4 integer bits, 4 fractional bits = 1/16-pixel resolution.# Range: -8.0 to +7.9375 pixels.# Encoded from font design-unit kerning values.## Both share 4 fractional bits so the renderer can add them directly into a# single int32_t accumulator and defer rounding until pixel placement.
def fp4_from_ft16_16(val): """Convert FreeType 16.16 fixed-point to 12.4 fixed-point with rounding.""" return (val + (1 << 11)) >> 12
def fp4_from_design_units(du, scale): """Convert a font design-unit value to 4.4 fixed-point, clamped to int8_t.
Multiplies by scale (ppem / units_per_em) and shifts into 4 fractional bits. The result is rounded to nearest and clamped to [-128, 127]. """ raw = round(du * scale * 16) return max(-128, min(127, raw))
def deflate_raw(data): """Raw-DEFLATE compress `data` (no zlib/gzip wrapper), decodable on-device by uzlib via inflate(wbits=-15). Uses Zopfli when --zopfli is set, else zlib -9.
Zopfli is a drop-in stronger DEFLATE encoder: the output is an ordinary DEFLATE stream, so nothing on the device changes -- the same inflater decodes it at the same speed. It is only much slower to compress, which is free here because this runs at font-generation time.
Zopfli's Python binding emits zlib-wrapped output, so strip the 2-byte header and 4-byte adler32 trailer to recover the raw block. The round-trip check guards that wrapper format -- raised rather than asserted, because `python -O` strips asserts and this one decides whether the font data we emit is decodable at all. """ if args.zopfli: import zopfli.zlib wrapped = zopfli.zlib.compress(bytes(data)) raw = wrapped[2:-4] if zlib.decompress(raw, -15) != bytes(data): raise RuntimeError("zopfli raw-DEFLATE round-trip failed; refusing to emit undecodable font data") return raw compressor = zlib.compressobj(level=9, wbits=-15) return compressor.compress(bytes(data)) + compressor.flush()
def chunks(l, n): for i in range(0, len(l), n): yield l[i:i + n]
def extract_pnum_subs(font_path): """Extract pnum (proportional figures) GSUB substitutions.
Parses the font's GSUB table for the 'pnum' feature, which replaces tabular-width figure glyphs with proportional-width alternates. Returns {original_glyph_name: substitute_glyph_name} or empty dict. """ font = TTFont(font_path) subs = {} if 'GSUB' not in font: font.close() return subs gsub = font['GSUB'].table pnum_indices = set() if gsub.FeatureList: for fr in gsub.FeatureList.FeatureRecord: if fr.FeatureTag == 'pnum': pnum_indices.update(fr.Feature.LookupListIndex) for li in pnum_indices: lookup = gsub.LookupList.Lookup[li] for st in lookup.SubTable: actual = st if lookup.LookupType == 7 and hasattr(st, 'ExtSubTable'): actual = st.ExtSubTable if hasattr(actual, 'mapping'): subs.update(actual.mapping) font.close() return subs
# Build proportional numeral glyph overrides when --pnum is active.# Maps (face_index, codepoint) -> freetype glyph index for the proportional alternate.pnum_glyph_overrides = {}pnum_kern_subs = {} # face_index -> {original_glyph_name: substitute_glyph_name}if args.pnum: for face_idx, font_path in enumerate(args.fontstack): subs = extract_pnum_subs(font_path) if not subs: continue pnum_kern_subs[face_idx] = subs tt_font = TTFont(font_path) cmap = tt_font.getBestCmap() or {} glyph_order = tt_font.getGlyphOrder() name_to_glyph_idx = {name: idx for idx, name in enumerate(glyph_order)} count = 0 for cp, glyph_name in cmap.items(): if glyph_name in subs: sub_name = subs[glyph_name] sub_idx = name_to_glyph_idx.get(sub_name, 0) if sub_idx > 0: pnum_glyph_overrides[(face_idx, cp)] = sub_idx count += 1 tt_font.close() if count > 0: print(f"pnum: {count} glyph substitutions from {font_path}", file=sys.stderr)
def load_glyph(code_point): face_index = 0 while face_index < len(font_stack): face = font_stack[face_index] glyph_index = pnum_glyph_overrides.get((face_index, code_point)) if glyph_index is None: glyph_index = face.get_char_index(code_point) if glyph_index > 0: face.load_glyph(glyph_index, load_flags) return face face_index += 1 return None
unmerged_intervals = sorted(intervals + add_ints)intervals = []unvalidated_intervals = []for i_start, i_end in unmerged_intervals: if len(unvalidated_intervals) > 0 and i_start <= unvalidated_intervals[-1][1] + 1: unvalidated_intervals[-1] = (unvalidated_intervals[-1][0], max(unvalidated_intervals[-1][1], i_end)) continue unvalidated_intervals.append((i_start, i_end))
for i_start, i_end in unvalidated_intervals: start = i_start for code_point in range(i_start, i_end + 1): face = load_glyph(code_point) if face is None: if start < code_point: intervals.append((start, code_point - 1)) start = code_point + 1 if start != i_end + 1: intervals.append((start, i_end))
for face in font_stack: face.set_char_size(size << 6, size << 6, 150, 150)
total_size = 0all_glyphs = []
for i_start, i_end in intervals: for code_point in range(i_start, i_end + 1): face = load_glyph(code_point) bitmap = face.glyph.bitmap
# Build out 4-bit greyscale bitmap pixels4g = [] px = 0 for i, v in enumerate(bitmap.buffer): y = i / bitmap.width x = i % bitmap.width if x % 2 == 0: px = (v >> 4) else: px = px | (v & 0xF0) pixels4g.append(px); px = 0 # eol if x == bitmap.width - 1 and bitmap.width % 2 > 0: pixels4g.append(px) px = 0
if is2Bit: # 0-3 white, 4-7 light grey, 8-11 dark grey, 12-15 black # Downsample to 2-bit bitmap pixels2b = [] px = 0 pitch = (bitmap.width // 2) + (bitmap.width % 2) for y in range(bitmap.rows): for x in range(bitmap.width): px = px << 2 bm = pixels4g[y * pitch + (x // 2)] bm = (bm >> ((x % 2) * 4)) & 0xF
if bm >= 12: px += 3 elif bm >= 8: px += 2 elif bm >= 4: px += 1
if (y * bitmap.width + x) % 4 == 3: pixels2b.append(px) px = 0 if (bitmap.width * bitmap.rows) % 4 != 0: px = px << (4 - (bitmap.width * bitmap.rows) % 4) * 2 pixels2b.append(px) else: # Downsample to 1-bit bitmap - treat any 2+ as black pixelsbw = [] px = 0 pitch = (bitmap.width // 2) + (bitmap.width % 2) for y in range(bitmap.rows): for x in range(bitmap.width): px = px << 1 bm = pixels4g[y * pitch + (x // 2)] px += 1 if ((x & 1) == 0 and bm & 0xE > 0) or ((x & 1) == 1 and bm & 0xE0 > 0) else 0
if (y * bitmap.width + x) % 8 == 7: pixelsbw.append(px) px = 0 if (bitmap.width * bitmap.rows) % 8 != 0: px = px << (8 - (bitmap.width * bitmap.rows) % 8) pixelsbw.append(px)
pixels = pixels2b if is2Bit else pixelsbw
# Build output data packed = bytes(pixels) glyph = GlyphProps( width = bitmap.width, height = bitmap.rows, # We use linearHoriAdvance (16.16 fixed-point, unhinted) instead of # advance.x (26.6 fixed-point, grid-fitted to whole pixels by hinter) advance_x = fp4_from_ft16_16(face.glyph.linearHoriAdvance), left = face.glyph.bitmap_left, top = face.glyph.bitmap_top, data_length = len(packed), data_offset = total_size, code_point = code_point, ) total_size += len(packed) all_glyphs.append((glyph, packed))
# pipe seems to be a good heuristic for the "real" descenderface = load_glyph(ord('|'))
glyph_data = []glyph_props = []for index, glyph in enumerate(all_glyphs): props, packed = glyph glyph_data.extend([b for b in packed]) glyph_props.append(props)
# --- Kerning pair extraction ---# Modern fonts store kerning in the OpenType GPOS table, which FreeType's# get_kerning() does not read. We use fonttools to parse both the legacy# kern table and the GPOS 'kern' feature (PairPos lookups, including# Extension wrappers).
COMBINING_MARKS_START = 0x0300COMBINING_MARKS_END = 0x036Fall_codepoints = [g.code_point for g in glyph_props]kernable_codepoints = set(cp for cp in all_codepoints if not (COMBINING_MARKS_START <= cp <= COMBINING_MARKS_END))
# Map each kernable codepoint to the font-stack index that serves it# (same priority logic as load_glyph).cp_to_face_idx = {}for cp in kernable_codepoints: for face_idx, f in enumerate(font_stack): if f.get_char_index(cp) > 0: cp_to_face_idx[cp] = face_idx break
# Group codepoints by face indexface_idx_cps = {}for cp, fi in cp_to_face_idx.items(): face_idx_cps.setdefault(fi, set()).add(cp)
def _extract_pairpos_subtable(subtable, glyph_to_cp, raw_kern): """Extract kerning from a PairPos subtable (Format 1 or 2).""" if subtable.Format == 1: # Individual pairs for i, coverage_glyph in enumerate(subtable.Coverage.glyphs): if coverage_glyph not in glyph_to_cp: continue pair_set = subtable.PairSet[i] for pvr in pair_set.PairValueRecord: if pvr.SecondGlyph not in glyph_to_cp: continue xa = 0 if hasattr(pvr, 'Value1') and pvr.Value1: xa = getattr(pvr.Value1, 'XAdvance', 0) or 0 if xa != 0: key = (coverage_glyph, pvr.SecondGlyph) raw_kern[key] = raw_kern.get(key, 0) + xa elif subtable.Format == 2: # Class-based pairs class_def1 = subtable.ClassDef1.classDefs if subtable.ClassDef1 else {} class_def2 = subtable.ClassDef2.classDefs if subtable.ClassDef2 else {} coverage_set = set(subtable.Coverage.glyphs) for left_glyph in glyph_to_cp: if left_glyph not in coverage_set: continue c1 = class_def1.get(left_glyph, 0) if c1 >= len(subtable.Class1Record): continue class1_rec = subtable.Class1Record[c1] for right_glyph in glyph_to_cp: c2 = class_def2.get(right_glyph, 0) if c2 >= len(class1_rec.Class2Record): continue c2_rec = class1_rec.Class2Record[c2] xa = 0 if hasattr(c2_rec, 'Value1') and c2_rec.Value1: xa = getattr(c2_rec.Value1, 'XAdvance', 0) or 0 if xa != 0: key = (left_glyph, right_glyph) raw_kern[key] = raw_kern.get(key, 0) + xa
def extract_kerning_fonttools(font_path, codepoints, ppem, pnum_subs=None): """Extract kerning pairs from a font file using fonttools.
Returns dict of {(leftCp, rightCp): pixel_adjust} for the given codepoints. Values are scaled from font design units to integer pixels at ppem.
When pnum_subs is provided, substitute glyph names are also included in the lookup so kern pairs referencing proportional alternates are found. """ font = TTFont(font_path) units_per_em = font['head'].unitsPerEm cmap = font.getBestCmap() or {}
# Build glyph_name -> codepoint map (only for requested codepoints). # When pnum is active, include both the original and substitute glyph # names so kern pairs referencing either are captured. glyph_to_cp = {} for cp in codepoints: gname = cmap.get(cp) if gname: glyph_to_cp[gname] = cp if pnum_subs and gname in pnum_subs: glyph_to_cp[pnum_subs[gname]] = cp
# Collect raw kerning values in font design units raw_kern = {} # (left_glyph_name, right_glyph_name) -> design_units
# 1. Legacy kern table if 'kern' in font: for subtable in font['kern'].kernTables: if hasattr(subtable, 'kernTable'): for (lg, rg), val in subtable.kernTable.items(): if lg in glyph_to_cp and rg in glyph_to_cp: raw_kern[(lg, rg)] = raw_kern.get((lg, rg), 0) + val
# 2. GPOS 'kern' feature if 'GPOS' in font: gpos = font['GPOS'].table kern_lookup_indices = set() if gpos.FeatureList: for fr in gpos.FeatureList.FeatureRecord: if fr.FeatureTag == 'kern': kern_lookup_indices.update(fr.Feature.LookupListIndex) for li in kern_lookup_indices: lookup = gpos.LookupList.Lookup[li] for st in lookup.SubTable: actual = st # Unwrap Extension (lookup type 9) wrappers if lookup.LookupType == 9 and hasattr(st, 'ExtSubTable'): actual = st.ExtSubTable if hasattr(actual, 'Format'): _extract_pairpos_subtable(actual, glyph_to_cp, raw_kern)
font.close()
# Scale design-unit kerning values to 4.4 fixed-point pixels. scale = ppem / units_per_em result = {} # (leftCp, rightCp) -> 4.4 fixed-point adjust for (lg, rg), du in raw_kern.items(): lcp = glyph_to_cp[lg] rcp = glyph_to_cp[rg] adjust = fp4_from_design_units(du, scale) if adjust != 0: result[(lcp, rcp)] = adjust return result
# The ppem used by the existing glyph rasterization:# face.set_char_size(size << 6, size << 6, 150, 150)# means size_pt at 150 DPI -> ppem = size * 150 / 72ppem = size * 150.0 / 72.0
kern_map = {} # (leftCp, rightCp) -> adjustfor face_idx, cps in face_idx_cps.items(): font_path = args.fontstack[face_idx] subs = pnum_kern_subs.get(face_idx) if args.pnum else None kern_map.update(extract_kerning_fonttools(font_path, cps, ppem, pnum_subs=subs))
print(f"kerning: {len(kern_map)} pairs extracted", file=sys.stderr)
# --- Derive class-based kerning from pairs ---kern_left_classes = [] # list of (codepoint, classId)kern_right_classes = [] # list of (codepoint, classId)kern_matrix = [] # flat list of int8_t valueskern_left_class_count = 0kern_right_class_count = 0
if kern_map: all_left_cps = {lcp for lcp, _ in kern_map} all_right_cps = {rcp for _, rcp in kern_map}
sorted_right_cps = sorted(all_right_cps) sorted_left_cps = sorted(all_left_cps)
# Group left codepoints by identical adjustment row left_profile_to_class = {} left_class_map = {} left_class_id = 1 for lcp in sorted(all_left_cps): row = tuple(kern_map.get((lcp, rcp), 0) for rcp in sorted_right_cps) if row not in left_profile_to_class: left_profile_to_class[row] = left_class_id left_class_id += 1 left_class_map[lcp] = left_profile_to_class[row]
# Group right codepoints by identical adjustment column right_profile_to_class = {} right_class_map = {} right_class_id = 1 for rcp in sorted(all_right_cps): col = tuple(kern_map.get((lcp, rcp), 0) for lcp in sorted_left_cps) if col not in right_profile_to_class: right_profile_to_class[col] = right_class_id right_class_id += 1 right_class_map[rcp] = right_profile_to_class[col]
kern_left_class_count = left_class_id - 1 kern_right_class_count = right_class_id - 1
if kern_left_class_count > 255 or kern_right_class_count > 255: print(f"WARNING: kerning class count exceeds uint8_t range " f"(left={kern_left_class_count}, right={kern_right_class_count})", file=sys.stderr)
# Build the class x class matrix kern_matrix = [0] * (kern_left_class_count * kern_right_class_count) for (lcp, rcp), adjust in kern_map.items(): lc = left_class_map[lcp] - 1 rc = right_class_map[rcp] - 1 kern_matrix[lc * kern_right_class_count + rc] = adjust
# Build sorted class entry lists kern_left_classes = sorted(left_class_map.items()) kern_right_classes = sorted(right_class_map.items())
matrix_size = kern_left_class_count * kern_right_class_count entries_size = (len(kern_left_classes) + len(kern_right_classes)) * 3 print(f"kerning: {kern_left_class_count} left classes, {kern_right_class_count} right classes, " f"{matrix_size + entries_size} bytes", file=sys.stderr)
# --- Ligature pair extraction ---# Parse the OpenType GSUB table for LigatureSubst (type 4) lookups.# Multi-character ligatures (3+ codepoints) are decomposed into chained# pairs when an intermediate ligature exists (e.g., ffi = ff + i where ff# is itself a ligature). Only pairs where both input codepoints and the# output codepoint are in the generated glyph set are included.
all_codepoints_set = set(all_codepoints)
# Standard Unicode ligature codepoints for known input sequences.# Used as a fallback when the GSUB substitute glyph has no cmap entry.STANDARD_LIGATURE_MAP = { (0x66, 0x66): 0xFB00, # ff (0x66, 0x69): 0xFB01, # fi (0x66, 0x6C): 0xFB02, # fl (0x66, 0x66, 0x69): 0xFB03, # ffi (0x66, 0x66, 0x6C): 0xFB04, # ffl (0x17F, 0x74): 0xFB05, # long-s + t (0x73, 0x74): 0xFB06, # st}
def extract_ligatures_fonttools(font_path, codepoints): """Extract ligature substitution pairs from a font file using fonttools.
Returns list of (packed_pair, ligature_codepoint) for the given codepoints. Multi-character ligatures are decomposed into chained pairs. """ font = TTFont(font_path) cmap = font.getBestCmap() or {}
# Build glyph_name -> codepoint and codepoint -> glyph_name maps glyph_to_cp = {} cp_to_glyph = {} for cp, gname in cmap.items(): glyph_to_cp[gname] = cp cp_to_glyph[cp] = gname
# Collect raw ligature rules: (sequence_of_codepoints) -> ligature_codepoint raw_ligatures = {} # tuple of codepoints -> ligature codepoint
if 'GSUB' in font: gsub = font['GSUB'].table
# Find lookup indices for ligature features. # Currently extracts 'liga' (standard) and 'rlig' (required) only. # To also extract discretionary or historical ligatures, add: # 'dlig' - Discretionary Ligatures (e.g., ft, st in Noto) # 'hlig' - Historical Ligatures (e.g., long-s+t in OpenDyslexic) # These are off by default in standard text renderers. LIGATURE_FEATURES = ('liga', 'rlig') liga_lookup_indices = set() if gsub.FeatureList: for fr in gsub.FeatureList.FeatureRecord: if fr.FeatureTag in LIGATURE_FEATURES: liga_lookup_indices.update(fr.Feature.LookupListIndex)
for li in liga_lookup_indices: lookup = gsub.LookupList.Lookup[li] for st in lookup.SubTable: actual = st # Unwrap Extension (lookup type 7) wrappers if lookup.LookupType == 7 and hasattr(st, 'ExtSubTable'): actual = st.ExtSubTable # LigatureSubst is lookup type 4 if not hasattr(actual, 'ligatures'): continue for first_glyph, ligature_list in actual.ligatures.items(): if first_glyph not in glyph_to_cp: continue first_cp = glyph_to_cp[first_glyph] for lig in ligature_list: # lig.Component is a list of subsequent glyph names # lig.LigGlyph is the substitute glyph name component_cps = [] valid = True for comp_glyph in lig.Component: if comp_glyph not in glyph_to_cp: valid = False break component_cps.append(glyph_to_cp[comp_glyph]) if not valid: continue seq = tuple([first_cp] + component_cps) if lig.LigGlyph in glyph_to_cp: lig_cp = glyph_to_cp[lig.LigGlyph] elif seq in STANDARD_LIGATURE_MAP: lig_cp = STANDARD_LIGATURE_MAP[seq] else: seq_str = ', '.join(f'U+{cp:04X}' for cp in seq) print(f"ligatures: WARNING: dropping ligature ({seq_str}) -> " f"glyph '{lig.LigGlyph}': output glyph has no cmap entry " f"and input sequence is not in STANDARD_LIGATURE_MAP", file=sys.stderr) continue raw_ligatures[seq] = lig_cp
font.close()
# Filter: only keep ligatures where all input and output codepoints are # in our generated glyph set filtered = {} for seq, lig_cp in raw_ligatures.items(): if lig_cp not in codepoints and lig_cp not in all_codepoints_set: continue if all(cp in codepoints for cp in seq): filtered[seq] = lig_cp
# Decompose into chained pairs # For 2-codepoint sequences: direct pair (a, b) -> lig # For 3+ codepoint sequences: chain through intermediates # e.g., (f, f, i) -> ffi requires (f, f) -> ff to exist, # then we add (ff, i) -> ffi pairs = [] # First pass: collect all 2-codepoint ligatures two_char = {seq: lig_cp for seq, lig_cp in filtered.items() if len(seq) == 2} for seq, lig_cp in two_char.items(): packed = (seq[0] << 16) | seq[1] pairs.append((packed, lig_cp))
# Second pass: decompose 3+ codepoint ligatures into chained pairs for seq, lig_cp in filtered.items(): if len(seq) < 3: continue # Try to find an intermediate: check if the first N-1 codepoints # form a known ligature, then chain (intermediate, last) -> lig prefix = seq[:-1] last_cp = seq[-1] if prefix in filtered: intermediate_cp = filtered[prefix] packed = (intermediate_cp << 16) | last_cp pairs.append((packed, lig_cp)) else: print(f"ligatures: skipping {len(seq)}-char ligature " f"({', '.join(f'U+{cp:04X}' for cp in seq)}) -> U+{lig_cp:04X}: " f"no intermediate ligature for prefix", file=sys.stderr)
return pairs
ligature_codepoints = set(cp for cp in all_codepoints if not (COMBINING_MARKS_START <= cp <= COMBINING_MARKS_END))
# Map ligature codepoints to the font-stack index that serves themlig_cp_to_face_idx = {}for cp in ligature_codepoints: for face_idx, f in enumerate(font_stack): if f.get_char_index(cp) > 0: lig_cp_to_face_idx[cp] = face_idx break
# Group by face indexlig_face_idx_cps = {}for cp, fi in lig_cp_to_face_idx.items(): lig_face_idx_cps.setdefault(fi, set()).add(cp)
ligature_pairs = []for face_idx, cps in lig_face_idx_cps.items(): font_path = args.fontstack[face_idx] ligature_pairs.extend(extract_ligatures_fonttools(font_path, cps))
# Deduplicate (keep first occurrence) and sortseen_lig_keys = set()unique_ligature_pairs = []for packed, lig_cp in ligature_pairs: if packed not in seen_lig_keys: seen_lig_keys.add(packed) unique_ligature_pairs.append((packed, lig_cp))ligature_pairs = sorted(unique_ligature_pairs, key=lambda p: p[0])print(f"ligatures: {len(ligature_pairs)} pairs extracted", file=sys.stderr)
compress = args.compress
def to_byte_aligned(packed, width, height): """Convert packed 2-bit bitmap to byte-aligned format (rows padded to byte boundary).
In packed format, pixels flow continuously across row boundaries (4 pixels/byte). In byte-aligned format, each row starts at a byte boundary, padding the last byte of each row with zero bits if width % 4 != 0. This improves DEFLATE compression because identical pixel rows produce identical byte patterns regardless of position. """ if width == 0 or height == 0: return b'' row_stride = (width + 3) // 4 # bytes per byte-aligned row aligned = bytearray(row_stride * height) for y in range(height): for x in range(width): # Read pixel from packed format (continuous bit stream) packed_pos = y * width + x packed_byte_idx = packed_pos // 4 packed_shift = (3 - (packed_pos % 4)) * 2 pixel = (packed[packed_byte_idx] >> packed_shift) & 0x3
# Write pixel to byte-aligned format (row-aligned) aligned_byte_idx = y * row_stride + x // 4 aligned_shift = (3 - (x % 4)) * 2 aligned[aligned_byte_idx] |= (pixel << aligned_shift) return bytes(aligned)
# Build groups for compressionif compress and not is2Bit: print("Error: --compress requires --2bit (byte-aligned compression only supports 2-bit format)", file=sys.stderr) sys.exit(1)if compress: # Script-based grouping: glyphs that co-occur in typical text rendering # are grouped together for efficient LRU caching on the embedded target. # Since glyphs are in codepoint order, glyphs in the same Unicode block # are contiguous in the array and form natural groups. # # On top of script boundaries, a hard size cap (GROUP_MAX_UNCOMPRESSED_BYTES) # is applied: if adding the next glyph would push the uncompressed group # size over the cap, the group is closed and a new one started with the # same script ID. This bounds the embedded decompressor's transient # malloc regardless of font density (CJK, Vietnamese, user-supplied # fonts with large Unicode blocks). Without it, a single dense script # group can balloon past what fits in a transient page-decompress # allocation on the device. SCRIPT_GROUP_RANGES = [ (0x0000, 0x007F), # ASCII (0x0080, 0x00FF), # Latin-1 Supplement (0x0100, 0x017F), # Latin Extended-A (0x0180, 0x024F), # Latin Extended-B (0x0300, 0x036F), # Combining Diacritical Marks (0x0400, 0x04FF), # Cyrillic (0x1EA0, 0x1EF9), # Vietnamese Extended (0x2000, 0x206F), # General Punctuation (0x2070, 0x209F), # Superscripts & Subscripts (0x20A0, 0x20CF), # Currency Symbols (0x2190, 0x21FF), # Arrows (0x2200, 0x22FF), # Math Operators (0xFB00, 0xFB06), # Alphabetic Presentation Forms (ligatures) (0xFFFD, 0xFFFD), # Replacement Character ]
# 64 KB cap: large enough to hold any single built-in script group with # headroom, small enough to be a comfortable transient malloc on the # ESP32-C3. GROUP_MAX_UNCOMPRESSED_BYTES = 65536
def get_script_group(code_point): for i, (start, end) in enumerate(SCRIPT_GROUP_RANGES): if start <= code_point <= end: return i return -1
groups = [] # list of (first_glyph_index, glyph_count) current_group_id = None group_start = 0 group_count = 0 group_uncompressed = 0
for i, (props, _) in enumerate(all_glyphs): sg = get_script_group(props.code_point) # Use the byte-aligned size (4-pixel-aligned row stride) rather than # the packed length, since the decompressor consumes byte-aligned # buffers. Empty glyphs contribute zero. glyph_aligned_size = (((props.width + 3) // 4) * props.height if props.width > 0 and props.height > 0 else 0) if glyph_aligned_size > GROUP_MAX_UNCOMPRESSED_BYTES: raise ValueError( f"Glyph {i} (code point U+{props.code_point:04X}) byte-aligned size " f"{glyph_aligned_size} exceeds GROUP_MAX_UNCOMPRESSED_BYTES=" f"{GROUP_MAX_UNCOMPRESSED_BYTES}. Consider: (1) increasing GROUP_MAX_UNCOMPRESSED_BYTES, " f"(2) reducing font size, or (3) excluding this codepoint." ) size_overflow = group_uncompressed + glyph_aligned_size > GROUP_MAX_UNCOMPRESSED_BYTES
if sg != current_group_id or size_overflow: if group_count > 0: groups.append((group_start, group_count)) current_group_id = sg group_start = i group_count = 1 group_uncompressed = glyph_aligned_size else: group_count += 1 group_uncompressed += glyph_aligned_size
if group_count > 0: groups.append((group_start, group_count))
# Compress each group compressed_groups = [] # list of (compressed_bytes, uncompressed_size, glyph_count, first_glyph_index) compressed_bitmap_data = [] compressed_offset = 0
# Also build modified glyph props with within-group offsets modified_glyph_props = list(glyph_props)
for first_idx, count in groups: # Concatenate bitmap data for this group packed_len = 0 group_aligned = bytearray() for gi in range(first_idx, first_idx + count): props, packed = all_glyphs[gi] # Update glyph's dataOffset to be within-group offset (packed offset) within_group_offset = packed_len old_props = modified_glyph_props[gi] modified_glyph_props[gi] = GlyphProps( width=old_props.width, height=old_props.height, advance_x=old_props.advance_x, left=old_props.left, top=old_props.top, data_length=old_props.data_length, data_offset=within_group_offset, code_point=old_props.code_point, ) packed_len += len(packed) group_aligned.extend(to_byte_aligned(packed, old_props.width, old_props.height))
# Compress byte-aligned data with raw DEFLATE (no zlib/gzip header) compressed = deflate_raw(group_aligned)
compressed_groups.append((compressed, len(group_aligned), count, first_idx)) compressed_bitmap_data.extend(compressed) compressed_offset += len(compressed)
glyph_props = modified_glyph_props total_compressed = len(compressed_bitmap_data) total_uncompressed = len(glyph_data) print(f"// Compression: {total_uncompressed} -> {total_compressed} bytes ({100*total_compressed/total_uncompressed:.1f}%), {len(groups)} groups", file=sys.stderr)
print(f"""/** * generated by fontconvert.py * name: {font_name} * size: {size} * mode: {'2-bit' if is2Bit else '1-bit'}{' compressed: true' if compress else ''} * Command used: {' '.join(sys.argv)} */#pragma once#include "EpdFontData.h"""")
if compress: print(f"static const uint8_t {font_name}Bitmaps[{len(compressed_bitmap_data)}] = {{") for c in chunks(compressed_bitmap_data, 16): print (" " + " ".join(f"0x{b:02X}," for b in c)) print ("};\n");else: print(f"static const uint8_t {font_name}Bitmaps[{len(glyph_data)}] = {{") for c in chunks(glyph_data, 16): print (" " + " ".join(f"0x{b:02X}," for b in c)) print ("};\n");
def cp_label(cp): if cp == 0x5C: return '<backslash>' return chr(cp) if 0x20 < cp < 0x7F else f'U+{cp:04X}'
print(f"static const EpdGlyph {font_name}Glyphs[] = {{")for i, g in enumerate(glyph_props): print (" { " + ", ".join([f"{a}" for a in list(g[:-1])]),"},", f"// {cp_label(g.code_point)}")print ("};\n");
print(f"static const EpdUnicodeInterval {font_name}Intervals[] = {{")offset = 0for i_start, i_end in intervals: print (f" {{ 0x{i_start:X}, 0x{i_end:X}, 0x{offset:X} }},") offset += i_end - i_start + 1print ("};\n");
if compress: print(f"static const EpdFontGroup {font_name}Groups[] = {{") compressed_offset = 0 for compressed, uncompressed_size, count, first_idx in compressed_groups: print(f" {{ {compressed_offset}, {len(compressed)}, {uncompressed_size}, {count}, {first_idx} }},") compressed_offset += len(compressed) print("};\n")
if kern_map: # Split class maps: codepoints in one array, class IDs in a parallel one. Same 3 bytes per # entry as the packed EpdKernClassEntry, but the binary search only reads codepoints, so # keeping the payload out of the searched array shrinks its footprint by a third and makes # every read naturally aligned. Measured -13 to -14% on the class lookup, which is ~96% of # getKerning(). SD-card fonts keep the packed form because .cpfont maps it in place. for side, entries in (("Left", kern_left_classes), ("Right", kern_right_classes)): print(f"static const uint16_t {font_name}Kern{side}Codepoints[] = {{") for chunk in chunks([cp for cp, _ in entries], 12): print(" " + ", ".join(f"0x{cp:04X}" for cp in chunk) + ",") print("};\n") print(f"static const uint8_t {font_name}Kern{side}ClassIds[] = {{") for chunk in chunks([cls for _, cls in entries], 16): print(" " + ", ".join(f"{cls:3d}" for cls in chunk) + ",") print("};\n")
# Sparse (CSR) kerning. The dense leftClass x rightClass matrix is overwhelmingly zero -- # measured 86.6% across the built-in set -- so storing only the non-zero entries is roughly a # quarter of the size. Values are unchanged, so nothing repaginates. SD-card fonts still emit # the dense matrix (fontconvert_sdcard.py): .cpfont maps it in place. row_offsets = [] sparse_cols = [] sparse_vals = [] for row in range(kern_left_class_count): row_offsets.append(len(sparse_cols)) row_start = row * kern_right_class_count row_vals = kern_matrix[row_start:row_start + kern_right_class_count] for col, v in enumerate(row_vals): if v != 0: sparse_cols.append(col) sparse_vals.append(v) row_offsets.append(len(sparse_cols)) if len(sparse_cols) > 0xFFFF: print(f"Error: {len(sparse_cols)} kern entries exceed the uint16 row-offset range", file=sys.stderr) sys.exit(1) if kern_right_class_count > 256: print(f"Error: {kern_right_class_count} right classes exceed the uint8 column range", file=sys.stderr) sys.exit(1) dense_bytes = kern_left_class_count * kern_right_class_count sparse_bytes = len(row_offsets) * 2 + len(sparse_cols) * 2 print(f"// Kerning: {len(sparse_cols)} of {dense_bytes} entries non-zero " f"({100.0 * len(sparse_cols) / dense_bytes:.1f}%), {dense_bytes} -> {sparse_bytes} bytes", file=sys.stderr)
print(f"static const uint16_t {font_name}KernRowOffsets[] = {{") for chunk in chunks(row_offsets, 16): print(" " + ", ".join(f"{v:5d}" for v in chunk) + ",") print("};\n")
print(f"static const uint8_t {font_name}KernSparseCols[] = {{") for chunk in chunks(sparse_cols, 16): print(" " + ", ".join(f"{v:3d}" for v in chunk) + ",") print("};\n")
print(f"static const int8_t {font_name}KernSparseValues[] = {{") for chunk in chunks(sparse_vals, 16): print(" " + ", ".join(f"{v:4d}" for v in chunk) + ",") print("};\n")
if ligature_pairs: print(f"static const EpdLigaturePair {font_name}LigaturePairs[] = {{") for packed_pair, lig_cp in ligature_pairs: print(f" {{ 0x{packed_pair:08X}, 0x{lig_cp:04X} }}, // {cp_label(packed_pair >> 16)} {cp_label(packed_pair & 0xFFFF)} -> {cp_label(lig_cp)}") print("};\n")
print(f"static const EpdFontData {font_name} = {{")print(f" {font_name}Bitmaps,")print(f" {font_name}Glyphs,")print(f" {font_name}Intervals,")print(f" {len(intervals)},")print(f" {norm_ceil(face.size.height)},")print(f" {norm_ceil(face.size.ascender)},")print(f" {norm_floor(face.size.descender)},")print(f" {'true' if is2Bit else 'false'},")if compress: print(f" {font_name}Groups,") print(f" {len(compressed_groups)},")else: print(" nullptr,") print(" 0,")# glyphToGroup (not used for script-grouped fonts)print(" nullptr,")if kern_map: print(" nullptr, // kernLeftClasses: built-in fonts use the split arrays below") print(" nullptr, // kernRightClasses") print(f" {font_name}KernLeftCodepoints,") print(f" {font_name}KernLeftClassIds,") print(f" {font_name}KernRightCodepoints,") print(f" {font_name}KernRightClassIds,") print(" nullptr, // kernMatrix: built-in fonts use the sparse form below") print(f" {font_name}KernRowOffsets,") print(f" {font_name}KernSparseCols,") print(f" {font_name}KernSparseValues,") print(f" {len(kern_left_classes)},") print(f" {len(kern_right_classes)},") print(f" {kern_left_class_count},") print(f" {kern_right_class_count},")else: for _ in range(10): print(" nullptr,") print(f" 0,") print(f" 0,") print(f" 0,") print(f" 0,")if ligature_pairs: print(f" {font_name}LigaturePairs,") print(f" {len(ligature_pairs)},")else: print(f" nullptr,") print(f" 0,")print("};")