Something went wrong. Try again.
A fork of https://github.com/crosspoint-reader/crosspoint-reader
Something went wrong. Try again.
38 kB · 1077 lines
Python
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078#!/usr/bin/env python3"""Generate I18n C++ files from per-language YAML translations.
Reads YAML files from a translations directory (one file per language) and generates:- I18nKeys.h: Language enum, StrId enum, helper functions- I18nStrings.h: String array declarations- I18nStrings.cpp: String array definitions with all translations
Each YAML file must contain: _language_name: "Native Name" (e.g. "Español") _language_code: "ENUM_NAME" (e.g. "ES") _order: "N" (e.g. "1"; historical metadata, unused) _bcp47: "tag" (e.g. "es"; drives Language enum order) STR_KEY: "translation text"
The English file is the reference. Missing keys in other languages areautomatically filled from English, with a warning.
By default the script scans the src/ and lib/ trees for STR_* references andreports any translation keys that are never used. Pass --strip-unused toomit those keys from the generated output entirely.
Usage: python gen_i18n.py [translations_dir [output_dir]] [options]
Examples: python gen_i18n.py python gen_i18n.py lib/I18n/translations lib/I18n/ python gen_i18n.py --strip-unused python gen_i18n.py --strip-unused --src-dirs src lib/EpdFont"""
import sysimport osimport refrom pathlib import Pathfrom typing import Dict, List, Optional, Set, Tuple
AUTO_GENERATED_COMMENT = "// THIS FILE IS AUTO-GENERATED BY gen_i18n.py. DO NOT EDIT."CLANG_FORMAT_OFF_COMMENT = "// clang-format off"I18N_NAMESPACE_OPEN = "namespace i18n_strings {"I18N_NAMESPACE_CLOSE = "} // namespace i18n_strings"YAML_KEY_RE = re.compile(r'^([A-Za-z_]\w*)\s*:\s*"(.*)"$', flags=re.ASCII)CPP_IDENTIFIER_RE = re.compile(r"^[A-Za-z_]\w*$", flags=re.ASCII)STR_KEY_RE = re.compile(r"\bSTR_\w+\b", flags=re.ASCII)
# ---------------------------------------------------------------------------# YAML file reading (simple key: "value" format, no PyYAML dependency)# ---------------------------------------------------------------------------
def _unescape_yaml_value(raw: str, filepath: str = "", line_num: int = 0) -> str: """ Process escape sequences in a YAML value string.
Recognized escapes: \\\\ → \\ \\" → " \\n → newline """ result: List[str] = [] i = 0 while i < len(raw): if raw[i] == "\\" and i + 1 < len(raw): nxt = raw[i + 1] if nxt == "\\": result.append("\\") elif nxt == '"': result.append('"') elif nxt == "n": result.append("\n") else: raise ValueError(f"{filepath}:{line_num}: unknown escape '\\{nxt}'") i += 2 else: result.append(raw[i]) i += 1 return "".join(result)
def parse_yaml_file(filepath: str) -> Dict[str, str]: """ Parse a simple YAML file of the form: key: "value"
Only supports flat key-value pairs with quoted string values. Aborts on formatting errors. """ result = {} with open(filepath, "r", encoding="utf-8") as f: for line_num, raw_line in enumerate(f, start=1): line = raw_line.rstrip("\n\r")
if not line.strip(): continue
match = YAML_KEY_RE.match(line) if not match: raise ValueError( f"{filepath}:{line_num}: bad format: {line!r}\n" f' Expected: KEY: "value"' )
key = match.group(1) raw_value = match.group(2)
# Un-escape: process character by character to handle # \\, \", and \n sequences correctly value = _unescape_yaml_value(raw_value, filepath, line_num)
if key in result: raise ValueError(f"{filepath}:{line_num}: duplicate key '{key}'")
result[key] = value
return result
# ---------------------------------------------------------------------------# Load all languages from a directory of YAML files# ---------------------------------------------------------------------------
def load_translations( translations_dir: str, verbose: bool = False,) -> Tuple[List[str], List[str], List[str], List[str], Dict[str, List[str]], List[Set[str]]]: """ Read every YAML file in *translations_dir* and return: language_codes e.g. ["EN", "ES", ...] language_names e.g. ["English", "Español", ...] language_bcp47 e.g. ["en", "es", ...] string_keys ordered list of STR_* keys (from English) translations {key: [translation_per_language]}
English is always first; the rest are sorted by _bcp47. """ yaml_dir = Path(translations_dir) if not yaml_dir.is_dir(): raise FileNotFoundError(f"Translations directory not found: {translations_dir}")
yaml_files = sorted(yaml_dir.glob("*.yaml")) if not yaml_files: raise FileNotFoundError(f"No .yaml files found in {translations_dir}")
# Parse every file parsed: Dict[str, Dict[str, str]] = {} for yf in yaml_files: parsed[yf.name] = parse_yaml_file(str(yf))
# Identify the English file (must exist) english_file = None for name, data in parsed.items(): if data.get("_language_code", "").upper() == "EN": english_file = name break
if english_file is None: raise ValueError("No YAML file with _language_code: EN found")
for fname, data in parsed.items(): if not data.get("_bcp47"): raise ValueError(f"{fname}: missing _bcp47")
def _check_unique(field: str) -> None: """Raise if any two files share the same value for *field* (e.g. "_order", "_bcp47").""" files_by_value: Dict[str, List[str]] = {} for fname, data in parsed.items(): value = data.get(field) if not value: continue files_by_value.setdefault(value, []).append(fname)
duplicates = {value: sorted(files) for value, files in files_by_value.items() if len(files) > 1} if duplicates: duplicate_messages = [ f"{field} {value}: {', '.join(files)}" for value, files in sorted(duplicates.items()) ] raise ValueError( f"Duplicate {field} values found:\n " + "\n ".join(duplicate_messages) + f"\nEach {field} value must be unique to ensure a deterministic language order." )
_check_unique("_order") _check_unique("_bcp47")
# Order: English first (enum value 0), then by _bcp47 tag alphabetically. # This assigns the Language enum ordinal and also drives the visible # Settings menu order (see SORTED_LANGUAGE_INDICES). It has no effect on # stored user preferences, since settings.json persists the # _language_code string, not the ordinal. _order is retained as metadata # and still validated for uniqueness, but no longer drives enum assignment. def sort_key(fname: str) -> Tuple[int, str]: """English always first (0), then by BCP47 tag.""" if fname == english_file: return (0, "") return (1, parsed[fname]["_bcp47"])
ordered_files = sorted(parsed, key=sort_key)
# Extract metadata language_codes: List[str] = [] language_names: List[str] = [] language_bcp47: List[str] = [] for fname in ordered_files: data = parsed[fname] code = data.get("_language_code") name = data.get("_language_name") if not code or not name: raise ValueError(f"{fname}: missing _language_code or _language_name") language_codes.append(code) language_names.append(name) language_bcp47.append(data["_bcp47"])
# String keys come from English (order matters) english_data = parsed[english_file] string_keys = [k for k in english_data if not k.startswith("_")]
# Validate all keys are valid C++ identifiers for key in string_keys: if not CPP_IDENTIFIER_RE.match(key): raise ValueError(f"Invalid C++ identifier in English file: '{key}'")
# Build translations dict, filling missing keys from English inherited_sets: List[Set[str]] = [set() for _ in ordered_files] translations: Dict[str, List[str]] = {} for key in string_keys: row: List[str] = [] for lang_idx, fname in enumerate(ordered_files): data = parsed[fname] value = data.get(key, "") if not value.strip() and fname != english_file: value = english_data[key] inherited_sets[lang_idx].add(key) if verbose: print( f" INFO: '{key}' missing in {language_codes[lang_idx]}, using English fallback" ) row.append(value) translations[key] = row
# Warn about extra keys in non-English files for fname in ordered_files: if fname == english_file: continue data = parsed[fname] extra = [k for k in data if not k.startswith("_") and k not in english_data] if extra: lang_code = data.get("_language_code", fname) if verbose: print( f" WARNING: {lang_code} has keys not in English: {', '.join(extra)}" )
if verbose: print(f"Loaded {len(language_codes)} languages, {len(string_keys)} string keys") return language_codes, language_names, language_bcp47, string_keys, translations, inherited_sets
# ---------------------------------------------------------------------------# Unused-string detection# ---------------------------------------------------------------------------
_GENERATED_FILENAMES: Set[str] = {"I18nKeys.h", "I18nStrings.h", "I18nStrings.cpp"}
def find_used_string_keys( src_dirs: List[str], skip_filenames: Optional[Set[str]] = None,) -> Set[str]: """ Scan C/C++ source files under *src_dirs* for STR_* identifiers.
Files whose basename appears in *skip_filenames* are skipped so that the generated I18n files don't count as "usage" of themselves.
Returns the set of all STR_KEY names that appear at least once. """ if skip_filenames is None: skip_filenames = _GENERATED_FILENAMES
used: Set[str] = set()
for src_dir in src_dirs: p = Path(src_dir) if not p.is_dir(): continue for f in p.rglob("*"): if f.suffix not in {".cpp", ".h", ".c"}: continue if f.name in skip_filenames: continue try: text = f.read_text(encoding="utf-8", errors="replace") except OSError: continue for m in STR_KEY_RE.finditer(text): used.add(m.group(0))
return used
def report_unused_keys( string_keys: List[str], used_keys: Set[str],) -> List[str]: """Return a sorted list of keys from *string_keys* absent in *used_keys*.""" return [k for k in sorted(string_keys) if k not in used_keys]
# ---------------------------------------------------------------------------# C++ string escaping# ---------------------------------------------------------------------------
def escape_cpp_string(s: str) -> List[str]: r""" Convert *s* into one or more C++ string literal segments.
Non-ASCII characters are emitted as \xNN hex sequences. After each hex escape a new segment is started so the compiler doesn't merge subsequent hex digits into the escape.
Returns a list of string segments (without quotes). For simple ASCII strings this is a single-element list. """ if not s: return [""]
s = s.replace("\n", "\\n")
# Build a flat list of "tokens", where each token is either a regular # character sequence or a hex escape. A segment break happens after # every hex escape. segments: List[str] = [] current: List[str] = [] i = 0
def _flush() -> None: segments.append("".join(current)) current.clear()
while i < len(s): ch = s[i]
if ch == "\\" and i + 1 < len(s): nxt = s[i + 1] if nxt in 'ntr"\\': current.append(ch + nxt) i += 2 elif nxt == "x" and i + 3 < len(s): current.append(s[i : i + 4]) _flush() # segment break after hex i += 4 else: current.append("\\\\") i += 1 elif ch == '"': current.append('\\"') i += 1 elif ord(ch) < 128: current.append(ch) i += 1 else: for byte in ch.encode("utf-8"): current.append(f"\\x{byte:02X}") _flush() # segment break after hex i += 1
# Flush remaining content _flush()
return segments
def format_cpp_string_literal(segments: List[str], indent: str = " ") -> List[str]: """ Format string segments (from escape_cpp_string) as indented C++ string literal lines, each wrapped in quotes. Also wraps long segments to respect ~120 column limit. """ # Effective limit for content: 120 - 4 (indent) - 2 (quotes) - 1 (comma/safety) = 113 # Using 113 to match clang-format exactly (120 - 4 - 2 - 1) MAX_CONTENT_LEN = 113
lines: List[str] = []
for seg in segments: # Short segment (e.g. hex escape or short text) if len(seg) <= MAX_CONTENT_LEN: lines.append(f'{indent}"{seg}"') continue
# Long segment - wrap it current = seg while len(current) > MAX_CONTENT_LEN: # Find best split point # Scan forward to find last space <= MAX_CONTENT_LEN last_space = -1 idx = 0 while idx <= MAX_CONTENT_LEN and idx < len(current): if current[idx] == " ": last_space = idx
# Handle escapes to step correctly if current[idx] == "\\": idx += 2 else: idx += 1
# If we found a space, split after it if last_space != -1: # Include the space in the first line split_point = last_space + 1 lines.append(f'{indent}"{current[:split_point]}"') current = current[split_point:] else: # No space, forced break at MAX_CONTENT_LEN (or slightly less) cut_at = MAX_CONTENT_LEN # Don't cut in the middle of an escape sequence if current[cut_at - 1] == "\\": cut_at -= 1
lines.append(f'{indent}"{current[:cut_at]}"') current = current[cut_at:]
if current: lines.append(f'{indent}"{current}"')
return lines
# ---------------------------------------------------------------------------# Character-set computation# ---------------------------------------------------------------------------
def compute_character_set(translations: Dict[str, List[str]], lang_index: int) -> str: """Return a sorted string of every unique character used in a language.""" chars = set() for values in translations.values(): for ch in values[lang_index]: chars.add(ord(ch)) return "".join(chr(cp) for cp in sorted(chars))
# ---------------------------------------------------------------------------# Code generators# ---------------------------------------------------------------------------
def generate_keys_header( languages: List[str], language_names: List[str], language_bcp47: List[str], string_keys: List[str], output_path: str, verbose: bool = False, builtin: Optional[Set[str]] = None,) -> None: """Generate I18nKeys.h.
Every language keeps its enum value, code and name, so code that names a language (keyboard layouts, the V1 migration table) compiles either way. Only built-in languages get string tables and appear in the picker. """ compiled = [c for c in languages if builtin is None or c in builtin] lines: List[str] = [ "#pragma once", "#include <cstdint>", "", AUTO_GENERATED_COMMENT, CLANG_FORMAT_OFF_COMMENT, "", "// Forward declarations for flat string data blobs and offset tables", I18N_NAMESPACE_OPEN, ]
for code in compiled: lines.append(f"extern const char STRINGS_{code}_DATA[];") lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append(I18N_NAMESPACE_CLOSE) lines.append("")
# Language enum lines.append("// Language enum (ordered by _bcp47, English first)") lines.append("enum class Language : uint8_t {") code_width = max(len(lang) for lang in languages) for i, (lang, bcp47) in enumerate(zip(languages, language_bcp47)): lines.append(f" {lang:<{code_width}} = {i}, // {bcp47}") lines.append(" _COUNT") lines.append("};") lines.append("")
# Extern declarations lines.append("// Language codes (defined in I18nStrings.cpp)") lines.append("extern const char* const LANGUAGE_CODES[];") lines.append("")
lines.append("// Language display names (defined in I18nStrings.cpp)") lines.append("extern const char* const LANGUAGE_NAMES[];") lines.append("") lines.append("// Character sets for each language (defined in I18nStrings.cpp)") lines.append("extern const char* const CHARACTER_SETS[];") lines.append("")
# StrId enum lines.append("// String IDs") lines.append("enum class StrId : uint16_t {") for key in string_keys: lines.append(f" {key},") lines.append(" // Sentinel - must be last") lines.append(" _COUNT") lines.append("};") lines.append("")
# LangStrings struct lines.append("// Holds a flat string blob and its offset table for one language") lines.append("struct LangStrings {") lines.append(" const char* data;") lines.append(" const uint16_t* offsets;") lines.append("};") lines.append("")
# getLanguageStrings helper lines.append("// Helper function to get string data for a language") lines.append("inline LangStrings getLanguageStrings(Language lang) {") lines.append(" switch (lang) {") # A language left out of the build falls through to English below. for code in compiled: lines.append(f" case Language::{code}:") lines.append( f" return {{i18n_strings::STRINGS_{code}_DATA, i18n_strings::OFFSETS_{code}}};" ) first_code = languages[0] lines.append(" default:") lines.append( f" return {{i18n_strings::STRINGS_{first_code}_DATA, i18n_strings::OFFSETS_{first_code}}};" ) lines.append(" }") lines.append("}") lines.append("")
# getLanguageCount helper (single line to match checked-in format) lines.append("// Helper function to get language count") lines.append( "constexpr uint8_t getLanguageCount() " "{ return static_cast<uint8_t>(Language::_COUNT); }" ) lines.append("")
# Sorted language indices for display order # (English first, then by _bcp47 tag alphabetically) english_idx = languages.index("EN") rest = sorted( (i for i in range(len(languages)) if i != english_idx and languages[i] in compiled), key=lambda i: language_bcp47[i], ) sorted_indices = [english_idx] + rest lines.append("// Sorted language indices by _bcp47 (auto-generated by gen_i18n.py)") for rank, idx in enumerate(sorted_indices): lines.append(f"// {rank:>2}: {languages[idx]:<4} {language_bcp47[idx]:<8} {language_names[idx]}") lines.append( "constexpr uint8_t SORTED_LANGUAGE_INDICES[] = {" f"{', '.join(str(i) for i in sorted_indices)}" "};" ) lines.append("") size_check = "==" if builtin is None else "<=" lines.append( f"static_assert(sizeof(SORTED_LANGUAGE_INDICES) / sizeof(SORTED_LANGUAGE_INDICES[0]) {size_check} getLanguageCount()," ) lines.append(' "SORTED_LANGUAGE_INDICES size mismatch");') lines.append("")
# V1 language.bin migration table -- frozen enum order from commit 2f969a9. # Maps the old uint8_t index stored on disk to the current Language enum. # If a Language enum value listed here is ever removed, this will fail to # compile, signalling that the migration table needs updating. v1_codes = [ "EN", "ES", "FR", "DE", "CS", "PT", "RU", "SV", "RO", "CA", "UK", "BE", "IT", "PL", "FI", "DA", "NL", "TR", "KK", "HU", "LT", "SI", ] lines.append("// V1 language.bin migration table (frozen enum order from 2f969a9)") lines.append("constexpr Language V1_LANGUAGES[] = {") lines.append(" " + ", ".join(f"Language::{c}" for c in v1_codes) + ",") lines.append("};") lines.append( f"constexpr uint8_t V1_LANGUAGE_COUNT = {len(v1_codes)};" )
_write_file(output_path, lines, verbose)
def generate_strings_header( languages: List[str], output_path: str, verbose: bool = False, builtin: Optional[Set[str]] = None,) -> None: """Generate I18nStrings.h.""" lines: List[str] = [ "#pragma once", '#include "I18nKeys.h"', "", AUTO_GENERATED_COMMENT, CLANG_FORMAT_OFF_COMMENT, "", I18N_NAMESPACE_OPEN, "", ]
for code in (c for c in languages if builtin is None or c in builtin): lines.append(f"extern const char STRINGS_{code}_DATA[];") lines.append(f"extern const uint16_t OFFSETS_{code}[];")
lines.append("") lines.append(I18N_NAMESPACE_CLOSE) _write_file(output_path, lines, verbose)
def generate_strings_cpp( languages: List[str], language_names: List[str], string_keys: List[str], translations: Dict[str, List[str]], output_path: str, verbose: bool = False, builtin: Optional[Set[str]] = None,) -> None: """Generate I18nStrings.cpp.""" compiled = [c for c in languages if builtin is None or c in builtin] lines: List[str] = [ AUTO_GENERATED_COMMENT, CLANG_FORMAT_OFF_COMMENT, '#include "I18nStrings.h"', "", "#include <cstddef>", "", ]
# LANGUAGE_CODES array lines.append("// Language codes") lines.append("const char* const LANGUAGE_CODES[] = {") for code in languages: _append_string_entry(lines, code) lines.append("};") lines.append("")
# LANGUAGE_NAMES array lines.append("// Language display names") lines.append("const char* const LANGUAGE_NAMES[] = {") for name in language_names: _append_string_entry(lines, name) lines.append("};") lines.append("")
# CHARACTER_SETS array lines.append("// Character sets for each language") lines.append("const char* const CHARACTER_SETS[] = {") for lang_idx, name in enumerate(language_names): # A language left out of the build renders English strings, so it needs English glyphs. in_build = languages[lang_idx] in compiled charset = compute_character_set(translations, lang_idx if in_build else 0) _append_string_entry(lines, charset, comment=name) lines.append("};") lines.append("")
# Per-language flat string blobs and offset tables. # Non-English languages skip strings identical to English; their offset # tables use bit 15 (0x8000) to flag "use English blob at offset & 0x7FFF". lines.append(I18N_NAMESPACE_OPEN) lines.append("")
en_strings = [translations[key][0] for key in string_keys] en_offsets: List[int] = []
for lang_idx, code in enumerate(languages): if code not in compiled: continue lang_strings = [translations[key][lang_idx] for key in string_keys] is_english = lang_idx == 0
if is_english: # Precompute byte offsets (UTF-8 encoded, +1 per string for null terminator) offsets: List[int] = [] current_offset = 0 for s in lang_strings: offsets.append(current_offset) current_offset += len(s.encode("utf-8")) + 1 if current_offset > 0x7FFF: raise ValueError( f"Language {code}: blob size ({current_offset} bytes) exceeds " "15-bit offset limit (32767)" ) en_offsets = list(offsets) blob_strings = lang_strings else: offsets = [] current_offset = 0 blob_strings = [] for i, (s, en_s) in enumerate(zip(lang_strings, en_strings)): if s == en_s: offsets.append(en_offsets[i] | 0x8000) else: offsets.append(current_offset) current_offset += len(s.encode("utf-8")) + 1 blob_strings.append(s) if current_offset > 0x7FFF: raise ValueError( f"Language {code}: blob size ({current_offset} bytes) exceeds " "15-bit offset limit (32767)" )
# Flat string data blob — all strings concatenated with \0 separators. lines.append(f"const char STRINGS_{code}_DATA[] =") for text in blob_strings: _append_string_data_entry(lines, text) if not blob_strings: _append_string_data_entry(lines, "") lines.append(";") lines.append("")
# Offset table — one uint16_t per StrId lines.append(f"const uint16_t OFFSETS_{code}[] = {{") chunk_size = 12 for i in range(0, len(offsets), chunk_size): chunk = offsets[i : i + chunk_size] lines.append(" " + ", ".join(str(o) for o in chunk) + ",") lines.append("};") lines.append("")
lines.append(I18N_NAMESPACE_CLOSE) lines.append("")
# Compile-time size checks lines.append("// Compile-time validation of array sizes") for code in compiled: lines.append( f"static_assert(sizeof(i18n_strings::OFFSETS_{code}) " f"/ sizeof(i18n_strings::OFFSETS_{code}[0]) ==" ) lines.append(" static_cast<size_t>(StrId::_COUNT),") lines.append(f' "OFFSETS_{code} size mismatch");')
_write_file(output_path, lines, verbose)
# ---------------------------------------------------------------------------# Helpers# ---------------------------------------------------------------------------
def _print_language_table( language_codes: List[str], language_names: List[str], language_bcp47: List[str], inherited_sets: List[Set[str]], string_keys: List[str], unused_keys: Set[str], data_sizes: List[int],) -> None: """Print a per-language summary table.""" total = len(string_keys) headers = ("Language", "Code", "BCP47", "Own", "Fallback", "Unused", "Data (B)")
rows = [] for code, name, bcp47, inherited, size in zip( language_codes, language_names, language_bcp47, inherited_sets, data_sizes ): own = total - len(inherited) fallback = len(inherited) # strings this language translated but the code never calls unused = len(unused_keys - inherited) rows.append((name, code, bcp47, str(own), str(fallback), str(unused), str(size)))
# EN first, then alphabetically by _bcp47 (matches the Language enum order) rows.sort(key=lambda r: (0 if r[1] == "EN" else 1, r[2]))
col_widths = [len(h) for h in headers] for row in rows: for i, cell in enumerate(row): col_widths[i] = max(col_widths[i], len(cell))
fmt = " ".join(f"{{:<{w}}}" for w in col_widths) sep = " ".join("-" * w for w in col_widths)
def _safe_print(line: str) -> None: print( line.encode(sys.stdout.encoding or "utf-8", errors="replace").decode( sys.stdout.encoding or "utf-8", errors="replace" ) )
_safe_print(fmt.format(*headers)) _safe_print(sep) for row in rows: _safe_print(fmt.format(*row)) used = total - len(unused_keys) dedup_size = sum(data_sizes) n_lang = len(rows) n_keys = len(string_keys) offset_table_size = n_lang * n_keys * 2 current_total = dedup_size + offset_table_size print( f"\n Total: {total} | Used in code: {used} | Never used: {len(unused_keys)}" ) print( f" Flash: {dedup_size:>7,} B strings (deduped) + {offset_table_size:>6,} B offset tables" f" = {current_total:>7,} B" )
def _append_string_data_entry(lines: List[str], text: str) -> None: """ Escape *text*, append a \\0 null separator, and format as indented C++ string literal lines for inclusion in a flat char data array blob. """ segments = escape_cpp_string(text) # Append the null entry separator to the last segment if segments and segments[-1] != "": segments[-1] += "\\0" elif segments: segments[-1] = "\\0" else: segments = ["\\0"] lines.extend(format_cpp_string_literal(segments))
def _append_string_entry(lines: List[str], text: str, comment: str = "") -> None: """Escape *text*, format as indented C++ lines, append comma (and optional comment).""" segments = escape_cpp_string(text) formatted = format_cpp_string_literal(segments) suffix = f", // {comment}" if comment else "," formatted[-1] += suffix lines.extend(formatted)
def _write_file(path: str, lines: List[str], verbose: bool = False) -> None: with open(path, "w", encoding="utf-8", newline="\n") as f: f.write("\n".join(lines)) f.write("\n") if verbose: print(f"Generated: {path}")
def parse_builtin_langs(value: Optional[str], languages: List[str]) -> Optional[Set[str]]: """Resolve a comma-separated language list to the set compiled into the firmware.
None, empty or "all" means every language (returns None). English is always included: it is the fallback every other language's offsets point into. An unknown code is an error rather than silently dropped, so a typo cannot ship a build that is missing a language someone asked for. """ if value is None or value.strip().lower() in ("", "all"): return None by_upper = {c.upper(): c for c in languages} requested = [t.strip().upper() for t in value.split(",") if t.strip()] unknown = [t for t in requested if t not in by_upper] if unknown: raise ValueError( f"Unknown language code(s) in built-in list: {', '.join(unknown)}. " f"Valid codes: {', '.join(languages)}" ) return {by_upper[t] for t in requested} | {"EN"}
# ---------------------------------------------------------------------------# Main# ---------------------------------------------------------------------------
def main( translations_dir: Optional[str] = None, output_dir: Optional[str] = None, src_dirs: Optional[List[str]] = None, strip_unused: bool = False, verbose: bool = False, builtin_langs: Optional[str] = None,) -> None: # Default paths (relative to project root) default_translations_dir = "lib/I18n/translations" default_output_dir = "lib/I18n/" default_src_dirs = ["src", "lib"]
if translations_dir is None or output_dir is None: if len(sys.argv) == 3: translations_dir = sys.argv[1] output_dir = sys.argv[2] else: # Default for no arguments or weird arguments (e.g. SCons) translations_dir = default_translations_dir output_dir = default_output_dir
if src_dirs is None: src_dirs = default_src_dirs
if not os.path.isdir(translations_dir): print(f"Error: Translations directory not found: {translations_dir}") sys.exit(1)
if not os.path.isdir(output_dir): print(f"Error: Output directory not found: {output_dir}") sys.exit(1)
if verbose: print(f"Reading translations from: {translations_dir}") print(f"Output directory: {output_dir}") print()
try: languages, language_names, language_bcp47, string_keys, translations, inherited_sets = ( load_translations(translations_dir, verbose) ) builtin = parse_builtin_langs(builtin_langs, languages)
# --- Unused-string detection --- scan_dirs = [d for d in src_dirs if os.path.isdir(d)] if scan_dirs: used_keys = find_used_string_keys(scan_dirs) unused_set = set(report_unused_keys(string_keys, used_keys)) else: used_keys = set(string_keys) unused_set = set()
# --- Missing-string detection (used in code but absent from English) --- missing_keys = sorted(used_keys - set(string_keys)) if missing_keys: print( f"\n CRITICAL: {len(missing_keys)} string(s) used in source but missing from english.yaml:" ) for key in missing_keys: print(f" - {key}") print() sys.exit(1)
# Compute per-language data blob sizes (after dedup). # Non-English languages omit strings identical to English. data_sizes = [] for i in range(len(languages)): if i == 0: data_sizes.append( sum(len(translations[k][0].encode("utf-8")) + 1 for k in string_keys) ) else: data_sizes.append( sum( len(translations[k][i].encode("utf-8")) + 1 for k in string_keys if translations[k][i] != translations[k][0] ) )
_print_language_table( languages, language_names, language_bcp47, inherited_sets, string_keys, unused_set, data_sizes, ) print()
if verbose and unused_set: print(f" Unused keys ({len(unused_set)}):") for key in sorted(unused_set): print(f" - {key}") print()
if unused_set and strip_unused: string_keys = [k for k in string_keys if k not in unused_set] translations = { k: v for k, v in translations.items() if k not in unused_set } inherited_sets = [s - unused_set for s in inherited_sets] print(f" Stripping {len(unused_set)} unused string(s) from output.")
out = Path(output_dir) generate_keys_header( languages, language_names, language_bcp47, string_keys, str(out / "I18nKeys.h"), verbose, builtin, ) generate_strings_header( languages, str(out / "I18nStrings.h"), verbose, builtin ) generate_strings_cpp( languages, language_names, string_keys, translations, str(out / "I18nStrings.cpp"), verbose, builtin, )
print() print("Code generation complete!") if builtin is None: print(f" Languages: {len(languages)}") else: print(f" Languages: {len(builtin)} of {len(languages)} built in ({', '.join(c for c in languages if c in builtin)})") print(f" String keys: {len(string_keys)}") if unused_set and not strip_unused: print( f" Unused keys: {len(unused_set)} (pass --strip-unused to remove them)" )
except Exception as e: print(f"\nError: {e}") sys.exit(1)
if __name__ == "__main__": import argparse
parser = argparse.ArgumentParser( description="Generate I18n C++ files from per-language YAML translations." ) parser.add_argument( "translations_dir", nargs="?", default=None, help="Path to the translations directory (default: lib/I18n/translations)", ) parser.add_argument( "output_dir", nargs="?", default=None, help="Path to the output directory (default: lib/I18n/)", ) parser.add_argument( "--src-dirs", nargs="+", metavar="DIR", default=None, help="Source directories to scan for STR_* usage (default: src lib)", ) parser.add_argument( "--strip-unused", action="store_true", help="Remove unused STR_* keys from the generated output", ) parser.add_argument( "--builtin-langs", metavar="CODES", default=None, help='Comma-separated language codes to compile in, e.g. "en,de,fr" (default: all)', ) parser.add_argument( "--verbose", "-v", action="store_true", help="Print per-key INFO/WARNING messages and file generation details", ) args = parser.parse_args() main( args.translations_dir, args.output_dir, args.src_dirs, args.strip_unused, args.verbose, args.builtin_langs, )else: if "Import" in globals(): Import("env") main( strip_unused=True, builtin_langs=env.GetProjectOption("custom_i18n_builtin_langs", "all"), )