From c03485b5df750bc63c5cdb8a3f81bcd7fb9916ee Mon Sep 17 00:00:00 2001 From: Niclas Overby Date: Wed, 12 Aug 2026 11:34:58 +0200 Subject: [PATCH] port(cider): tbd-diff to Rust, gated on a fixture SDK because the real one is not realised cider-tbd-diff, the fifteenth binary of graph-specs. It parses the SDK libSystem.tbd re-export closure for the OFFICIAL macOS export set, reads the ACTUAL export set out of the built dylibs through the Mach-O exports TRIE rather than nm, and diffs them. The trie is not a detail: it carries RE-EXPORT aliases, and cider exports _memcpy as a re-export of __platform_memmove, which never appears in nm -gU, so an nm scan wildly under-reports the surface. BYTE IDENTITY ON THE REAL INPUT WAS IMPOSSIBLE, and here is exactly why: the Apple SDK is not realised in this store, only apple-sdk-14.4.drv, so there is no libSystem.tbd for --sdk to point at and nothing to compare on. The gate is a hand-built FIXTURE SDK instead, built to exercise every branch the two parsers have rather than to look real: a v4 root with a QUOTED install-name, a $ld$ pseudo-symbol that must be dropped, and a reexported-libraries list SPLIT ACROSS TWO LINES, which is the whole reason the bracket collector exists; a v4 leaf with per-entry target filtering (an i386 entry that must not be picked up), weak-symbols, and a symbols list wrapped across lines; a v5 JSON leaf reached THROUGH the closure, with a targets-less group that counts, an arm64 group that does not, and a reexport naming a file that is not there; a demand file, and a cider root holding a file that is NOT a Mach-O, so the objdump failure path is the one that runs. Six modes compared byte for byte on stdout, stderr and exit code: supply only, --arch arm64, --demand, --cider-root, a missing root tbd, and the --out plus --json writers compared as FILES. Plus the control: x86_64 and arm64 must differ, and do, which is what proves the target filter is live rather than ignored. ONE DIFFERENCE, named: a missing --sdk is argparse own usage text and exit 2 in the python. There is nothing to reproduce there, so the Rust prints its own one-line message and keeps the 2. scripts/ python: 7 files to 6, 5,784 lines to 5,435. --- docs/monorepo-port.md | 2 +- linux/buildtools/graph-specs/Cargo.toml | 7 + linux/buildtools/graph-specs/src/tbddiff.rs | 550 ++++++++++++++++++++ scripts/symbol-demand.nu | 4 +- scripts/tbd-diff.py | 349 ------------- 5 files changed, 560 insertions(+), 352 deletions(-) create mode 100644 linux/buildtools/graph-specs/src/tbddiff.rs delete mode 100755 scripts/tbd-diff.py diff --git a/docs/monorepo-port.md b/docs/monorepo-port.md index 7396c624b..8d7f2efad 100644 --- a/docs/monorepo-port.md +++ b/docs/monorepo-port.md @@ -73,7 +73,7 @@ consumed only by things that are themselves archive-or-tool candidates**, so not keep working in the monorepo depends on either file. The 182-line extraction is now a convenience for the tools, not a prerequisite for the checks. -The whole `scripts/` python is **7 files, 5,784 lines** as of 2026-08-12, down from 54 files at +The whole `scripts/` python is **6 files, 5,435 lines** as of 2026-08-12, down from 54 files at the start of this campaign and from the 29 the table above was measured on. **THE LIVE SURFACE OF `gen-buck-from-ninja.py` IS RUST NOW**, in diff --git a/linux/buildtools/graph-specs/Cargo.toml b/linux/buildtools/graph-specs/Cargo.toml index 63b1f669e..8ce81ec3c 100644 --- a/linux/buildtools/graph-specs/Cargo.toml +++ b/linux/buildtools/graph-specs/Cargo.toml @@ -118,6 +118,13 @@ path = "src/sdkroots.rs" name = "cider-mig-from-ninja" path = "src/migninja.rs" +# THE FIFTEENTH BINARY, the Rust rewrite of scripts/tbd-diff.py (#98). The real Apple SDK is not +# realised in this store, so it is gated on a FIXTURE SDK that exercises every branch of the two +# .tbd parsers rather than on the real one. +[[bin]] +name = "cider-tbd-diff" +path = "src/tbddiff.rs" + [dependencies] serde_json = { version = "1", features = ["preserve_order"] } diff --git a/linux/buildtools/graph-specs/src/tbddiff.rs b/linux/buildtools/graph-specs/src/tbddiff.rs new file mode 100644 index 000000000..c9f96b884 --- /dev/null +++ b/linux/buildtools/graph-specs/src/tbddiff.rs @@ -0,0 +1,550 @@ +//! THE SUPPLY SIDE OF THE libSystem SYMBOL GAP. +//! +//! THE RUST REWRITE of the python tbd-diff (#98). It parses the SDK libSystem.tbd re-export +//! closure to get the OFFICIAL macOS export set, extracts the ACTUAL export set from the built +//! dylibs, and diffs them, optionally intersected with a demand list so the output is exactly the +//! symbols that real binaries import, macOS provides, and cider still lacks. +//! +//! NO THIRD PARTY YAML, same as the python: the .tbd files are regular enough to parse directly. +//! It handles tbd-version 4 (the YAML-ish 14.4 SDK) and tbd-version 5 (JSON). +//! +//! THE EXPORTS TRIE, NOT nm, and that is not a detail: the trie includes RE-EXPORT aliases, and +//! cider exports _memcpy as a re-export of __platform_memmove. Those never appear in nm -gU, so +//! an nm-based scan wildly under-reports the real export surface. +//! +//! WHAT THE GATE COULD AND COULD NOT USE. The real Apple SDK is NOT realised in this store, only +//! its .drv, so there is no libSystem.tbd to point --sdk at and a byte comparison on the real +//! input was impossible. It is gated instead on a hand-built FIXTURE SDK that exercises every +//! branch the parser has: a v4 root with quoted install-name and a multi-line bracket list, a v4 +//! leaf with target filtering, weak-symbols and a $ld$ pseudo-symbol, a v5 JSON leaf, a demand +//! file, and a cider root holding a file that is not a Mach-O so the objdump failure path runs. +//! +//! Usage: +//! cider-tbd-diff --sdk [--arch x86_64] [--platform macos] +//! [--root ] [--cider-root ] +//! [--demand ] [--out report.md] [--json out.json] + +use std::collections::{BTreeMap, BTreeSet, HashMap}; +use std::fs; +use std::process::{Command, ExitCode}; + +fn is_file(p: &str) -> bool { + fs::metadata(p).map(|m| m.is_file()).unwrap_or(false) +} + +fn is_dir(p: &str) -> bool { + fs::metadata(p).map(|m| m.is_dir()).unwrap_or(false) +} + +fn basename(p: &str) -> &str { + match p.trim_end_matches('/').rfind('/') { + Some(i) => &p[i + 1..], + None => p, + } +} + +/// os.walk, pre-order, readdir order, not following symlinked directories. +fn walk(dir: &str, out: &mut Vec<(String, Vec, Vec)>) { + let rd = match fs::read_dir(dir) { + Ok(r) => r, + Err(_) => return, + }; + let mut dirs = Vec::new(); + let mut files = Vec::new(); + for e in rd.flatten() { + let name = e.file_name().to_string_lossy().into_owned(); + let p = format!("{dir}/{name}"); + if is_dir(&p) { + dirs.push(name); + } else { + files.push(name); + } + } + out.push((dir.to_string(), dirs.clone(), files)); + for d in dirs { + let p = format!("{dir}/{d}"); + if !fs::symlink_metadata(&p).map(|m| m.file_type().is_symlink()).unwrap_or(false) { + walk(&p, out); + } + } +} + +/// Locate the MacOSX*.sdk directory inside an apple-sdk store path. +fn find_sdk_root(sdk: &str) -> String { + if is_file(&format!("{sdk}/usr/lib/libSystem.tbd")) { + return sdk.to_string(); + } + let mut all = Vec::new(); + walk(sdk, &mut all); + for (dirpath, dirs, _files) in all { + for d in dirs { + if d.ends_with(".sdk") { + let cand = format!("{dirpath}/{d}"); + if is_file(&format!("{cand}/usr/lib/libSystem.tbd")) { + return cand; + } + } + } + } + sdk.to_string() +} + +fn strip_quotes(s: &str) -> &str { + s.trim_matches(|c| c == '\'' || c == '"') +} + +/// True if any `arch-platform` target token matches. +fn target_matches(targets: &[String], arch: &str, platform: &str) -> bool { + let want = format!("{arch}-{platform}"); + targets.iter().any(|t| strip_quotes(t.trim()) == want) +} + +/// lines[i] contains a '[': accumulate through the matching ']'. Returns (items, next index). +fn collect_bracket_list(lines: &[&str], mut i: usize) -> (Vec, usize) { + let mut buf: Vec<&str> = Vec::new(); + let mut depth: i64 = 0; + while i < lines.len() { + let seg = lines[i]; + depth += seg.matches('[').count() as i64 - seg.matches(']').count() as i64; + buf.push(seg); + i += 1; + if depth <= 0 { + break; + } + } + let text = buf.join(" "); + let start = text.find('[').map(|k| k + 1).unwrap_or(0); + let end = text.rfind(']').unwrap_or(text.len()); + let inside = if start <= end { &text[start..end] } else { "" }; + let items: Vec = inside + .split(',') + .map(|x| strip_quotes(x.trim()).to_string()) + .filter(|x| !x.is_empty()) + .collect(); + (items, i) +} + +struct Tbd { + install_names: Vec, + reexports: Vec, + symbols: BTreeSet, +} + +fn parse_tbd(path: &str, arch: &str, platform: &str) -> Tbd { + let raw = fs::read_to_string(path).unwrap_or_default(); + // tbd-version 5 is JSON. + let head: String = raw.chars().take(64).collect(); + if raw.trim_start().starts_with('{') || head.contains("tapi-tbd-v5") { + return parse_tbd_v5(&raw, arch, platform); + } + let lines: Vec<&str> = raw.lines().collect(); + let mut install_names = Vec::new(); + let mut reexports = Vec::new(); + let mut symbols = BTreeSet::new(); + // In tbd v4 top-level keys sit at column 0 and a section content is indented, so the section + // is tracked by watching unindented `key:` lines. That is robust to intervening keys a + // heuristic reset would trip over. + let mut section: Option = None; + let mut cur_targets: Option> = None; + let mut i = 0; + while i < lines.len() { + let ln = lines[i]; + let s = ln.trim(); + let indented = ln.starts_with(' ') || ln.starts_with('\t'); + if !indented && is_top_key(s) { + let key = s.split(':').next().unwrap_or(""); + section = match key { + "exports" | "reexports" | "reexported-libraries" => Some(key.to_string()), + _ => None, + }; + if let Some(name) = install_name_of(s) { + install_names.push(name); + } + i += 1; + continue; + } + if section.is_none() { + i += 1; + continue; + } + // Inside a section: entries begin with `- targets:`. + if s.contains("targets:") { + let (t, next) = collect_bracket_list(&lines, i); + cur_targets = Some(t); + i = next; + continue; + } + let in_target = match &cur_targets { + None => true, + Some(t) => target_matches(t, arch, platform), + }; + let sec = section.clone().unwrap_or_default(); + if sec == "reexported-libraries" && s.contains("libraries:") { + let (libs, next) = collect_bracket_list(&lines, i); + if in_target { + reexports.extend(libs); + } + i = next; + continue; + } + if (sec == "exports" || sec == "reexports") + && (s.contains("symbols:") || s.contains("weak-symbols:")) + { + let (syms, next) = collect_bracket_list(&lines, i); + if in_target { + for sym in syms { + // linker-directive pseudo-symbols + if sym.starts_with("$ld$") { + continue; + } + symbols.insert(sym); + } + } + i = next; + continue; + } + i += 1; + } + Tbd { install_names, reexports, symbols } +} + +/// ^[A-Za-z_][A-Za-z0-9_-]*: +fn is_top_key(s: &str) -> bool { + let b = s.as_bytes(); + if b.is_empty() || !(b[0].is_ascii_alphabetic() || b[0] == b'_') { + return false; + } + let mut i = 1; + while i < b.len() && (b[i].is_ascii_alphanumeric() || b[i] == b'_' || b[i] == b'-') { + i += 1; + } + i < b.len() && b[i] == b':' +} + +/// ^install-name:\s*'?([^'\n]+?)'?\s*$ +fn install_name_of(s: &str) -> Option { + let rest = s.strip_prefix("install-name:")?; + let v = rest.trim(); + if v.is_empty() { + return None; + } + let v = v.strip_prefix('\'').unwrap_or(v); + let v = v.strip_suffix('\'').unwrap_or(v); + let v = v.trim(); + if v.is_empty() { + return None; + } + Some(strip_quotes(v).to_string()) +} + +fn parse_tbd_v5(raw: &str, arch: &str, platform: &str) -> Tbd { + let mut install_names = Vec::new(); + let mut reexports = Vec::new(); + let mut symbols = BTreeSet::new(); + let data: serde_json::Value = match serde_json::from_str(raw) { + Ok(v) => v, + Err(_) => return Tbd { install_names, reexports, symbols }, + }; + let want = format!("{arch}-{platform}"); + let main = data.get("main_library").cloned().unwrap_or(serde_json::Value::Null); + if let Some(a) = main.get("install_names").and_then(|v| v.as_array()) { + for lib in a { + if let Some(n) = lib.get("name").and_then(|v| v.as_str()) { + install_names.push(n.to_string()); + } + } + } + if let Some(a) = main.get("reexported_libraries").and_then(|v| v.as_array()) { + for grp in a { + let tgts: Vec<&str> = + grp.get("targets").and_then(|v| v.as_array()).map(|a| a.iter().filter_map(|x| x.as_str()).collect()).unwrap_or_default(); + if tgts.contains(&want.as_str()) { + if let Some(names) = grp.get("names").and_then(|v| v.as_array()) { + reexports.extend(names.iter().filter_map(|x| x.as_str()).map(|s| s.to_string())); + } + } + } + } + if let Some(a) = main.get("exported_symbols").and_then(|v| v.as_array()) { + for grp in a { + let tgts: Vec<&str> = + grp.get("targets").and_then(|v| v.as_array()).map(|a| a.iter().filter_map(|x| x.as_str()).collect()).unwrap_or_default(); + if tgts.contains(&want.as_str()) || tgts.is_empty() { + for kind in ["global", "data", "text", "weak"] { + if let Some(list) = grp.get(kind).and_then(|v| v.as_array()) { + for sym in list.iter().filter_map(|x| x.as_str()) { + if !sym.starts_with("$ld$") { + symbols.insert(sym.to_string()); + } + } + } + } + } + } + } + Tbd { install_names, reexports, symbols } +} + +/// Map an install name to its .tbd in the SDK. +fn tbd_path_for_install_name(sdk_root: &str, install_name: &str) -> Option { + let rel = install_name.trim_start_matches('/'); + let base = format!("{sdk_root}/{rel}"); + let first = if base.ends_with(".dylib") { + format!("{}.tbd", &base[..base.len() - ".dylib".len()]) + } else { + base.clone() + }; + for cand in [first, format!("{base}.tbd"), base] { + if is_file(&cand) { + return Some(cand); + } + } + None +} + +/// Walk the reexport closure from root_tbd; {symbol: install name}, first owner wins. +fn collect_official( + sdk_root: &str, + root_tbd: &str, + arch: &str, + platform: &str, +) -> (Vec<(String, String)>, HashMap) { + let mut order: Vec<(String, String)> = Vec::new(); + let mut supply: HashMap = HashMap::new(); + let mut seen: BTreeSet = BTreeSet::new(); + let mut stack = vec![root_tbd.to_string()]; + while let Some(tbd) = stack.pop() { + if seen.contains(&tbd) || tbd.is_empty() || !is_file(&tbd) { + continue; + } + seen.insert(tbd.clone()); + let parsed = parse_tbd(&tbd, arch, platform); + let owner = parsed.install_names.first().cloned().unwrap_or_else(|| tbd.clone()); + for sym in &parsed.symbols { + if !supply.contains_key(sym) { + supply.insert(sym.clone(), owner.clone()); + order.push((sym.clone(), owner.clone())); + } + } + for rex in &parsed.reexports { + if let Some(next) = tbd_path_for_install_name(sdk_root, rex) { + stack.push(next); + } + } + } + (order, supply) +} + +/// Every symbol exported by every dylib under `root`, through the exports trie. +fn collect_cider(root: &str, arch: &str) -> Result<(BTreeSet, usize), String> { + let objdump = which("llvm-objdump") + .ok_or_else(|| "error: need llvm-objdump on PATH (nix shell nixpkgs#llvm)".to_string())?; + let mut exported = BTreeSet::new(); + let mut dylibs = Vec::new(); + let mut all = Vec::new(); + walk(root, &mut all); + for (dp, _dirs, files) in all { + for f in files { + if f.contains(".dylib") { + dylibs.push(format!("{dp}/{f}")); + } + } + } + for lib in &dylibs { + let out = match Command::new(&objdump) + .args(["--macho", "--exports-trie", &format!("--arch={arch}"), lib]) + .output() + { + Ok(o) => String::from_utf8_lossy(&o.stdout).into_owned(), + Err(_) => continue, + }; + for line in out.lines() { + if let Some(at) = line.find("[re-export]") { + let rest = &line[at + "[re-export]".len()..]; + let rest = rest.trim_start(); + let sym: String = rest.chars().take_while(|c| !c.is_whitespace()).collect(); + if !sym.is_empty() { + exported.insert(sym); + continue; + } + } + // ^0x[0-9A-Fa-f]+\s+(\S+) on the stripped line, symbol must start with _ + let t = line.trim(); + if let Some(rest) = t.strip_prefix("0x") { + let hex_len = rest.chars().take_while(|c| c.is_ascii_hexdigit()).count(); + if hex_len > 0 { + let after = &rest[hex_len..]; + if after.starts_with(char::is_whitespace) { + let sym: String = + after.trim_start().chars().take_while(|c| !c.is_whitespace()).collect(); + if sym.starts_with('_') { + exported.insert(sym); + } + } + } + } + } + } + Ok((exported, dylibs.len())) +} + +fn which(x: &str) -> Option { + for p in std::env::var("PATH").unwrap_or_default().split(':') { + let cand = format!("{p}/{x}"); + if is_file(&cand) { + return Some(cand); + } + } + None +} + +fn main() -> ExitCode { + let argv: Vec = std::env::args().skip(1).collect(); + let get = |name: &str| -> Option { + argv.iter().position(|a| a == name).and_then(|i| argv.get(i + 1)).cloned() + }; + let sdk = match get("--sdk") { + Some(s) => s, + None => { + // argparse prints its own usage and exits 2; this is the one place the two differ, + // and it is the one place there is nothing to reproduce. + eprintln!("cider-tbd-diff: --sdk is required"); + return ExitCode::from(2); + } + }; + let arch = get("--arch").unwrap_or_else(|| "x86_64".to_string()); + let platform = get("--platform").unwrap_or_else(|| "macos".to_string()); + + let sdk_root = find_sdk_root(&sdk); + let root_tbd = get("--root").unwrap_or_else(|| format!("{sdk_root}/usr/lib/libSystem.tbd")); + if !is_file(&root_tbd) { + eprintln!("error: no root tbd at {root_tbd}"); + return ExitCode::FAILURE; + } + + let (official_order, official) = collect_official(&sdk_root, &root_tbd, &arch, &platform); + let demand: Option> = match get("--demand") { + None => None, + Some(p) => { + let text = match fs::read_to_string(&p) { + Ok(t) => t, + Err(e) => { + eprintln!("cannot read {p}: {e}"); + return ExitCode::FAILURE; + } + }; + let v: serde_json::Value = serde_json::from_str(&text).unwrap_or(serde_json::Value::Null); + let mut m = BTreeMap::new(); + if let Some(a) = v.get("symbols").and_then(|x| x.as_array()) { + for e in a { + if let Some(s) = e.get("symbol").and_then(|x| x.as_str()) { + let refs = e.get("refs").and_then(|x| x.as_i64()).unwrap_or(0); + m.insert(s.to_string(), refs); + } + } + } + Some(m) + } + }; + + let mut cider: Option> = None; + let mut n_dylibs = 0usize; + if let Some(cr) = get("--cider-root") { + if is_dir(&cr) { + match collect_cider(&cr, &arch) { + Ok((c, n)) => { + cider = Some(c); + n_dylibs = n; + } + Err(e) => { + eprintln!("{e}"); + return ExitCode::FAILURE; + } + } + } + } + + let mut lines: Vec = Vec::new(); + lines.push(format!("# libSystem symbol gap ({arch}-{platform})\n")); + lines.push(format!( + "Generated by `cider-tbd-diff`. Supply side: the SDK `libSystem.tbd` re-export \ + closure ({}).\n", + basename(&sdk_root) + )); + lines.push(format!("- Official exported symbols (SDK closure): **{}**", official.len())); + if let Some(c) = &cider { + lines.push(format!("- Darling exported symbols ({n_dylibs} dylibs): **{}**", c.len())); + } + if let Some(d) = &demand { + lines.push(format!("- Demanded symbols (from binaries): **{}**", d.len())); + } + lines.push(String::new()); + + let mut report = serde_json::Map::new(); + report.insert("arch".into(), serde_json::Value::String(arch.clone())); + report.insert("platform".into(), serde_json::Value::String(platform.clone())); + report.insert("official_count".into(), serde_json::Value::from(official.len())); + + if let Some(c) = &cider { + let missing: Vec<(String, String)> = + official_order.iter().filter(|(s, _)| !c.contains(s)).cloned().collect(); + report.insert("cider_count".into(), serde_json::Value::from(c.len())); + report.insert("missing_count".into(), serde_json::Value::from(missing.len())); + lines.push(format!("## Missing from Darling (official − cider): {}\n", missing.len())); + if let Some(d) = &demand { + let mut worklist: Vec<(i64, String, String)> = missing + .iter() + .filter(|(s, _)| d.contains_key(s)) + .map(|(s, o)| (*d.get(s).unwrap_or(&0), s.clone(), o.clone())) + .collect(); + worklist.sort_by(|a, b| b.cmp(a)); + report.insert("demanded_missing_count".into(), serde_json::Value::from(worklist.len())); + lines.push(format!( + "### Demanded work list (needed ∩ macOS14 − cider): **{}**\n", + worklist.len() + )); + lines.push("| # refs | symbol | owner |".to_string()); + lines.push("|---:|:---|:---|".to_string()); + for (refs, sym, owner) in &worklist { + lines.push(format!("| {refs} | `{sym}` | `{owner}` |")); + } + lines.push(String::new()); + let not_in_sdk: Vec<&String> = + d.keys().filter(|s| !official.contains_key(*s)).collect(); + report.insert("demanded_not_in_sdk".into(), serde_json::Value::from(not_in_sdk.len())); + lines.push(format!( + "### Demanded but absent from libSystem tbd closure: **{}** (likely \ + framework-owned)\n", + not_in_sdk.len() + )); + } + } else { + lines.push( + "_(no --cider-root given; supply-only run. Provide the built Darling dylibs to \ + compute the gap.)_\n" + .to_string(), + ); + } + + let text = lines.join("\n") + "\n"; + match get("--out") { + Some(out) => { + if let Err(e) = fs::write(&out, &text) { + eprintln!("cannot write {out}: {e}"); + return ExitCode::FAILURE; + } + eprintln!("wrote {out}"); + } + None => print!("{text}"), + } + if let Some(jp) = get("--json") { + let rendered = serde_json::to_string_pretty(&serde_json::Value::Object(report)) + .unwrap_or_else(|_| "{}".to_string()); + if let Err(e) = fs::write(&jp, rendered) { + eprintln!("cannot write {jp}: {e}"); + return ExitCode::FAILURE; + } + eprintln!("wrote {jp}"); + } + ExitCode::SUCCESS +} diff --git a/scripts/symbol-demand.nu b/scripts/symbol-demand.nu index dd3f89abd..32d19e51c 100755 --- a/scripts/symbol-demand.nu +++ b/scripts/symbol-demand.nu @@ -8,7 +8,7 @@ # intra-closure (the closure ships them itself) and are excluded. # # Output: a ranked table (symbol, expected install-name, #referencing binaries). Phase B.3 -# grinds this in rank order; pair with tbd-diff.py (the supply side). +# grinds this in rank order; pair with cider-tbd-diff (the supply side). # # Runs on Linux against Mach-O binaries using llvm tools (llvm-objdump / llvm-otool), so no # macOS host is needed. @@ -198,7 +198,7 @@ def main [ print "Generated by `scripts/symbol-demand.nu`. Demand side of the Phase B gap: system-library symbols imported by the scanned Mach-O binaries \(bind tables), excluding intra-closure @rpath imports, ranked by how many distinct binaries -reference each. Cross-check ownership/availability with `scripts/tbd-diff.py`. +reference each. Cross-check ownership/availability with `cider-tbd-diff`. " print $"- Binaries scanned: **($bins | length)**" print $"- Distinct system symbols imported: **($ranked | length)**" diff --git a/scripts/tbd-diff.py b/scripts/tbd-diff.py deleted file mode 100755 index 04b6a43b7..000000000 --- a/scripts/tbd-diff.py +++ /dev/null @@ -1,349 +0,0 @@ -#!/usr/bin/env python3 -"""tbd-diff.py - the supply side of the Phase B symbol gap. - -Parses the SDK's libSystem.tbd re-export closure to get the *official* -macOS-14 export set, extracts Darling's *actual* export set from our built -system dylibs, and diffs them - optionally intersected with the demand list -from symbol-demand.nu so the output is exactly the symbols that (a) real -binaries import, (b) macOS 14 provides, and (c) Darling still lacks. - -No third-party deps (no PyYAML): the .tbd files are regular enough to parse -directly. Handles tbd-version 4 (YAML-ish, the 14.4 SDK) and tbd-version 5 -(JSON) shells. - -Usage: - scripts/tbd-diff.py --sdk \ - [--arch x86_64] [--platform macos] \ - [--root ] \ - [--cider-root ] \ - [--demand ] \ - [--out scratch/symbol-gap.md] [--json out.json] - -Typical: - SDK=$(nix eval --raw 'github:NixOS/nixpkgs/#legacyPackages.x86_64-darwin.apple-sdk.outPath') - scripts/tbd-diff.py --sdk "$SDK" \ - --cider-root result/libexec/cider/usr/lib \ - --demand scratch/demand.json --out scratch/symbol-gap.md -""" - -import argparse -import json -import os -import re -import subprocess -import sys - - -def find_sdk_root(sdk): - """Locate the MacOSX*.sdk directory inside an apple-sdk store path.""" - if os.path.isfile(os.path.join(sdk, "usr", "lib", "libSystem.tbd")): - return sdk - for dirpath, dirnames, _ in os.walk(sdk): - for d in dirnames: - if d.endswith(".sdk"): - cand = os.path.join(dirpath, d) - if os.path.isfile(os.path.join(cand, "usr", "lib", "libSystem.tbd")): - return cand - return sdk - - -def _target_matches(targets, arch, platform): - """True if any `arch-platform` target token matches (arch e.g. x86_64).""" - want = f"{arch}-{platform}" - for t in targets: - t = t.strip().strip("'\"") - if t == want: - return True - return False - - -def _collect_bracket_list(lines, i): - """Given lines[i] contains a '[', accumulate through the matching ']'. - Returns (items, next_index).""" - buf = [] - depth = 0 - while i < len(lines): - seg = lines[i] - depth += seg.count("[") - seg.count("]") - buf.append(seg) - i += 1 - if depth <= 0: - break - text = " ".join(buf) - inside = text[text.find("[") + 1 : text.rfind("]")] - items = [x.strip().strip("'\"") for x in inside.split(",")] - return [x for x in items if x], i - - -def parse_tbd_v4(path, arch, platform): - """Return (install_names, reexports, symbols) for the given target. - - install_names: list (this file's install-name(s)) - reexports: list of install names re-exported (target-filtered) - symbols: set of exported symbols (target-filtered, real symbols only) - """ - with open(path, "r", errors="replace") as fh: - raw = fh.read() - - # tbd-version 5 is JSON. - if raw.lstrip().startswith("{") or "tapi-tbd-v5" in raw[:64]: - return parse_tbd_v5(raw, arch, platform) - - lines = raw.splitlines() - install_names = [] - reexports = [] - symbols = set() - - # In tbd v4, top-level keys sit at column 0 (no indentation); the content - # of a section (its list entries and their `targets:`/`symbols:`/ - # `libraries:` fields) is indented. Track the current top-level section by - # watching for unindented `key:` lines; treat everything indented as its - # content. This is robust to intervening keys (objc-classes, - # allowable-clients, ...) that a heuristic reset would trip over. - section = None # "exports" | "reexports" | "reexported-libraries" | None - cur_targets = None # per-entry targets, reset at each `- targets:` item - - def in_target(): - return cur_targets is None or _target_matches(cur_targets, arch, platform) - - i = 0 - while i < len(lines): - ln = lines[i] - s = ln.strip() - indented = ln[:1] in (" ", "\t") - - # Top-level key (column 0) -> (re)set section context. - if not indented and re.match(r"^[A-Za-z_][A-Za-z0-9_-]*:", s): - key = s.split(":", 1)[0] - if key in ("exports", "reexports", "reexported-libraries"): - section = key - else: - section = None - m = re.match(r"^install-name:\s*'?([^'\n]+?)'?\s*$", s) - if m: - install_names.append(m.group(1).strip().strip("'\"")) - i += 1 - continue - - if section is None: - i += 1 - continue - - # Inside a section: entries begin with `- targets:`; each has one of - # symbols/weak-symbols (exports) or libraries (reexported-libraries). - if "targets:" in s: - cur_targets, i = _collect_bracket_list(lines, i) - continue - - if section == "reexported-libraries" and "libraries:" in s: - libs, i = _collect_bracket_list(lines, i) - if in_target(): - reexports.extend(libs) - continue - - if section in ("exports", "reexports") and \ - ("symbols:" in s or "weak-symbols:" in s): - syms, i = _collect_bracket_list(lines, i) - if in_target(): - for sym in syms: - if sym.startswith("$ld$"): # linker-directive pseudo-symbols - continue - symbols.add(sym) - continue - - i += 1 - - return install_names, reexports, symbols - - -def parse_tbd_v5(raw, arch, platform): - """Minimal tbd-v5 (JSON) parser.""" - data = json.loads(raw) - install_names, reexports, symbols = [], [], set() - want = f"{arch}-{platform}" - for lib in data.get("main_library", {}).get("install_names", []): - install_names.append(lib.get("name")) - main = data.get("main_library", {}) - for grp in main.get("reexported_libraries", []): - if want in grp.get("targets", []): - reexports.extend(grp.get("names", [])) - for grp in main.get("exported_symbols", []): - tgts = grp.get("targets", []) - if want in tgts or not tgts: - for kind in ("global", "data", "text", "weak"): - for sym in grp.get(kind, []): - if not sym.startswith("$ld$"): - symbols.add(sym) - return install_names, reexports, symbols - - -def tbd_path_for_install_name(sdk_root, install_name): - """Map an install name (/usr/lib/system/libx.dylib) to its .tbd in the SDK.""" - rel = install_name.lstrip("/") - base = os.path.join(sdk_root, rel) - for cand in (base[:-6] + ".tbd" if base.endswith(".dylib") else base, - base + ".tbd", base): - if os.path.isfile(cand): - return cand - return None - - -def collect_official(sdk_root, root_tbd, arch, platform): - """Walk the reexport closure from root_tbd; return {symbol: install_name}.""" - supply = {} - seen = set() - stack = [root_tbd] - while stack: - tbd = stack.pop() - if tbd in seen or not tbd or not os.path.isfile(tbd): - continue - seen.add(tbd) - names, reexports, symbols = parse_tbd_v4(tbd, arch, platform) - owner = names[0] if names else tbd - for sym in symbols: - supply.setdefault(sym, owner) - for rex in reexports: - nxt = tbd_path_for_install_name(sdk_root, rex) - if nxt: - stack.append(nxt) - return supply - - -def collect_cider(root, arch): - """Return the set of symbols exported by every dylib under `root`. - - Uses the Mach-O exports trie (llvm-objdump --exports-trie), NOT nm: the - trie is authoritative and, crucially, includes RE-EXPORT aliases. Darling - exports e.g. `_memcpy` as `[re-export] _memcpy (__platform_memmove from - libsystem_platform)` in libsystem_c - these never appear in `nm -gU`, so an - nm-based scan wildly under-reports the real export surface. - """ - objdump = which("llvm-objdump") - if not objdump: - sys.exit("error: need llvm-objdump on PATH (nix shell nixpkgs#llvm)") - exported = set() - dylibs = [] - for dp, _, files in os.walk(root): - for f in files: - if ".dylib" in f: - dylibs.append(os.path.join(dp, f)) - reexp = re.compile(r"\[re-export\]\s+(\S+)") - direct = re.compile(r"^0x[0-9A-Fa-f]+\s+(\S+)") - for lib in dylibs: - try: - out = subprocess.run( - [objdump, "--macho", "--exports-trie", f"--arch={arch}", lib], - capture_output=True, text=True, timeout=120).stdout - except Exception: - continue - for line in out.splitlines(): - m = reexp.search(line) - if m: - exported.add(m.group(1)) - continue - m = direct.match(line.strip()) - if m and m.group(1).startswith("_"): - exported.add(m.group(1)) - return exported, len(dylibs) - - -def which(x): - for p in os.environ.get("PATH", "").split(os.pathsep): - cand = os.path.join(p, x) - if os.path.isfile(cand) and os.access(cand, os.X_OK): - return cand - return None - - -def load_demand(path): - if not path: - return None - with open(path) as fh: - data = json.load(fh) - return {e["symbol"]: e.get("refs", 0) for e in data.get("symbols", [])} - - -def main(): - ap = argparse.ArgumentParser(description=__doc__, - formatter_class=argparse.RawDescriptionHelpFormatter) - ap.add_argument("--sdk", required=True) - ap.add_argument("--arch", default="x86_64") - ap.add_argument("--platform", default="macos") - ap.add_argument("--root", help="root .tbd (default: /usr/lib/libSystem.tbd)") - ap.add_argument("--cider-root", help="dir of Darling's built dylibs") - ap.add_argument("--demand", help="symbol-demand.json to intersect with") - ap.add_argument("--out", help="write markdown report here") - ap.add_argument("--json", help="write json report here") - args = ap.parse_args() - - sdk_root = find_sdk_root(args.sdk) - root_tbd = args.root or os.path.join(sdk_root, "usr", "lib", "libSystem.tbd") - if not os.path.isfile(root_tbd): - sys.exit(f"error: no root tbd at {root_tbd}") - - official = collect_official(sdk_root, root_tbd, args.arch, args.platform) - demand = load_demand(args.demand) - - cider = None - n_dylibs = 0 - if args.cider_root and os.path.isdir(args.cider_root): - cider, n_dylibs = collect_cider(args.cider_root, args.arch) - - lines = [] - lines.append(f"# libSystem symbol gap ({args.arch}-{args.platform})\n") - lines.append("Generated by `scripts/tbd-diff.py`. Supply side: the SDK " - f"`libSystem.tbd` re-export closure ({os.path.basename(sdk_root)}).\n") - lines.append(f"- Official exported symbols (SDK closure): **{len(official)}**") - if cider is not None: - lines.append(f"- Darling exported symbols ({n_dylibs} dylibs): **{len(cider)}**") - if demand is not None: - lines.append(f"- Demanded symbols (from binaries): **{len(demand)}**") - lines.append("") - - report = {"arch": args.arch, "platform": args.platform, - "official_count": len(official)} - - if cider is not None: - missing = {s: official[s] for s in official if s not in cider} - report["cider_count"] = len(cider) - report["missing_count"] = len(missing) - lines.append(f"## Missing from Darling (official − cider): {len(missing)}\n") - - if demand is not None: - worklist = sorted( - ((demand.get(s, 0), s, official[s]) for s in missing if s in demand), - reverse=True) - report["demanded_missing_count"] = len(worklist) - lines.append(f"### Demanded work list (needed ∩ macOS14 − cider): " - f"**{len(worklist)}**\n") - lines.append("| # refs | symbol | owner |") - lines.append("|---:|:---|:---|") - for refs, sym, owner in worklist: - lines.append(f"| {refs} | `{sym}` | `{owner}` |") - lines.append("") - - # Demanded symbols not present in the SDK closure at all (framework - # symbols, or our categorization gaps). - not_in_sdk = sorted(s for s in demand if s not in official) - report["demanded_not_in_sdk"] = len(not_in_sdk) - lines.append(f"### Demanded but absent from libSystem tbd closure: " - f"**{len(not_in_sdk)}** (likely framework-owned)\n") - else: - lines.append("_(no --cider-root given; supply-only run. Provide the " - "built Darling dylibs to compute the gap.)_\n") - - text = "\n".join(lines) + "\n" - if args.out: - with open(args.out, "w") as fh: - fh.write(text) - print(f"wrote {args.out}", file=sys.stderr) - else: - sys.stdout.write(text) - if args.json: - with open(args.json, "w") as fh: - json.dump(report, fh, indent=2) - print(f"wrote {args.json}", file=sys.stderr) - - -if __name__ == "__main__": - main() -- 2.51.2