diff --git a/CLAUDE.md b/CLAUDE.md index c302e57..1d4a4ea 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -7,7 +7,7 @@ attempt shelved — see docs/evented-attempt.md). ## before pushing - `zig fmt --check .` and `zig build test` -- MUST use `-Dtarget=x86_64-linux-gnu` for production (musl breaks RocksDB) +- production is `-Dtarget=x86_64-linux-gnu` (glibc malloc is load-bearing for RSS). a static musl+mimalloc build also works on 0.16 (`-Duse_mimalloc=true`) but is unproven at runtime — see [docs/musl-investigation.md](docs/musl-investigation.md) - ReleaseFast has a known double-free — do not use ## deploy diff --git a/build.zig b/build.zig index e66d15a..eefb8d7 100644 --- a/build.zig +++ b/build.zig @@ -42,6 +42,11 @@ pub fn build(b: *std.Build) void { build_options.addOption([]const u8, "optimize", @tagName(optimize)); const use_gpa = b.option(bool, "use_gpa", "use GeneralPurposeAllocator for leak detection (slow)") orelse false; build_options.addOption(bool, "use_gpa", use_gpa); + // link mimalloc as the malloc override. required for musl (whose own malloc + // is unusable at our thread count); aliases malloc/free so both c_allocator + // and RocksDB's C++ allocations route through it. see docs/musl-investigation.md. + const use_mimalloc = b.option(bool, "use_mimalloc", "link mimalloc as the system malloc override") orelse false; + build_options.addOption(bool, "use_mimalloc", use_mimalloc); // relay executable const relay_mod = b.createModule(.{ @@ -53,6 +58,7 @@ pub fn build(b: *std.Build) void { relay_mod.addImport("build_options", build_options.createModule()); relay_mod.link_libc = true; relay_mod.link_libcpp = true; + if (use_mimalloc) linkMimalloc(b, relay_mod, target); const relay = b.addExecutable(.{ .name = "zlay", .root_module = relay_mod, @@ -81,3 +87,27 @@ pub fn build(b: *std.Build) void { }); test_step.dependOn(&b.addRunArtifact(t).step); } + +/// compile mimalloc's single-source amalgamation into `mod` with malloc override +/// enabled, so the standard malloc/free symbols (used by both std.heap.c_allocator +/// and RocksDB's C++) resolve to mimalloc instead of the host libc allocator. +fn linkMimalloc(b: *std.Build, mod: *std.Build.Module, target: std.Build.ResolvedTarget) void { + const mimalloc = b.dependency("mimalloc", .{}); + const base_flags = [_][]const u8{ + "-DMI_MALLOC_OVERRIDE", // alias malloc/free/realloc to mi_* + "-DNDEBUG", + "-DMI_SECURE=0", + "-std=c11", + "-O2", + "-fno-sanitize=undefined", // mimalloc relies on benign UB the ubsan trips on + "-Wno-date-time", // options.c uses __DATE__/__TIME__; zig cc errors otherwise + }; + // musl needs MI_LIBC_MUSL so mimalloc doesn't assume glibc internals (prim.h). + const musl_flags = base_flags ++ [_][]const u8{"-DMI_LIBC_MUSL"}; + const flags: []const []const u8 = if (target.result.abi.isMusl()) &musl_flags else &base_flags; + mod.addIncludePath(mimalloc.path("include")); + mod.addCSourceFile(.{ + .file = mimalloc.path("src/static.c"), + .flags = flags, + }); +} diff --git a/build.zig.zon b/build.zig.zon index a8abdd6..10e8974 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -20,6 +20,10 @@ .url = "https://github.com/zzstoatzz/rocksdb-zig/archive/cdef67bb2141df4eb4906d367d60285d1527ab3f.tar.gz", .hash = "rocksdb-9.7.4-z_CUTsLJAAArLT8DEcD1Li7XthpIinhXrcitqQLFIKLK", }, + .mimalloc = .{ + .url = "https://github.com/microsoft/mimalloc/archive/refs/tags/v2.3.2.tar.gz", + .hash = "N-V-__8AAFWPVAACRdwIuqz69BNLZ8fncUWjjv1FzFHiSsQW", + }, }, .paths = .{ "build.zig", diff --git a/docs/deployment.md b/docs/deployment.md index 92fec42..0e1b332 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -26,7 +26,7 @@ the full `Dockerfile` exists for CI/standalone builds but is slow on Mac (cross- ### build flags -- `-Dtarget=x86_64-linux-gnu` — **must use glibc**, not musl. zig 0.15's C++ codegen for musl produces illegal instructions in RocksDB's LRU cache. +- `-Dtarget=x86_64-linux-gnu` — production default, links glibc. the glibc malloc (per-thread arenas + `madvise` page return + `malloc_trim`) is load-bearing for RSS at ~2,800 threads. **NB:** the old "musl breaks RocksDB / illegal instructions in the LRU cache" claim was a zig 0.15 finding and **no longer reproduces on 0.16** — RocksDB compiles and links clean for `x86_64-linux-musl`; the only remaining coupling to glibc is three malloc-introspection calls we make (`mallinfo`, `malloc_info`, `malloc_trim`). see [musl-investigation.md](musl-investigation.md) for the full 0.16 ground-truth and the mimalloc static-binary experiment. - `-Dcpu=baseline` — required when building inside Docker/QEMU (not needed for `zlay-publish-remote` since it builds natively). - `-Doptimize=ReleaseSafe` — safety checks on, optimizations on. production default since 2026-03-05. previously caused OOM (see [incident-2026-03-04.md](incident-2026-03-04.md)) — resolved by the frame pool moving heavy work off reader threads. diff --git a/docs/musl-investigation.md b/docs/musl-investigation.md new file mode 100644 index 0000000..921f194 --- /dev/null +++ b/docs/musl-investigation.md @@ -0,0 +1,92 @@ +# musl + mimalloc static binary (2026-06-12) + +investigation into shipping zlay as a fully static `FROM scratch` binary instead +of a glibc-dynamic one. motivation: a static musl build runs on any linux distro +with zero shared-lib deps, shrinks the runtime image to just the binary, and is +hermetic/reproducible — the marquee zig deployment story. + +## the stale assumption, corrected + +`docs/deployment.md` long claimed "must use glibc, not musl — zig 0.15's C++ +codegen for musl produces illegal instructions in RocksDB's LRU cache." that was +a **0.15** finding. on **0.16 it no longer reproduces at build time**: a +`-Dtarget=x86_64-linux-musl` build compiles RocksDB's C++ clean and gets all the +way to linking. the only things that blocked the link were three glibc-only +malloc calls *we* make: + +| symbol | site | role | +|---|---|---| +| `mallinfo` | `broadcaster.zig` metrics | main-arena heap gauge — already superseded by smaps | +| `malloc_info` | `broadcaster.zig` metrics | all-arena heap gauge (glibc XML) | +| `malloc_trim` | `main.zig` `gcLoop` | **load-bearing**: returns freed pages to the OS | + +so the real coupling was never RocksDB — it was our dependence on glibc's malloc +introspection/control surface. + +> caveat: "compiles + links clean" is necessary, not sufficient. the 0.15 bug was +> an *illegal instruction* (runtime), not a compile error. confirming it's truly +> gone on 0.16 requires *running* the musl binary and exercising RocksDB's LRU +> cache under load — that's Phase 3 below, which needs a linux/musl runtime. + +## why mimalloc, not bare musl or jemalloc + +musl's own malloc has limited/no per-thread arenas and contends badly under +multithreading — unusable at zlay's ~2,800 threads. of the alternatives +(current as of 2026-06): + +- **mimalloc** (Microsoft, v2.3.2 / 2026-04-29): builds and runs cleanly on musl + (`MI_LIBC_MUSL`), and benchmarks push musl *above* glibc. has `mi_collect` for + page return (our `malloc_trim` replacement). +- **jemalloc**: reported to segfault under musl. avoided. + +mimalloc's static override (`-DMI_MALLOC_OVERRIDE`) aliases `malloc`/`free`/etc +to `mi_*`, so **both** `std.heap.c_allocator` (our zig base allocator) and +RocksDB's C++ allocations route through mimalloc with no custom zig wrapper. + +## what was built (Phase 0→2, completed) + +- `build.zig`: `-Duse_mimalloc=true` compiles mimalloc's single-source + `src/static.c` into the relay module with override on (`linkMimalloc`). pinned + via `build.zig.zon` (`mimalloc` dep, v2.3.2). +- the three glibc malloc calls are re-gated on `builtin.target.isGnuLibC()` + instead of `os.tag == .linux`; `malloc_trim` → `mi_collect(true)` when + `use_mimalloc`. on a non-glibc/mimalloc build the metrics block and externs + compile out entirely. + +### verified on this machine +- `-Dtarget=x86_64-linux-musl -Doptimize=ReleaseSafe -Duse_mimalloc=true` → + **statically linked** ELF (no interpreter), RocksDB included. +- `nm`: `malloc` resolves to the same address as `mi_malloc` (override active); + `mi_collect` present. +- glibc production build (`x86_64-linux-gnu`, no `-Duse_mimalloc`) unchanged; + native test suite 95 pass / 1 skip; `zig fmt` clean. + +### build invocations +``` +# production (unchanged) +zig build -Dtarget=x86_64-linux-gnu -Doptimize=ReleaseSafe + +# static musl + mimalloc (new) +zig build -Dtarget=x86_64-linux-musl -Doptimize=ReleaseSafe -Duse_mimalloc=true +``` + +## Phase 3 — runtime validation (NOT done; needs a linux/musl runtime) + +cannot be verified by cross-compiling on macOS. run in an alpine container or a +k8s canary: + +1. **kill the 0.15 ghost for real:** run the musl binary, exercise the collection + index / RocksDB LRU cache under load, confirm zero illegal instructions. +2. **RSS at thread-scale:** measure against the glibc baseline (~1.1 GiB at + ~2,255 hosts). mimalloc should match or beat; this is the whole point. +3. confirm pthreads/io behave on musl (we're on the Threaded backend → plain + pthreads, low risk). + +## decision (deferred to Phase 3 results) + +if RSS holds and nothing faults: we gain a `FROM scratch` static image option +(`Dockerfile.runtime` could drop to `FROM scratch`). if RocksDB faults at +runtime on musl/0.16: we've now *proven* the blocker on 0.16 and can close the +door deliberately, rather than inheriting a year-old assumption. + +current prod is unchanged: glibc, `-Dtarget=x86_64-linux-gnu`, no mimalloc. diff --git a/src/internal/broadcaster.zig b/src/internal/broadcaster.zig index 47ff95f..17823e5 100644 --- a/src/internal/broadcaster.zig +++ b/src/internal/broadcaster.zig @@ -1265,28 +1265,32 @@ fn appendProcMetrics(w: *Io.Writer, io: Io) void { } } else |_| {} - // glibc malloc stats — mallinfo fields are c_int, bitcast to u32 to extend range to 4 GiB. - // note: mallinfo only reports the main arena, not per-thread arenas. - // for accurate total heap picture, rely on RssAnon from /proc/self/status above. - const mi = mallinfo(); - const arena: u64 = @as(u32, @bitCast(mi.arena)); - const in_use: u64 = @as(u32, @bitCast(mi.uordblks)); - const free_bytes: u64 = @as(u32, @bitCast(mi.fordblks)); - const mmap_bytes: u64 = @as(u32, @bitCast(mi.hblkhd)); - w.print( - \\# TYPE relay_malloc_arena_bytes gauge - \\relay_malloc_arena_bytes {d} - \\ - \\# TYPE relay_malloc_in_use_bytes gauge - \\relay_malloc_in_use_bytes {d} - \\ - \\# TYPE relay_malloc_free_bytes gauge - \\relay_malloc_free_bytes {d} - \\ - \\# TYPE relay_malloc_mmap_bytes gauge - \\relay_malloc_mmap_bytes {d} - \\ - , .{ arena, in_use, free_bytes, mmap_bytes }) catch {}; + // glibc malloc stats — mallinfo/malloc_info are glibc-only. on musl (where we + // run mimalloc instead) these symbols don't exist; the whole block compiles + // out and RssAnon from /proc/self/status above remains the source of truth. + if (comptime builtin.target.isGnuLibC()) { + // mallinfo fields are c_int, bitcast to u32 to extend range to 4 GiB. + // note: mallinfo only reports the main arena, not per-thread arenas. + const mi = mallinfo(); + const arena: u64 = @as(u32, @bitCast(mi.arena)); + const in_use: u64 = @as(u32, @bitCast(mi.uordblks)); + const free_bytes: u64 = @as(u32, @bitCast(mi.fordblks)); + const mmap_bytes: u64 = @as(u32, @bitCast(mi.hblkhd)); + w.print( + \\# TYPE relay_malloc_arena_bytes gauge + \\relay_malloc_arena_bytes {d} + \\ + \\# TYPE relay_malloc_in_use_bytes gauge + \\relay_malloc_in_use_bytes {d} + \\ + \\# TYPE relay_malloc_free_bytes gauge + \\relay_malloc_free_bytes {d} + \\ + \\# TYPE relay_malloc_mmap_bytes gauge + \\relay_malloc_mmap_bytes {d} + \\ + , .{ arena, in_use, free_bytes, mmap_bytes }) catch {}; + } // all-arena heap picture via malloc_info() (mallinfo above is main-arena only). // with ~2670 worker threads glibc spreads allocations across many per-thread @@ -1341,9 +1345,10 @@ const c_free = @extern(*const fn (?*anyopaque) callconv(.c) void, .{ .name = "fr const malloc_info_fn = @extern(*const fn (options: c_int, stream: *CFile) callconv(.c) c_int, .{ .name = "malloc_info" }); fn readMallocInfo() ?MallocInfo { - // comptime if prunes the linux-only externs from analysis on other hosts, - // so `zig build test` on macOS doesn't try to link malloc_info/open_memstream. - return if (comptime builtin.os.tag == .linux) readMallocInfoLinux() else null; + // comptime if prunes the glibc-only externs from analysis elsewhere, so neither + // `zig build test` on macOS nor a musl/mimalloc build tries to link + // malloc_info/open_memstream (which exist only in glibc). + return if (comptime builtin.target.isGnuLibC()) readMallocInfoLinux() else null; } fn readMallocInfoLinux() ?MallocInfo { diff --git a/src/main.zig b/src/main.zig index 84454f8..9cf0ee6 100644 --- a/src/main.zig +++ b/src/main.zig @@ -42,10 +42,17 @@ const build_options = @import("build_options"); const getenv = util.getenv; const parseEnvInt = util.parseEnvInt; -const malloc_trim: ?*const fn (pad: usize) callconv(.c) c_int = if (builtin.os.tag == .linux) +// page-return after GC. glibc exposes malloc_trim; when linked against mimalloc +// (musl builds) we use mi_collect instead — musl's own malloc has neither and the +// branch compiles out. see docs/musl-investigation.md. +const malloc_trim: ?*const fn (pad: usize) callconv(.c) c_int = if (builtin.target.isGnuLibC()) @extern(*const fn (pad: usize) callconv(.c) c_int, .{ .name = "malloc_trim" }) else null; +const mi_collect: ?*const fn (force: bool) callconv(.c) void = if (build_options.use_mimalloc) + @extern(*const fn (force: bool) callconv(.c) void, .{ .name = "mi_collect" }) +else + null; const log = std.log.scoped(.relay); @@ -477,8 +484,11 @@ fn gcLoop(dp: *event_log_mod.DiskPersist, io: Io) void { log.warn("event log GC failed: {s}", .{@errorName(err)}); }; - // return freed pages to OS (glibc-specific, no-op on other platforms) - if (comptime malloc_trim) |trim| { + // return freed pages to OS (no-op when neither allocator is linked) + if (comptime mi_collect) |collect| { + collect(true); + log.info("gc: mi_collect complete", .{}); + } else if (comptime malloc_trim) |trim| { _ = trim(0); log.info("gc: malloc_trim complete", .{}); }