diff --git a/Dockerfile b/Dockerfile index 1b60a63..d991b0d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -8,8 +8,9 @@ RUN curl -fSL https://ziglang.org/builds/zig-x86_64-linux-0.16.0-dev.3059+42e33d ENV PATH=/opt/zig-x86_64-linux-0.16.0-dev.3059+42e33db9d:$PATH WORKDIR /build -# Uring networking patch preserved in patches/ for when upstream stabilizes -# (see docs/evented-attempt.md). Not applied — using Io.Threaded backend. +# patch Io.Uring networking (stubbed as Unavailable upstream, zig#31723) +COPY patches/ patches/ +RUN patch /opt/zig-x86_64-linux-0.16.0-dev.3059+42e33db9d/lib/std/Io/Uring.zig < patches/uring-networking.patch # fetch dependencies first (cacheable — only changes when build.zig.zon changes) COPY build.zig build.zig.zon ./ @@ -17,7 +18,8 @@ RUN zig build --fetch-only 2>/dev/null || true # then copy source and build COPY src/ src/ -# ReleaseSafe: Evented GPF was the reason for ReleaseFast — back to Threaded now. +# ReleaseSafe: the production SIGSEGV was a websocket bug, not a fiber issue. +# trying Evented + ReleaseSafe — if the repro GPF hits production, fall back to ReleaseFast. RUN zig build -Doptimize=ReleaseSafe -Dcpu=baseline -Dtarget=x86_64-linux-gnu FROM --platform=linux/amd64 debian:bookworm-slim diff --git a/docs/evented-attempt.md b/docs/evented-attempt.md index b598deb..10757bd 100644 --- a/docs/evented-attempt.md +++ b/docs/evented-attempt.md @@ -1,9 +1,12 @@ -# Evented backend attempt (2026-03 to 2026-04) +# Evented backend (2026-03 to present) -we spent roughly a month trying to run zlay on `Io.Evented` (io_uring fibers) -instead of `Io.Threaded` (one OS thread per task). the goal was to drop from -~2,800 threads to ~35. it didn't work out — the upstream fiber machinery is -still experimental and unstable. this doc records what we did so it's not lost. +log of getting zlay running on `Io.Evented` (io_uring fibers) instead of +`Io.Threaded` (one OS thread per task). goal: drop from ~2,800 threads to ~35. + +after 28 commits fixing cross-Io issues and a brief revert to Threaded, we +discovered the production SIGSEGV was a websocket handshake bug (TCP split +mid-CRLF, fixed in websocket.zig `9ac64da`), not a fiber context-switch +issue. Evented backend is back, now running ReleaseSafe. ## what we built @@ -70,14 +73,16 @@ configurations. - `docs/stdlib-patches.md` — catalog of all 6 workarounds we needed - this document -## when to try again +## timeline + +- 2026-03-08: first Evented deploy (`39134d1`) +- 2026-03-08 to 2026-04-04: 28 commits fixing cross-Io crashes +- 2026-04-05: reverted to Threaded after SIGSEGV blamed on fiber machinery +- 2026-04-05: discovered real cause — websocket handshake TCP split bug +- 2026-04-05: fixed websocket.zig (`9ac64da`), re-enabled Evented + ReleaseSafe -revisit when: -1. upstream implements uring networking (zig#31723 closed) -2. `fiber.zig` gets meaningful changes (context switch rewrite or fix) -3. a zig release (not dev build) ships with Evented marked as stable +## fallback -the dual-Io architecture, DbRequestQueue, and cross-Io segregation patterns -are all still in the codebase (dead code paths behind `Backend == Io.Evented` -checks). switching back is a one-line change in `src/main.zig:58` plus -re-enabling the patch in the Dockerfile. +if Evented + ReleaseSafe hits the repro GPF in production, switch to +ReleaseFast in the Dockerfile. if that also crashes, flip `Backend` back +to `Io.Threaded` in `src/main.zig:58` — one-line change. diff --git a/src/main.zig b/src/main.zig index 5cbe314..4a8231c 100644 --- a/src/main.zig +++ b/src/main.zig @@ -48,18 +48,17 @@ const log = std.log.scoped(.relay); pub const default_stack_size = 8 * 1024 * 1024; // -- Io backend selection -- -// Io.Threaded: one OS thread per concurrent task (~2,800 subscribers + services). -// not what we want long-term — Evented (fibers on io_uring) drops thread count -// from ~2,800 to ~35 — but the upstream fiber machinery is not ready. +// Io.Evented (fibers on io_uring): ~35 threads instead of ~2,800 (one per PDS). +// Networking via patched Uring.zig (patches/uring-networking.patch) — implements +// listen, accept, connect, read, write, send via io_uring opcodes. DNS (netLookup) +// is NOT patched; subscribers resolve hostnames through pool_io (Threaded) instead. // -// We built and shipped a full Evented backend (see docs/evented-attempt.md and -// patches/uring-networking.patch). After 28 commits fixing cross-Io crashes, -// heap corruption, mutex incompatibilities, and a ReleaseSafe GPF in fiber -// context-switch, the remaining bug is a probabilistic SIGSEGV under ReleaseFast -// (~every 30-90 min under load). Root cause is in std.Io.fiber.contextSwitch -// (zig 0.16-dev stdlib). As of dev.3091, fiber.zig is unchanged and Uring -// networking is still fully stubbed upstream. Switch back when upstream stabilizes. -const Backend = Io.Threaded; +// History: 28 commits of cross-Io fixes (see docs/evented-attempt.md). The +// production SIGSEGV that prompted a Threaded revert turned out to be a websocket +// handshake bug (TCP split mid-CRLF), not a fiber issue. Fixed in websocket.zig +// 9ac64da. The repro (scripts/repro_evented.zig) still GPFs under ReleaseSafe +// but production code paths may differ — deploying ReleaseSafe to find out. +const Backend = Io.Evented; var backend: Backend = undefined; var debug_threaded_io: Io.Threaded = undefined;