From efb4fbe6bafabf43f8979eafa8c6bab4267b799d Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Tue, 21 Jul 2026 15:39:12 -0500 Subject: [PATCH] verify jss block checksums --- docs/jss-format-v1.md | 5 ++-- docs/semantic-parity.md | 2 +- docs/upstream-harness.md | 1 + src/internal/segment.zig | 23 +++++------------ src/internal/segment_fixture_test.zig | 20 +++++++++++++++ src/internal/segment_writer.zig | 2 +- src/internal/zstd.zig | 37 +++++++++++++++++++++++---- 7 files changed, 64 insertions(+), 26 deletions(-) diff --git a/docs/jss-format-v1.md b/docs/jss-format-v1.md index 1991d4c..1838d0c 100644 --- a/docs/jss-format-v1.md +++ b/docs/jss-format-v1.md @@ -79,8 +79,9 @@ no-false-negative query accelerators. active: magic + 252 zero bytes, then fully-flushed frames, no footer. checksum field 0 = active. sealing: footer_offset = size; walk frames for stats; write footer; fsync; pwrite header; fsync. per-block integrity = zstd frame CRC only (file xxh3 covers header+footer, not block bytes). -NOTE: zig std's zstd Decompress cannot verify frame CRC yet (panics if enabled) — stream skips -it for now; revisit when std implements it or libzstd is linked for the writer. +Stream decodes through its pinned vendored libzstd, which verifies that content checksum. +The upstream-produced fixture mutation test flips only a block checksum byte, proves the +JSS metadata checksum still passes, and requires block decode to fail. ## ops notes - no CLI flag forces small segments; sealed files come from the bootstrap→merge cutover diff --git a/docs/semantic-parity.md b/docs/semantic-parity.md index eb80920..1312dbc 100644 --- a/docs/semantic-parity.md +++ b/docs/semantic-parity.md @@ -14,7 +14,7 @@ this document or `bootstrap-semantic-parity.md` is open. |---|---|---| | Bootstrap, merge, retry | Detailed row-by-row contract and adversity receipt | **closed at the current implementation boundary.** Every row in `bootstrap-semantic-parity.md` is closed; the offline candidate rerun survived all eight lifecycle crash seams. The experiment gate still requires rerunning that oracle against the exact eventual build. | | JSS v1 storage | Upstream-produced fixture, reciprocal segment parsing, checksum/index/gloom vectors | closed for sealed format; rerun reciprocal corpus before experiment | -| Archive XRPC | Official Go client plus Stream conformance replay over listSegments/getSegment/getBlock/planBackfill | **closed.** With the pinned Zig dependencies cached, `just archive-contract` seeds three real sealed JSS segments and runs without network access. The pinned Go client consumed 5,000 ordered whole-segment rows, the exact 2,500-row DID block plan, 50 required identity sentinels for an empty collection match, the 234-row `(1000,1234]` bound, and a duplicate-free archive-to-live cutover at cursor 5000. Direct conformance also proves checksums/ETags, byte-identical segments, raw block decode, error names, DID/collection blooms, plan modes, and HTTP ranges. The planner uses refcounted manifest-resident headers, block envelopes, DID blooms, and collection summaries; the offline contract hides every JSS file after startup and still requires an exact plan, proving the request path performs no segment-file I/O. The harness rejects any upstream checkout not at the recorded pin. | +| Archive XRPC | Official Go client plus Stream conformance replay over listSegments/getSegment/getBlock/planBackfill | **closed.** With the pinned Zig dependencies cached, `just archive-contract` seeds three real sealed JSS segments and runs without network access. The pinned Go client consumed 5,000 ordered whole-segment rows, the exact 2,500-row DID block plan, 50 required identity sentinels for an empty collection match, the 234-row `(1000,1234]` bound, and a duplicate-free archive-to-live cutover at cursor 5000. Direct conformance also proves checksums/ETags, byte-identical segments, raw block decode, error names, DID/collection blooms, plan modes, and HTTP ranges. Sealed reads use the pinned vendored libzstd so per-block content checksums are enforced independently of the header/footer xxh3; mutating only an upstream fixture's block checksum leaves metadata parsing valid and is rejected at decode, matching upstream. The planner uses refcounted manifest-resident headers, block envelopes, DID blooms, and collection summaries; the offline contract hides every JSS file after startup and still requires an exact plan, proving the request path performs no segment-file I/O. The harness rejects any upstream checkout not at the recorded pin. | | Subscribe v1/v2 | Wire/filter/cursor/compression unit and local e2e coverage | Cursor parsing/resolution precedes upgrade; both endpoints use the seq/time-us magnitude split, v1 clamps below-floor seqs, and v2 rejects them. Timestamp translation uses bounded positional reads of real sealed JSS headers, block indexes, and one selected block before upgrade; disk/decode/index faults are retryable generic 503s, logged internally and counted as `resolve_failed`. Hot/cold scan, skip, encode, options-update, oversize-parser, clean peer/server close, cursor-resolution, and sustained adversarial-rate boundaries feed canonical metrics, with deterministic offline tests derived from upstream. Subscriber transport sends a ping every 30 seconds, applies a kernel-enforced five-second deadline to every frame write, and sends close code 1001 before intentional server shutdown while keeping transport failure out of clean-disconnect accounting. A failed delivery, cold read, ping, or adversarial-rate check interrupts the server reader and removes the connection instead of leaving a ping-only zombie. V1 now negotiates genuine RFC 7692 deflate with the upstream 128-byte threshold and 32 KiB context takeover; v1 custom zstd and all v2 connections explicitly decline it. V2 dictionary negotiation, exact pinned dictionary download/validators, and v2 binary delivery remain independent of the frozen v1 dictionary contract; pinned official Go clients have decoded both compression families offline. | | Sync 1.1 live verification | Upstream requires durable per-DID chain/hosting state, MST inversion, op-CID consistency, rev replay/future guards, default acceptance of legacy-shaped commits, transparent whole-repo repair, and Atmos delivery scheduling | **closed at the implementation boundary.** Diff-CAR/MST inversion (including upstream's narrow default lenient carve-out), post-state op-CID proof, signed inner/outer consistency, exact decimal size/future-rev/replay gates, exact legacy-shape detection with default `LegacyAccept`, two-phase durable chain/hosting state, and account/identity replay ratchets are implemented. Recoverable decode/inversion/duplicate-path/op-CID/chain failures route to repair; signature failures and outer/inner producer-integrity mismatches bypass it, and no failed event is archived. `prepareRepo` retains and authenticates the canonical signed complete head without a second multi-gigabyte parse/walk. The live path now uses Atmos's 32 worker slots, one worker-held FIFO chain per DID, a 64-pending drop-oldest boundary, completion-order batches of 50/500 ms across DIDs, and `min(inflight)-1` cursor watermarks. Dropped and silent events leave the inflight set without inventing a delivery batch; cursor advancement remains coupled to durable archive boundaries. Offline receipts use real encoded frames, real RocksDB verifier state, and actual worker blocking rather than fixture verdicts or scheduler mocks. See `live-scheduler.md`. | | Sync 1.1 resync ordering | Upstream serializes per DID, drops stale async repairs, and emits a sync tombstone plus authoritative replacements | **closed at the implementation boundary.** The live coordinator has 32 real fetch workers, a 64-job queue, a five-minute fetch budget plus the 1,000 B/s-for-30-seconds slow-transfer guard, a bounded 16,384-entry 5/minute-per-DID limiter, 2,048 pending commits with drop-oldest overflow, fetch outside the DID lane, authenticated apply with older/equal-contradictory head rejection, ordered pending replay, and trigger-ticket-gated outbox delivery. Commit divergence schedules async repair; `#sync` divergence attempts inline repair and queues one retry only for transient failure, preserving the original relay cursor only on inline success. Matching Atmos's gate ownership, a divergent `#sync` repairs before envelope/inner validation; an invalid envelope rev then durably advances only the authenticated chain and drops the original tombstone plus every replacement row as one `live/invalid_rev` event. Valid repairs emit `sync` then bounded 1,024-row `create_resync` batches and stage fetched/pending chain checkpoints under the archive lock before any matching fsync. The shared hot tail is an amortized physical deque: mass replacement eviction cannot shift the full 256 MiB log once per row, and a 20,000-entry burst regression locks that boundary. Wire encoding uses a per-row scratch arena before the tail takes its durable copy, matching upstream's bounded append/publish lifetime instead of retaining v1 and v2 JSON for the entire fetched repository. A real 4,096-record MST repair now completes under a 1 MiB repair-scratch cap; the prior ownership model deterministically fails the same test with `ArchiveAppendFailed`. Loopback-only tests exercise a real HTTP getRepo mmap, real DID resolution/signature, buffering while the response is held open, slow-transfer cancellation, durable RocksDB promotion/cursor coupling, v2 tail publication, and sealed JSS row order. | diff --git a/docs/upstream-harness.md b/docs/upstream-harness.md index 8188fa6..3537f45 100644 --- a/docs/upstream-harness.md +++ b/docs/upstream-harness.md @@ -32,6 +32,7 @@ driver boots real server against simulator, walks lifecycle gated on durable-app - readable log (hot tail): writer-owned deque, deep-copied entries at seq allocation, 256MiB budget, evicts only below durable watermark (pinned above), notify channel per append, encode-once wire memo per entry shared across fan-out; cursor below floor → cold reader over sealed segments. Stream's tail enforces the same pin: an undurable suffix is retained even when it overruns the byte budget, `publishDurable` releases only the newly durable byte suffix before eviction, and scrape-time readable/pinned/overrun gauges are O(1) accumulators rather than a scan under the hot lock. Its physical deque retires prefix entries in O(1) and compacts only after a large amortized prefix; the 20,000-row burst regression catches the former `orderedRemove(0)` implementation that made a real 8,192-row repair quadratic and stalled global ingest. - resync memory lifetime: upstream appends each converted replacement and immediately forwards it to the bounded readable log; it does not retain a second whole-repository wire representation. Stream batches physical archive writes in groups of 1,024 but releases the complete v1/v2 encoder arena after each row is copied into the tail. The offline 4,096-record MST receipt is capped at 1 MiB of repair scratch and mutation-checks the former repair-lifetime JSON ownership by reproducing its exact `ArchiveAppendFailed`. - archive health: free space is collected from the filesystem containing the open archive directory on every scrape; segment-seal latency starts before the pending-block flush and is observed only after footer write/fsync plus finalized-header pwrite/fsync succeeds, using upstream's `0.01 × 2^n` bucket boundaries. +- block integrity: the JSS xxh3 authenticates only header/footer metadata, while upstream's zstd decoder independently verifies each frame's content checksum. Stream uses its pinned vendored libzstd for the same check; flipping only the final checksum byte of an upstream-produced block is the mutation receipt. ## zig oracle v1 (2026-07-12) diff --git a/src/internal/segment.zig b/src/internal/segment.zig index ebbf17b..9a17ea6 100644 --- a/src/internal/segment.zig +++ b/src/internal/segment.zig @@ -5,13 +5,13 @@ //! serves these raw bytes to clients. pure format code — no goroutine-shaped //! lifecycle, no I/O beyond the caller handing us bytes. //! -//! v0 scope: sealed-file parsing over an in-memory buffer (header, block -//! index, columnar block decode, collection index). blooms are skipped -//! (query accelerators, not needed for correct iteration). zstd frame CRC -//! is NOT verified — zig std's Decompress can't yet; the file-level xxh3 -//! (header+footer) IS verified. +//! sealed-file parsing over an in-memory buffer (header, block index, +//! columnar block decode, collection index). blooms are decoded by the +//! resident-manifest layer. The file-level xxh3 authenticates header/footer +//! metadata and libzstd independently verifies every block-frame checksum. const std = @import("std"); +const zstd = @import("zstd.zig"); const Allocator = std.mem.Allocator; @@ -444,18 +444,7 @@ pub fn decodeBlock(allocator: Allocator, buffer: []u8) Error!Block { } fn decompress(allocator: Allocator, frame: []const u8, expected_len: u32) ![]u8 { - var in: std.Io.Reader = .fixed(frame); - // zstd frame CRC verification is unimplemented in std — see module doc - var stream: std.compress.zstd.Decompress = .init(&in, &.{}, .{}); - var out: std.Io.Writer.Allocating = try .initCapacity(allocator, expected_len); - errdefer out.deinit(); - _ = stream.reader.streamRemaining(&out.writer) catch return error.DecompressFailed; - const buffer = try out.toOwnedSlice(); - if (buffer.len != expected_len) { - allocator.free(buffer); - return error.DecompressFailed; - } - return buffer; + return zstd.decompressExact(allocator, frame, expected_len); } inline fn readInt(comptime T: type, bytes: []const u8, offset: usize) T { diff --git a/src/internal/segment_fixture_test.zig b/src/internal/segment_fixture_test.zig index 8ee8328..ebbca64 100644 --- a/src/internal/segment_fixture_test.zig +++ b/src/internal/segment_fixture_test.zig @@ -94,6 +94,26 @@ test "fixture: all blocks decode; events are well-formed and ordered" { try testing.expect(commit_payloads > 0); } +test "fixture: block checksum corruption is rejected" { + const bytes = (try loadFixture(testing.allocator)) orelse return error.SkipZigTest; + defer testing.allocator.free(bytes); + + var sealed = try segment.Sealed.parse(testing.allocator, bytes); + const first = sealed.block_index[0]; + sealed.deinit(testing.allocator); + + // The JSS checksum authenticates header + footer metadata. The zstd frame + // checksum independently authenticates the compressed block body, so this + // mutation deliberately leaves Sealed.parse valid and must fail only when + // the block is decompressed. + const checksum_byte = first.offset + 8 + first.compressed_size - 1; + bytes[checksum_byte] ^= 0x01; + + var corrupted = try segment.Sealed.parse(testing.allocator, bytes); + defer corrupted.deinit(testing.allocator); + try testing.expectError(error.DecompressFailed, corrupted.readBlock(testing.allocator, 0)); +} + test "fixture: collection index matches upstream inspect output" { const bytes = (try loadFixture(testing.allocator)) orelse return error.SkipZigTest; defer testing.allocator.free(bytes); diff --git a/src/internal/segment_writer.zig b/src/internal/segment_writer.zig index 3530948..123b42e 100644 --- a/src/internal/segment_writer.zig +++ b/src/internal/segment_writer.zig @@ -9,7 +9,7 @@ //! upstream's walkActiveFrames, builds the footer (block index, gloom DID //! blooms, collection index with first-seen ids + $-sentinels), and //! finalizes the header with the file xxh3. per-block zstd frame CRCs are -//! written (libzstd) even though our reader can't verify them yet. +//! written and verified through the vendored libzstd implementation. const std = @import("std"); const footer_mod = @import("segment_footer.zig"); diff --git a/src/internal/zstd.zig b/src/internal/zstd.zig index fccfc08..722b03e 100644 --- a/src/internal/zstd.zig +++ b/src/internal/zstd.zig @@ -1,10 +1,9 @@ -//! minimal zstd bindings — compression only +//! minimal zstd bindings //! //! backed by the vendored facebook/zstd source (pinned by url+hash in -//! build.zig.zon, compiled in build.zig — no system library). decompression -//! stays on std.compress.zstd; the writer needs a compressor (absent from -//! zig std) producing frames with content checksums, matching upstream's -//! encoder settings (SpeedDefault ≈ level 3, WithEncoderCRC). +//! build.zig.zon, compiled in build.zig — no system library). Both directions +//! use libzstd: the writer needs a compressor (absent from Zig std), and the +//! reader needs content-checksum verification matching upstream's decoder. const std = @import("std"); @@ -14,6 +13,7 @@ const c = struct { extern fn ZSTD_CCtx_setParameter(cctx: ?*anyopaque, param: c_int, value: c_int) usize; extern fn ZSTD_compress2(cctx: ?*anyopaque, dst: [*]u8, dst_cap: usize, src: [*]const u8, src_len: usize) usize; extern fn ZSTD_compressBound(src_len: usize) usize; + extern fn ZSTD_decompress(dst: [*]u8, dst_cap: usize, src: [*]const u8, src_len: usize) usize; extern fn ZSTD_isError(code: usize) c_uint; // ZSTD_cParameter values (zstd.h, stable API) @@ -41,6 +41,17 @@ pub fn compress(allocator: std.mem.Allocator, src: []const u8) ![]u8 { return allocator.realloc(dst, written); } +/// Decompress one frame into its declared exact-size output. libzstd verifies +/// the frame content checksum when present; all JSS writers enable it. +pub fn decompressExact(allocator: std.mem.Allocator, frame: []const u8, expected_len: usize) ![]u8 { + const dst = try allocator.alloc(u8, expected_len); + errdefer allocator.free(dst); + const written = c.ZSTD_decompress(dst.ptr, dst.len, frame.ptr, frame.len); + if (c.ZSTD_isError(written) != 0 or written != expected_len) + return error.DecompressFailed; + return dst; +} + // === tests === test "round-trip through std decompress" { @@ -57,6 +68,22 @@ test "round-trip through std decompress" { try std.testing.expectEqualStrings(src, out.written()); } +test "libzstd decompress verifies the frame content checksum" { + const src = "checksum-protected block" ** 100; + const frame = try compress(std.testing.allocator, src); + defer std.testing.allocator.free(frame); + + const decoded = try decompressExact(std.testing.allocator, frame, src.len); + defer std.testing.allocator.free(decoded); + try std.testing.expectEqualStrings(src, decoded); + + frame[frame.len - 1] ^= 0x01; + try std.testing.expectError( + error.DecompressFailed, + decompressExact(std.testing.allocator, frame, src.len), + ); +} + // === dictionary compression (the /subscribe wire's v1 scheme) === const cdict = struct { -- 2.51.2