From 48ce9b3ab39ce042d49c4fe6c6690e6ff76e5917 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Tue, 8 Sep 2026 21:12:27 -0500 Subject: [PATCH] fix: decode live zstd frames with omitted content size --- docs/compression-review.md | 21 ++++++++--- src/zstd.zig | 72 +++++++++++++++++++++++++++++++++++++- 2 files changed, 88 insertions(+), 5 deletions(-) diff --git a/docs/compression-review.md b/docs/compression-review.md index f91c1a2..59d98ad 100644 --- a/docs/compression-review.md +++ b/docs/compression-review.md @@ -23,10 +23,23 @@ jetstream-compression. The 20-pair quiet baseline uses a synthetic corpus and must not be presented as a production compression ratio. No allocation reuse optimization has been made. -The existing decoder still requires a declared frame content size. This -review verifies default negotiation and dictionary recovery, not support for -every valid zstd frame encoding. Unknown-content-size frames remain a separate -decoder conformance item. +## Omitted live frame sizes, 2026-09-08 + +The live dictionary decoder now also accepts frames without declared content +size. It uses libzstd's decompression bound, capped by the existing read limit, +and returns only the actual decoded bytes. A conservative bound larger than +the limit does not itself reject a frame: actual output must fit. Declared-size +frames retain their existing fast path; archive decoding is unchanged. + +Two regressions failed first with UnknownFrameSize. Tests cover exact-limit +output, excess output, bad checksums, truncation, and decoder reuse after errors. +A third fixture was generated with klauspost/compress v1.19.2's streaming +writer (Write, Flush, Close) and independently decoded with upstream's +DecodeAll configuration. Its 8 MiB window produces only 448 bytes; the Zig +client accepts it at a 448-byte limit and rejects it at 447 bytes. +All 60 tests and the examples build pass. No consumer API change is required. +This closes the omitted-size live-frame gap, not a claim to cover every zstd +encoding or resource-policy difference. ## Production verification, 2026-09-08 diff --git a/src/zstd.zig b/src/zstd.zig index 5593574..17dd4fd 100644 --- a/src/zstd.zig +++ b/src/zstd.zig @@ -14,6 +14,9 @@ const c = struct { extern fn ZSTD_getFrameContentSize(src: [*]const u8, src_size: usize) c_ulonglong; extern fn ZSTD_decompress(dst: [*]u8, dst_cap: usize, src: [*]const u8, src_len: usize) usize; extern fn ZSTD_isError(code: usize) c_uint; + extern fn ZSTD_decompressBound(src: [*]const u8, src_size: usize) c_ulonglong; + extern fn ZSTD_getErrorCode(code: usize) c_uint; + const error_dst_size_too_small: c_uint = 70; // ZSTD_error_dstSize_tooSmall, zstd_errors.h extern fn ZSTD_createDCtx() ?*anyopaque; extern fn ZSTD_freeDCtx(dctx: ?*anyopaque) usize; extern fn ZSTD_createDDict(dict: [*]const u8, dict_size: usize) ?*anyopaque; @@ -91,7 +94,23 @@ pub const DictDecoder = struct { pub fn decompressAlloc(self: *DictDecoder, allocator: Allocator, frame: []const u8) DecompressError![]u8 { if (frame.len == 0) return error.DecompressFailed; const size = c.ZSTD_getFrameContentSize(frame.ptr, frame.len); - if (size == c.contentsize_unknown) return error.UnknownFrameSize; + if (size == c.contentsize_unknown) { + const bound = c.ZSTD_decompressBound(frame.ptr, frame.len); + if (bound == c.contentsize_error) return error.DecompressFailed; + // The bound may exceed actual output; cap the allocation, not the + // frame's window. libzstd checks whether decoded bytes fit. + const capacity: usize = @intCast(@min(bound, self.max_out)); + const dst = try allocator.alloc(u8, capacity); + errdefer allocator.free(dst); + const written = c.ZSTD_decompress_usingDDict(self.dctx, dst.ptr, dst.len, frame.ptr, frame.len, self.ddict); + if (c.ZSTD_isError(written) != 0) { + if (c.ZSTD_getErrorCode(written) == c.error_dst_size_too_small) return error.BlockOversize; + return error.DecompressFailed; + } + // Return an allocation with the actual length so callers can free + // it normally, including allocators that cannot shrink in place. + return try allocator.realloc(dst, written); + } if (size == c.contentsize_error) return error.DecompressFailed; if (size > self.max_out) return error.BlockOversize; const dst = try allocator.alloc(u8, @intCast(size)); @@ -136,3 +155,54 @@ test "decompressAlloc rejects garbage" { try testing.expectError(error.DecompressFailed, decompressAlloc(testing.allocator, "not a zstd frame")); try testing.expectError(error.DecompressFailed, decompressAlloc(testing.allocator, "")); } + +// A complete dictionary frame with a window descriptor but no content size. +// Its 1 KiB window is deliberately larger than the four-byte output limit. +fn unknownSizeFrame(buf: *[21]u8) []u8 { + buf[0..6].* = .{ 0x28, 0xb5, 0x2f, 0xfd, 0x07, 0x00 }; + std.mem.writeInt(u32, buf[6..10], dictId(@embedFile("testdata/live.dict")), .little); + buf[10..13].* = .{ 0x21, 0, 0 }; + buf[13..17].* = "zat!".*; + std.mem.writeInt(u32, buf[17..21], @truncate(std.hash.XxHash64.hash(0, "zat!")), .little); + return buf; +} + +test "live dictionary accepts omitted size with output exactly at limit" { + var decoder = try DictDecoder.init(@embedFile("testdata/live.dict"), 4); + defer decoder.deinit(); + var buf: [21]u8 = undefined; + const out = try decoder.decompressAlloc(testing.allocator, unknownSizeFrame(&buf)); + defer testing.allocator.free(out); + try testing.expectEqualStrings("zat!", out); +} + +test "live dictionary limits unknown output and preserves checksum validation" { + var decoder = try DictDecoder.init(@embedFile("testdata/live.dict"), 3); + defer decoder.deinit(); + var buf: [21]u8 = undefined; + const frame = unknownSizeFrame(&buf); + try testing.expectError(error.BlockOversize, decoder.decompressAlloc(testing.allocator, frame)); + decoder.max_out = 4; + frame[20] ^= 1; + try testing.expectError(error.DecompressFailed, decoder.decompressAlloc(testing.allocator, frame)); + frame[20] ^= 1; + try testing.expectError(error.DecompressFailed, decoder.decompressAlloc(testing.allocator, frame[0..20])); + const out = try decoder.decompressAlloc(testing.allocator, frame); + defer testing.allocator.free(out); + try testing.expectEqualStrings("zat!", out); +} + +test "live dictionary decodes upstream Go streaming frame without declared size" { + // klauspost/compress v1.19.2: encoder dictionary + Write, Flush, Close. + // Verified with upstream's DecodeAll options; the window is 8 MiB, + // but actual output is only 448 bytes. + const frame = [_]u8{ 0x28, 0xb5, 0x2f, 0xfd, 0x07, 0x68, 0xcb, 0x27, 0x35, 0x01, 0x0c, 0x01, 0x00, 0x34, 0x01, 0x7b, 0x70, 0x6f, 0x73, 0x74, 0x22, 0x68, 0x65, 0x6c, 0x6c, 0x6f, 0x20, 0x77, 0x6f, 0x72, 0x6c, 0x64, 0x22, 0x7d, 0x03, 0x00, 0x85, 0x5b, 0xa9, 0x3f, 0x45, 0x34, 0xdd, 0x60, 0x16, 0x1b, 0x01, 0x00, 0x00, 0x01, 0xa7, 0xb1, 0x7d }; + try testing.expectEqual(c.contentsize_unknown, c.ZSTD_getFrameContentSize(&frame, frame.len)); + var decoder = try DictDecoder.init(@embedFile("testdata/live.dict"), 448); + defer decoder.deinit(); + const out = try decoder.decompressAlloc(testing.allocator, &frame); + defer testing.allocator.free(out); + try testing.expectEqualStrings("{\"collection\":\"app.bsky.feed.post\",\"text\":\"hello world\"}" ** 8, out); + decoder.max_out = 447; + try testing.expectError(error.BlockOversize, decoder.decompressAlloc(testing.allocator, &frame)); +} -- 2.51.2