diff --git a/build.zig b/build.zig index de5bbb0..dcc832c 100644 --- a/build.zig +++ b/build.zig @@ -1,13 +1,65 @@ const std = @import("std"); -fn linkZstd(mod: *std.Build.Module, b: *std.Build) void { - // homebrew installs outside the default search path on macos - if (@import("builtin").os.tag == .macos) { - mod.addLibraryPath(.{ .cwd_relative = "/opt/homebrew/lib" }); - mod.addIncludePath(.{ .cwd_relative = "/opt/homebrew/include" }); - } - mod.linkSystemLibrary("zstd", .{}); - _ = b; +// vendored C dependencies, pinned by url+hash in build.zig.zon — no system +// libraries, no host-specific search paths; cross-compiles with -Dtarget. +fn linkVendoredC(mod: *std.Build.Module, b: *std.Build, target: std.Build.ResolvedTarget, optimize: std.builtin.OptimizeMode) void { + const zstd_dep = b.dependency("zstd", .{}); + const xxhash_dep = b.dependency("xxhash", .{}); + + const clib_mod = b.createModule(.{ + .target = target, + .optimize = optimize, + .link_libc = true, + }); + const zstd_flags: []const []const u8 = &.{ + // upstream libzstd's own defaults, minus assembly (portable across + // -Dtarget) and multithreading (we compress on our own threads) + "-DZSTD_DISABLE_ASM", + "-DZSTD_MULTITHREAD=0", + // zstd namespaces its bundled xxhash the same way its Makefile does, + // so it can't collide with the standalone xxhash below + "-DXXH_NAMESPACE=ZSTD_", + }; + clib_mod.addCSourceFiles(.{ + .root = zstd_dep.path("lib"), + .files = &.{ + "common/debug.c", + "common/entropy_common.c", + "common/error_private.c", + "common/fse_decompress.c", + "common/pool.c", + "common/threading.c", + "common/xxhash.c", + "common/zstd_common.c", + "compress/fse_compress.c", + "compress/hist.c", + "compress/huf_compress.c", + "compress/zstd_compress_literals.c", + "compress/zstd_compress_sequences.c", + "compress/zstd_compress_superblock.c", + "compress/zstd_compress.c", + "compress/zstd_double_fast.c", + "compress/zstd_fast.c", + "compress/zstd_lazy.c", + "compress/zstd_ldm.c", + "compress/zstd_opt.c", + "compress/zstd_preSplit.c", + "compress/zstdmt_compress.c", + "decompress/huf_decompress.c", + "decompress/zstd_ddict.c", + "decompress/zstd_decompress_block.c", + "decompress/zstd_decompress.c", + }, + .flags = zstd_flags, + }); + clib_mod.addCSourceFiles(.{ + .root = xxhash_dep.path("."), + .files = &.{"xxhash.c"}, + .flags = &.{}, + }); + + const clib = b.addLibrary(.{ .name = "stream-vendored-c", .root_module = clib_mod }); + mod.linkLibrary(clib); } pub fn build(b: *std.Build) void { @@ -49,7 +101,7 @@ pub fn build(b: *std.Build) void { }); exe_mod.addImport("build_options", build_options.createModule()); exe_mod.link_libc = true; - linkZstd(exe_mod, b); + linkVendoredC(exe_mod, b, target, optimize); const exe = b.addExecutable(.{ .name = "stream", .root_module = exe_mod, @@ -67,7 +119,7 @@ pub fn build(b: *std.Build) void { .optimize = optimize, }); sample_mod.link_libc = true; - linkZstd(sample_mod, b); + linkVendoredC(sample_mod, b, target, optimize); const write_sample = b.addExecutable(.{ .name = "write-sample-segment", .root_module = sample_mod }); const run_sample = b.addRunArtifact(write_sample); if (b.args) |args| run_sample.addArgs(args); @@ -82,7 +134,7 @@ pub fn build(b: *std.Build) void { }); test_mod.addImport("build_options", build_options.createModule()); test_mod.link_libc = true; - linkZstd(test_mod, b); + linkVendoredC(test_mod, b, target, optimize); const t = b.addTest(.{ .root_module = test_mod, }); diff --git a/build.zig.zon b/build.zig.zon index d1da7e8..88d9d31 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -12,6 +12,14 @@ .url = "https://tangled.org/zzstoatzz.io/websocket.zig/archive/v0.1.9.tar.gz", .hash = "websocket-0.1.9-ZPISdYtnBABW10-3-CAGGR7fd6q5t8vlOj6Sxd-6Qu-C", }, + .zstd = .{ + .url = "https://github.com/facebook/zstd/releases/download/v1.5.7/zstd-1.5.7.tar.gz", + .hash = "N-V-__8AAPZ7fwBg4JoCzM_0o2A8wxH2hsUUeiU1iuZv53L5", + }, + .xxhash = .{ + .url = "https://github.com/Cyan4973/xxHash/archive/refs/tags/v0.8.3.tar.gz", + .hash = "N-V-__8AABgCUwB4fhP90zPC1u1U84-BEirnJ0dxOPi5ZM7Z", + }, }, .paths = .{ "build.zig", diff --git a/src/internal/xxhash128.zig b/src/internal/xxhash128.zig new file mode 100644 index 0000000..4bb6ecb --- /dev/null +++ b/src/internal/xxhash128.zig @@ -0,0 +1,41 @@ +//! xxh3-128 via the vendored reference xxHash (Cyan4973/xxHash v0.8.3, +//! pinned in build.zig.zon). needed for gloom bloom filters, whose wire +//! format hashes DIDs with xxh3-128 — a hand-ported hash would risk silent +//! false negatives, so we bind the canonical implementation and verify it +//! against independently generated vectors. + +const std = @import("std"); + +const XXH128_hash_t = extern struct { + low64: u64, + high64: u64, +}; + +extern fn XXH3_128bits(data: [*]const u8, len: usize) XXH128_hash_t; + +pub const Hash128 = struct { + lo: u64, + hi: u64, +}; + +pub fn hash(data: []const u8) Hash128 { + const h = XXH3_128bits(data.ptr, data.len); + return .{ .lo = h.low64, .hi = h.high64 }; +} + +// === tests === +// vectors generated with python xxhash (independent of the C library) + +test "xxh3-128 reference vectors" { + const empty = hash(""); + try std.testing.expectEqual(@as(u64, 0x99aa06d3014798d8), empty.hi); + try std.testing.expectEqual(@as(u64, 0x6001c324468d497f), empty.lo); + + const did = hash("did:plc:streamsampleaccount"); + try std.testing.expectEqual(@as(u64, 0xb60ab1dc6bcdfbc4), did.hi); + try std.testing.expectEqual(@as(u64, 0xb4d6e89485049a70), did.lo); + + const fox = hash("the quick brown fox"); + try std.testing.expectEqual(@as(u64, 0x2830e9c6299d0ff9), fox.hi); + try std.testing.expectEqual(@as(u64, 0xab2ec4cf7e8d85d2), fox.lo); +} diff --git a/src/internal/zstd.zig b/src/internal/zstd.zig index d04ef96..864ec5e 100644 --- a/src/internal/zstd.zig +++ b/src/internal/zstd.zig @@ -1,8 +1,10 @@ -//! minimal libzstd bindings — compression only +//! minimal zstd bindings — compression only //! -//! decompression stays on std.compress.zstd; the writer needs a compressor -//! (absent from zig std) producing frames with content checksums, matching -//! upstream's encoder settings (SpeedDefault ≈ level 3, WithEncoderCRC). +//! backed by the vendored facebook/zstd source (pinned by url+hash in +//! build.zig.zon, compiled in build.zig — no system library). decompression +//! stays on std.compress.zstd; the writer needs a compressor (absent from +//! zig std) producing frames with content checksums, matching upstream's +//! encoder settings (SpeedDefault ≈ level 3, WithEncoderCRC). const std = @import("std"); diff --git a/src/tests.zig b/src/tests.zig index 35b4557..6c302ca 100644 --- a/src/tests.zig +++ b/src/tests.zig @@ -5,6 +5,7 @@ comptime { _ = @import("internal/metrics.zig"); _ = @import("internal/segment.zig"); _ = @import("internal/zstd.zig"); + _ = @import("internal/xxhash128.zig"); _ = @import("internal/segment_writer.zig"); _ = @import("internal/segment_fixture_test.zig"); _ = @import("internal/verify.zig");