diff --git a/build.zig b/build.zig index 3711111..e66d15a 100644 --- a/build.zig +++ b/build.zig @@ -65,35 +65,19 @@ pub fn build(b: *std.Build) void { run_step.dependOn(&run_relay.step); // tests + // tests — a single module rooted at src/ so every internal/ import resolves const test_step = b.step("test", "run unit tests"); - const test_files = .{ - "src/lru.zig", - "src/api.zig", - "src/broadcaster.zig", - "src/validator.zig", - "src/subscriber.zig", - "src/event_log.zig", - "src/slurper.zig", - "src/collection_index.zig", - "src/backfill.zig", - "src/thread_pool.zig", - "src/frame_worker.zig", - "src/util/util.zig", - "src/atproto/main.zig", - }; - inline for (test_files) |file| { - const test_mod = b.createModule(.{ - .root_source_file = b.path(file), - .target = target, - .optimize = optimize, - .imports = imports, - }); - test_mod.addImport("build_options", build_options.createModule()); - test_mod.link_libc = true; - test_mod.link_libcpp = true; - const t = b.addTest(.{ - .root_module = test_mod, - }); - test_step.dependOn(&b.addRunArtifact(t).step); - } + const test_mod = b.createModule(.{ + .root_source_file = b.path("src/tests.zig"), + .target = target, + .optimize = optimize, + .imports = imports, + }); + test_mod.addImport("build_options", build_options.createModule()); + test_mod.link_libc = true; + test_mod.link_libcpp = true; + const t = b.addTest(.{ + .root_module = test_mod, + }); + test_step.dependOn(&b.addRunArtifact(t).step); } diff --git a/build.zig.zon b/build.zig.zon index 3dcf24a..441830b 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = .zlay, - .version = "0.0.1", + .version = "0.0.2", .fingerprint = 0x31343ede133f3e58, .minimum_zig_version = "0.16.0", .dependencies = .{ diff --git a/docs/backfill.md b/docs/backfill.md index 28291f2..c2de619 100644 --- a/docs/backfill.md +++ b/docs/backfill.md @@ -100,6 +100,6 @@ safe to trigger again after completion — existing progress rows are preserved, ## source code -- `src/backfill.zig` — Backfiller struct with all backfill logic -- `src/event_log.zig` — backfill_progress table creation (in `init()`) +- `src/internal/collection_index/backfill.zig` — Backfiller struct with all backfill logic +- `src/internal/event_log.zig` — backfill_progress table creation (in `init()`) - `src/main.zig` — admin route handlers diff --git a/docs/design.md b/docs/design.md index 37797fe..2e32a69 100644 --- a/docs/design.md +++ b/docs/design.md @@ -5,22 +5,35 @@ signatures, and rebroadcasts to downstream consumers over WebSocket. ## code organization -`src/` root holds the domain modules (one per pipeline stage: `slurper`, -`subscriber`, `frame_worker`, `validator`, `broadcaster`, `event_log`, …). -shared helpers live in two packages so they don't get re-defined per module: - -- `src/util/` — extensions to the standard library: things that could have - been in `std` but aren't (`env`, `time`). flat namespace (`util.timestamp`, - not `util.time.timestamp`); no domain knowledge. add here when a helper is - generic and would make sense in any zig project. -- `src/atproto/` — AT Protocol / relay domain logic shared across modules +`src/` holds exactly two things: `main.zig` (the composition root / entrypoint) +and `internal/` (everything else). a top-level file is a declaration that it is +core to the system; only `main.zig` qualifies. this mirrors the sibling `zat` +repo's `root.zig` + `internal/` convention. + +inside `internal/`, most modules are flat files — each is one cohesive concern +(`slurper`, `subscriber`, `frame_worker`, `host_ops`, `validator`, `event_log`, +`broadcaster`). a subfolder is introduced **only when a cluster earns it with +several cohesive files**, never to make an abstract pipeline-stage statement: + +- `internal/collection_index/` — the secondary index feature: the RocksDB + `index` plus its `backfill`, `cleaner`, and `resync` maintenance. +- `internal/api/` — XRPC + admin HTTP handlers (`router`, `http`, `xrpc`, + `admin`, `main` facade). +- `internal/util/` — extensions to the standard library: things that could + have been in `std` but aren't (`env`, `time`, `lru`, `ring_buffer`, + `thread_pool`). flat namespace (`util.timestamp`); no domain knowledge. +- `internal/atproto/` — AT Protocol / relay domain logic shared across modules (`hostname` parsing + validation, `host_check` describeServer/SSRF/relay-loop - detection). these encode relay *policy* (mirroring indigo), so they are not - `util`. each package's `main.zig` doc-comment defines its scope — that scope - is what decides where a new helper goes. + detection). encodes relay *policy*, so it is not `util`. -if a helper is being copy-pasted into a third module, that's the signal to -move it into one of these two packages rather than re-declaring it. +the rule: a folder must justify itself with multiple cohesive files; otherwise +it's a flat file in `internal/`. a single-file folder is the smell we avoid. +if a helper gets copy-pasted into a third module, move it into `util/` or +`atproto/` rather than re-declaring it. + +tests are aggregated through `src/tests.zig` (rooted at `src/` so every +`internal/` import resolves) — add an `@import` line there for a new file with +`test` blocks. ## data flow diff --git a/src/api/admin.zig b/src/internal/api/admin.zig similarity index 99% rename from src/api/admin.zig rename to src/internal/api/admin.zig index fba9397..5918222 100644 --- a/src/api/admin.zig +++ b/src/internal/api/admin.zig @@ -12,9 +12,9 @@ const h = @import("http.zig"); const router = @import("router.zig"); const websocket = @import("websocket"); const event_log_mod = @import("../event_log.zig"); -const backfill_mod = @import("../backfill.zig"); -const cleaner_mod = @import("../cleaner.zig"); -const resync_mod = @import("../resync.zig"); +const backfill_mod = @import("../collection_index/backfill.zig"); +const cleaner_mod = @import("../collection_index/cleaner.zig"); +const resync_mod = @import("../collection_index/resync.zig"); const util = @import("../util/util.zig"); const log = std.log.scoped(.relay); diff --git a/src/api/http.zig b/src/internal/api/http.zig similarity index 100% rename from src/api/http.zig rename to src/internal/api/http.zig diff --git a/src/api.zig b/src/internal/api/main.zig similarity index 72% rename from src/api.zig rename to src/internal/api/main.zig index 27ef1ee..a1ad130 100644 --- a/src/api.zig +++ b/src/internal/api/main.zig @@ -6,15 +6,15 @@ //! admin.zig — admin endpoint handlers //! http.zig — response helpers, query string parsing -const router = @import("api/router.zig"); +const router = @import("router.zig"); pub const HttpContext = router.HttpContext; pub const handleHttpRequest = router.handleHttpRequest; test { @import("std").testing.refAllDecls(@This()); - _ = @import("api/http.zig"); - _ = @import("api/xrpc.zig"); - _ = @import("api/admin.zig"); - _ = @import("api/router.zig"); + _ = @import("http.zig"); + _ = @import("xrpc.zig"); + _ = @import("admin.zig"); + _ = @import("router.zig"); } diff --git a/src/api/router.zig b/src/internal/api/router.zig similarity index 96% rename from src/api/router.zig rename to src/internal/api/router.zig index a4f84e4..2a00aba 100644 --- a/src/api/router.zig +++ b/src/internal/api/router.zig @@ -10,10 +10,10 @@ const broadcaster = @import("../broadcaster.zig"); const validator_mod = @import("../validator.zig"); const slurper_mod = @import("../slurper.zig"); const event_log_mod = @import("../event_log.zig"); -const collection_index_mod = @import("../collection_index.zig"); -const backfill_mod = @import("../backfill.zig"); -const cleaner_mod = @import("../cleaner.zig"); -const resync_mod = @import("../resync.zig"); +const collection_index_mod = @import("../collection_index/index.zig"); +const backfill_mod = @import("../collection_index/backfill.zig"); +const cleaner_mod = @import("../collection_index/cleaner.zig"); +const resync_mod = @import("../collection_index/resync.zig"); const host_ops_mod = @import("../host_ops.zig"); const h = @import("http.zig"); const xrpc = @import("xrpc.zig"); diff --git a/src/api/xrpc.zig b/src/internal/api/xrpc.zig similarity index 99% rename from src/api/xrpc.zig rename to src/internal/api/xrpc.zig index 579232e..3c8f3a4 100644 --- a/src/api/xrpc.zig +++ b/src/internal/api/xrpc.zig @@ -13,7 +13,7 @@ const pg = @import("pg"); const h = @import("http.zig"); const router = @import("router.zig"); const event_log_mod = @import("../event_log.zig"); -const collection_index_mod = @import("../collection_index.zig"); +const collection_index_mod = @import("../collection_index/index.zig"); const slurper_mod = @import("../slurper.zig"); const Allocator = std.mem.Allocator; diff --git a/src/atproto/host_check.zig b/src/internal/atproto/host_check.zig similarity index 100% rename from src/atproto/host_check.zig rename to src/internal/atproto/host_check.zig diff --git a/src/atproto/hostname.zig b/src/internal/atproto/hostname.zig similarity index 100% rename from src/atproto/hostname.zig rename to src/internal/atproto/hostname.zig diff --git a/src/atproto/main.zig b/src/internal/atproto/main.zig similarity index 100% rename from src/atproto/main.zig rename to src/internal/atproto/main.zig diff --git a/src/broadcaster.zig b/src/internal/broadcaster.zig similarity index 99% rename from src/broadcaster.zig rename to src/internal/broadcaster.zig index e684466..209948a 100644 --- a/src/broadcaster.zig +++ b/src/internal/broadcaster.zig @@ -11,7 +11,7 @@ const std = @import("std"); const Io = std.Io; const websocket = @import("websocket"); const zat = @import("zat"); -const ring_buffer = @import("ring_buffer.zig"); +const ring_buffer = @import("util/ring_buffer.zig"); const event_log_mod = @import("event_log.zig"); const builtin = @import("builtin"); diff --git a/src/backfill.zig b/src/internal/collection_index/backfill.zig similarity index 99% rename from src/backfill.zig rename to src/internal/collection_index/backfill.zig index 5da2f44..b5189ec 100644 --- a/src/backfill.zig +++ b/src/internal/collection_index/backfill.zig @@ -11,8 +11,8 @@ const std = @import("std"); const Io = std.Io; const http = std.http; const pg = @import("pg"); -const collection_index_mod = @import("collection_index.zig"); -const event_log_mod = @import("event_log.zig"); +const collection_index_mod = @import("index.zig"); +const event_log_mod = @import("../event_log.zig"); const Allocator = std.mem.Allocator; const log = std.log.scoped(.backfill); diff --git a/src/cleaner.zig b/src/internal/collection_index/cleaner.zig similarity index 97% rename from src/cleaner.zig rename to src/internal/collection_index/cleaner.zig index 75c0ce6..8d547b1 100644 --- a/src/cleaner.zig +++ b/src/internal/collection_index/cleaner.zig @@ -7,8 +7,8 @@ const std = @import("std"); const Io = std.Io; const pg = @import("pg"); -const collection_index_mod = @import("collection_index.zig"); -const event_log_mod = @import("event_log.zig"); +const collection_index_mod = @import("index.zig"); +const event_log_mod = @import("../event_log.zig"); const Allocator = std.mem.Allocator; const log = std.log.scoped(.cleaner); diff --git a/src/collection_index.zig b/src/internal/collection_index/index.zig similarity index 100% rename from src/collection_index.zig rename to src/internal/collection_index/index.zig diff --git a/src/resync.zig b/src/internal/collection_index/resync.zig similarity index 99% rename from src/resync.zig rename to src/internal/collection_index/resync.zig index df8fc67..08b6099 100644 --- a/src/resync.zig +++ b/src/internal/collection_index/resync.zig @@ -11,7 +11,7 @@ const std = @import("std"); const Io = std.Io; const http = std.http; -const collection_index_mod = @import("collection_index.zig"); +const collection_index_mod = @import("index.zig"); const Allocator = std.mem.Allocator; const log = std.log.scoped(.resync); diff --git a/src/event_log.zig b/src/internal/event_log.zig similarity index 99% rename from src/event_log.zig rename to src/internal/event_log.zig index 66362e2..8649996 100644 --- a/src/event_log.zig +++ b/src/internal/event_log.zig @@ -13,7 +13,7 @@ const std = @import("std"); const Io = std.Io; const pg = @import("pg"); -const lru = @import("lru.zig"); +const lru = @import("util/lru.zig"); const util = @import("util/util.zig"); const getenv = util.getenv; diff --git a/src/frame_worker.zig b/src/internal/frame_worker.zig similarity index 98% rename from src/frame_worker.zig rename to src/internal/frame_worker.zig index 78a5a0c..68c5628 100644 --- a/src/frame_worker.zig +++ b/src/internal/frame_worker.zig @@ -13,9 +13,9 @@ const zat = @import("zat"); const broadcaster = @import("broadcaster.zig"); const validator_mod = @import("validator.zig"); const event_log_mod = @import("event_log.zig"); -const collection_index_mod = @import("collection_index.zig"); -const resync_mod = @import("resync.zig"); -const thread_pool = @import("thread_pool.zig"); +const collection_index_mod = @import("collection_index/index.zig"); +const resync_mod = @import("collection_index/resync.zig"); +const thread_pool = @import("util/thread_pool.zig"); const util = @import("util/util.zig"); const Allocator = std.mem.Allocator; diff --git a/src/host_ops.zig b/src/internal/host_ops.zig similarity index 100% rename from src/host_ops.zig rename to src/internal/host_ops.zig diff --git a/src/slurper.zig b/src/internal/slurper.zig similarity index 99% rename from src/slurper.zig rename to src/internal/slurper.zig index 032a815..1912aa5 100644 --- a/src/slurper.zig +++ b/src/internal/slurper.zig @@ -17,8 +17,8 @@ const broadcaster = @import("broadcaster.zig"); const validator_mod = @import("validator.zig"); const event_log_mod = @import("event_log.zig"); const subscriber_mod = @import("subscriber.zig"); -const collection_index_mod = @import("collection_index.zig"); -const resync_mod = @import("resync.zig"); +const collection_index_mod = @import("collection_index/index.zig"); +const resync_mod = @import("collection_index/resync.zig"); const frame_worker_mod = @import("frame_worker.zig"); const host_ops_mod = @import("host_ops.zig"); const atproto = @import("atproto/main.zig"); @@ -118,7 +118,7 @@ pub const Slurper = struct { self.frame_pool = try frame_worker_mod.FramePool.init(self.allocator, .{ .num_workers = self.options.frame_workers, .queue_capacity = self.options.frame_queue_capacity, - .stack_size = @import("main.zig").default_stack_size, + .stack_size = @import("../main.zig").default_stack_size, }, self.pool_io); log.info("frame pool started: {d} workers, queue capacity {d}", .{ self.options.frame_workers, self.options.frame_queue_capacity }); diff --git a/src/subscriber.zig b/src/internal/subscriber.zig similarity index 99% rename from src/subscriber.zig rename to src/internal/subscriber.zig index 33911ed..eef3d16 100644 --- a/src/subscriber.zig +++ b/src/internal/subscriber.zig @@ -12,8 +12,8 @@ const zat = @import("zat"); const broadcaster = @import("broadcaster.zig"); const validator_mod = @import("validator.zig"); const event_log_mod = @import("event_log.zig"); -const collection_index_mod = @import("collection_index.zig"); -const resync_mod = @import("resync.zig"); +const collection_index_mod = @import("collection_index/index.zig"); +const resync_mod = @import("collection_index/resync.zig"); const frame_worker_mod = @import("frame_worker.zig"); const host_ops_mod = @import("host_ops.zig"); const util = @import("util/util.zig"); diff --git a/src/util/env.zig b/src/internal/util/env.zig similarity index 100% rename from src/util/env.zig rename to src/internal/util/env.zig diff --git a/src/lru.zig b/src/internal/util/lru.zig similarity index 100% rename from src/lru.zig rename to src/internal/util/lru.zig diff --git a/src/ring_buffer.zig b/src/internal/util/ring_buffer.zig similarity index 100% rename from src/ring_buffer.zig rename to src/internal/util/ring_buffer.zig diff --git a/src/thread_pool.zig b/src/internal/util/thread_pool.zig similarity index 100% rename from src/thread_pool.zig rename to src/internal/util/thread_pool.zig diff --git a/src/util/time.zig b/src/internal/util/time.zig similarity index 100% rename from src/util/time.zig rename to src/internal/util/time.zig diff --git a/src/util/util.zig b/src/internal/util/util.zig similarity index 100% rename from src/util/util.zig rename to src/internal/util/util.zig diff --git a/src/validator.zig b/src/internal/validator.zig similarity index 99% rename from src/validator.zig rename to src/internal/validator.zig index d4f108e..08a87a9 100644 --- a/src/validator.zig +++ b/src/internal/validator.zig @@ -10,7 +10,7 @@ const Io = std.Io; const zat = @import("zat"); const broadcaster = @import("broadcaster.zig"); const event_log_mod = @import("event_log.zig"); -const lru = @import("lru.zig"); +const lru = @import("util/lru.zig"); const util = @import("util/util.zig"); const atproto = @import("atproto/main.zig"); diff --git a/src/main.zig b/src/main.zig index 22e01b0..84454f8 100644 --- a/src/main.zig +++ b/src/main.zig @@ -27,17 +27,17 @@ const std = @import("std"); const Io = std.Io; const http = std.http; const websocket = @import("websocket"); -const broadcaster = @import("broadcaster.zig"); -const validator_mod = @import("validator.zig"); -const slurper_mod = @import("slurper.zig"); -const event_log_mod = @import("event_log.zig"); -const collection_index_mod = @import("collection_index.zig"); -const backfill_mod = @import("backfill.zig"); -const cleaner_mod = @import("cleaner.zig"); -const resync_mod = @import("resync.zig"); -const host_ops_mod = @import("host_ops.zig"); -const api = @import("api.zig"); -const util = @import("util/util.zig"); +const broadcaster = @import("internal/broadcaster.zig"); +const validator_mod = @import("internal/validator.zig"); +const slurper_mod = @import("internal/slurper.zig"); +const event_log_mod = @import("internal/event_log.zig"); +const collection_index_mod = @import("internal/collection_index/index.zig"); +const backfill_mod = @import("internal/collection_index/backfill.zig"); +const cleaner_mod = @import("internal/collection_index/cleaner.zig"); +const resync_mod = @import("internal/collection_index/resync.zig"); +const host_ops_mod = @import("internal/host_ops.zig"); +const api = @import("internal/api/main.zig"); +const util = @import("internal/util/util.zig"); const build_options = @import("build_options"); const getenv = util.getenv; diff --git a/src/tests.zig b/src/tests.zig new file mode 100644 index 0000000..4101b70 --- /dev/null +++ b/src/tests.zig @@ -0,0 +1,21 @@ +//! test aggregator — rooted at src/ so every module's imports resolve. +//! one test module pulls in each file's `test` blocks; referencing a file +//! with `_ = @import(...)` forces its analysis and includes its tests. + +test { + _ = @import("internal/util/lru.zig"); + _ = @import("internal/util/ring_buffer.zig"); + _ = @import("internal/util/thread_pool.zig"); + _ = @import("internal/util/util.zig"); + _ = @import("internal/atproto/main.zig"); + _ = @import("internal/api/main.zig"); + _ = @import("internal/broadcaster.zig"); + _ = @import("internal/validator.zig"); + _ = @import("internal/subscriber.zig"); + _ = @import("internal/event_log.zig"); + _ = @import("internal/slurper.zig"); + _ = @import("internal/frame_worker.zig"); + _ = @import("internal/collection_index/index.zig"); + _ = @import("internal/collection_index/backfill.zig"); + _ = @import("main.zig"); +}