From 33162d4ac87b32bd67e82f15dc072ab9e87338e1 Mon Sep 17 00:00:00 2001 From: zzstoatzz Date: Thu, 11 Jun 2026 01:41:31 -0500 Subject: [PATCH] fix(recovery): validate seq chain in scanForLastSeq (v0.0.4) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit recovery hardening for the quintillion-seq incident: a torn tail left bytes that decoded to a structurally-valid header (good kind, plausible len, body present) but whose seq field was actually CBOR payload (0x6363a382...). the scan trusted that seq and every restart since resumed garbage+1, relocating the whole sequencer into a junk space. the writer assigns seqs densely (seq = cur_seq; cur_seq += 1) and a file begins at exactly its seq_start, so scanForLastSeq now also requires the first record's seq == seq_start and each subsequent == prev + 1. a record that breaks the chain is the torn-tail boundary even if kind+len look plausible — recovery stops there and resumes in the real space (or falls back to seq_start if the very first record is already garbage). regression test plants a structurally-valid record carrying the garbage CBOR seq and asserts the scan rejects it; a second case asserts a garbage-from- record-one file recovers nothing (→ resumeLog falls back to seq_start). does not repair the existing bogus on-disk space — that's a separate coordinated operator action once a restore target is chosen. Co-Authored-By: Claude Opus 4.8 --- build.zig.zon | 2 +- src/internal/event_log.zig | 76 +++++++++++++++++++++++++++++++++++--- 2 files changed, 72 insertions(+), 6 deletions(-) diff --git a/build.zig.zon b/build.zig.zon index 7c62b95..a8abdd6 100644 --- a/build.zig.zon +++ b/build.zig.zon @@ -1,6 +1,6 @@ .{ .name = .zlay, - .version = "0.0.3", + .version = "0.0.4", .fingerprint = 0x31343ede133f3e58, .minimum_zig_version = "0.16.0", .dependencies = .{ diff --git a/src/internal/event_log.zig b/src/internal/event_log.zig index 7a01e99..5814f30 100644 --- a/src/internal/event_log.zig +++ b/src/internal/event_log.zig @@ -1207,7 +1207,7 @@ pub const DiskPersist = struct { }; // scan for last seq (validates records, reports the last good offset) - const scan = scanForLastSeq(file, self.io) catch { + const scan = scanForLastSeq(file, self.io, seq_start) catch { file.close(self.io); try self.initLogFile(); return; @@ -1417,11 +1417,21 @@ const ScanResult = struct { }; /// scan a log file for the last valid sequence number. each record is validated -/// (known kind, plausible length, body fully present) before being accepted; the -/// scan stops at the first record it can't trust rather than following a garbage +/// (known kind, plausible length, body fully present, AND a seq that matches the +/// dense monotonic space the writer guarantees) before being accepted; the scan +/// stops at the first record it can't trust rather than following a garbage /// length and silently mis-recovering. this is what bounds a torn tail to the /// torn record itself — see docs/incident-2026-05-31-seq-rewind.md. -fn scanForLastSeq(file: Io.File, io: Io) !ScanResult { +/// +/// `seq_start` is the file's declared starting seq (from its name / log_file_refs +/// row). the writer assigns seqs densely (seq = cur_seq; cur_seq += 1) and a file +/// begins at exactly seq_start, so the first record must have seq == seq_start and +/// every subsequent record seq == prev + 1. a record whose seq breaks that chain +/// is a torn/garbage tail even if its kind+len happen to look plausible — this is +/// what stops `scanForLastSeq` from accepting CBOR payload bytes as a seq and +/// relocating the whole sequencer into a junk space (the 2026-06 quintillion-seq +/// incident). pass 0 to skip the seq-chain check (genuinely unknown start). +fn scanForLastSeq(file: Io.File, io: Io, seq_start: u64) !ScanResult { const file_size = (try file.stat(io)).size; var last_seq: ?u64 = null; @@ -1439,6 +1449,13 @@ fn scanForLastSeq(file: Io.File, io: Io) !ScanResult { const valid_kind = hdr.kind >= @intFromEnum(EvtKind.commit) and hdr.kind <= @intFromEnum(EvtKind.sync); if (!valid_kind or hdr.len > max_record_len or pos + header_size + hdr.len > file_size) break; + // seq must follow the dense monotonic chain: the first record == seq_start, + // each next == prev + 1. a break here is a torn/garbage tail. + if (seq_start > 0) { + const expected = if (last_seq) |ls| ls + 1 else seq_start; + if (hdr.seq != expected) break; + } + last_seq = hdr.seq; pos += header_size + hdr.len; } @@ -1544,11 +1561,60 @@ test "scanForLastSeq stops at a torn tail" { try file.writePositionalAll(io, &hbuf, pos); try file.writePositionalAll(io, "xy", pos + header_size); - const scan = try scanForLastSeq(file, io); + const scan = try scanForLastSeq(file, io, 1); try std.testing.expectEqual(@as(?u64, 2), scan.last_seq); // torn record 3 rejected try std.testing.expectEqual(valid_end, scan.valid_end); // stops at end of record 2 } +test "scanForLastSeq rejects a record with a garbage seq" { + // regression for the 2026-06 quintillion-seq incident: a torn tail left bytes + // that decode to a structurally-valid header (good kind, plausible len, body + // present) but whose seq field is actually CBOR payload (0x6363a382...). the + // old scan trusted that seq and relocated the whole sequencer; the seq-chain + // check must reject it so recovery resumes in the real space. + const io = std.testing.io; + var tmp = std.testing.tmpDir(.{}); + defer tmp.cleanup(); + + var file = try tmp.dir.createFile(io, "log", .{ .truncate = true, .read = true }); + defer file.close(io); + + var pos: u64 = 0; + const writeRecord = struct { + fn go(f: Io.File, tio: Io, at: *u64, seq: u64, payload: []const u8) !void { + var hbuf: [header_size]u8 = undefined; + (EvtHeader{ .flags = 0, .kind = @intFromEnum(EvtKind.commit), .len = @intCast(payload.len), .uid = 0, .seq = seq }).encode(&hbuf); + try f.writePositionalAll(tio, &hbuf, at.*); + at.* += header_size; + try f.writePositionalAll(tio, payload, at.*); + at.* += payload.len; + } + }.go; + + // file declares seq_start = 100; two good records continue the dense chain + try writeRecord(file, io, &pos, 100, "aa"); + try writeRecord(file, io, &pos, 101, "bbbb"); + const valid_end = pos; + + // a record that is structurally valid (good kind, small len, body present) + // but whose seq is the garbage CBOR value — breaks the chain (expected 102) + try writeRecord(file, io, &pos, 0x6363a38200000000, "zz"); + + const scan = try scanForLastSeq(file, io, 100); + try std.testing.expectEqual(@as(?u64, 101), scan.last_seq); // garbage-seq record rejected + try std.testing.expectEqual(valid_end, scan.valid_end); // stops at end of record 101 + + // and a file whose very first record is already garbage recovers nothing — + // resumeLog then falls back to seq_start rather than the junk value + var f2 = try tmp.dir.createFile(io, "log2", .{ .truncate = true, .read = true }); + defer f2.close(io); + var p2: u64 = 0; + try writeRecord(f2, io, &p2, 0x6363a38200000000, "aa"); + const scan2 = try scanForLastSeq(f2, io, 100); + try std.testing.expectEqual(@as(?u64, null), scan2.last_seq); + try std.testing.expectEqual(@as(u64, 0), scan2.valid_end); +} + fn requireDatabaseUrl() ![]const u8 { return getenv("DATABASE_URL") orelse return error.SkipZigTest; } -- 2.51.2