From 37be4abe260a51dfd69146d35564a2bcd8d049eb Mon Sep 17 00:00:00 2001 From: onevcat Date: Sun, 2 Aug 2026 16:31:48 +0900 Subject: [PATCH] Add ratio-assertion benchmarks pinning the performance-wave hot paths Four serialized Swift Testing suites pit the shipped implementations from the #644-#665 wave against verbatim copies of the code they replaced: the untracked line-count scanner vs the Data.reduce reader, normalize vs the regex-always formulation, the worktree directory index vs the per-row PathPolicy scan, and warm TranscriptFragmentCache scoring vs cold re-parsing. Each suite asserts output equivalence before timing, and asserts a floor ratio measured interleaved so machine speed and load cancel out. Ratios whose slow side is a C call or filesystem I/O assert in every build mode; the two Swift-vs-Swift ratios (escape guard, dense-input hybrid fallback) collapse under -Onone and assert only in optimized builds, detected at runtime through an assert side effect. Release test builds surfaced two pre-existing snags: tests exercising Debug-only members (CommandIconMap.debugAllEntries, CLISocketServer.debugFileDescriptors) now compile away outside Debug, and GhosttyRuntime declares @MainActor explicitly because whole-module swiftmodule deserialization loses default-isolation inference for its isolated deinit. Claude-Session: https://claude.ai/code/session_01TWSNesKV4dk8HLKmcNEJjS --- .../Ghostty/GhosttyRuntime.swift | 6 + supacodeTests/BenchmarkMeasurement.swift | 131 ++++++++++++++++++ supacodeTests/CLISocketServerTests.swift | 24 ++-- supacodeTests/CommandIconMapTests.swift | 30 ++-- .../FingerprintNormalizeBenchmarks.swift | 77 ++++++++++ supacodeTests/LineCountScanBenchmarks.swift | 121 ++++++++++++++++ supacodeTests/SessionScoringBenchmarks.swift | 90 ++++++++++++ .../WorktreeDirectoryIndexBenchmarks.swift | 99 +++++++++++++ 8 files changed, 555 insertions(+), 23 deletions(-) create mode 100644 supacodeTests/BenchmarkMeasurement.swift create mode 100644 supacodeTests/FingerprintNormalizeBenchmarks.swift create mode 100644 supacodeTests/LineCountScanBenchmarks.swift create mode 100644 supacodeTests/SessionScoringBenchmarks.swift create mode 100644 supacodeTests/WorktreeDirectoryIndexBenchmarks.swift diff --git a/supacode/Infrastructure/Ghostty/GhosttyRuntime.swift b/supacode/Infrastructure/Ghostty/GhosttyRuntime.swift index def854b8..d6c0a2eb 100644 --- a/supacode/Infrastructure/Ghostty/GhosttyRuntime.swift +++ b/supacode/Infrastructure/Ghostty/GhosttyRuntime.swift @@ -5,6 +5,12 @@ import UniformTypeIdentifiers nonisolated let ghosttyLogger = SupaLogger("GhosttyRuntime") +// Explicitly isolated rather than relying on the module's MainActor default: +// when a Release (whole-module) build's swiftmodule is deserialized by the test +// target, default-isolation inference is lost for the `isolated deinit` check +// and compilation fails with "containing class is not isolated to an actor". +// The explicit attribute is semantically identical and serializes correctly. +@MainActor final class GhosttyRuntime { nonisolated static let ghosttyExecutableCandidates = [ "/Applications/Ghostty.app/Contents/MacOS/ghostty", diff --git a/supacodeTests/BenchmarkMeasurement.swift b/supacodeTests/BenchmarkMeasurement.swift new file mode 100644 index 00000000..7f999714 --- /dev/null +++ b/supacodeTests/BenchmarkMeasurement.swift @@ -0,0 +1,131 @@ +import Foundation +import Testing + +@testable import supacode + +/// Root suite for the timed benchmarks that pin the hot paths optimized in the +/// #644–#665 performance wave against the naive implementations they replaced. +/// +/// Assertions are ratios, never absolute times: reference and shipped bodies run +/// interleaved in one process, so machine speed and background load cancel out. +/// Thresholds sit far below the ratios measured in a Release build, which is +/// what keeps the default Debug test run from flaking. Serialized recursively +/// because two timing suites running concurrently would distort each other's +/// medians in a way interleaving cannot compensate for. +/// +/// `make bench` runs this suite alone with `-O` and `PROWL_BENCH_REPORT=1`, +/// switching to full-size inputs and appending absolute medians to the bench +/// log — the durable per-machine time series the ratio assertions never read. +@Suite(.serialized) +struct PerformanceBenchmarks {} + +nonisolated enum BenchmarkMeasurement { + /// Full mode uses input sizes comparable to the docs-ai/056 baselines and + /// reports absolute numbers; the default sizes keep the Debug-mode run short. + static var isFullMode: Bool { + ProcessInfo.processInfo.environment["PROWL_BENCH_REPORT"] == "1" + } + + static var iterations: Int { isFullMode ? 15 : 5 } + + /// True in `-O` builds: `assert` bodies execute only under `-Onone`. This is + /// what distinguishes `make bench` from the regular Debug test run — `DEBUG` + /// stays defined in both, so a compilation condition cannot tell them apart. + /// + /// Ratios whose slow side is a C call (`memchr`, filesystem I/O) hold in any + /// build mode; ratios between two Swift-level formulations only mean anything + /// once both sides are optimized, so those assertions gate on this. Measured + /// in Debug before gating: the escape-absence guard's own byte scan drops to + /// 1.13x the regex it guards, and the hybrid scanner's raw-pointer fallback + /// runs at 0.45x the `Data.reduce` reader it replaced. + static var isOptimizedBuild: Bool { + var optimized = true + assert( + { + optimized = false + return true + }() + ) + return optimized + } + + /// Medians for two bodies measured alternately, so a load spike lands on both + /// sides of the ratio instead of biasing whichever side it happened to hit. + static func interleavedMedians( + reference: () -> Void, + shipped: () -> Void + ) -> (reference: Duration, shipped: Duration) { + // One untimed round faults in file caches and lazy runtime state. + reference() + shipped() + var referenceTimes: [Duration] = [] + var shippedTimes: [Duration] = [] + for _ in 0.. Void) -> Duration { + let start = ContinuousClock.now + body() + return ContinuousClock.now - start + } + + static func median(_ values: [Duration]) -> Duration { + values.sorted()[values.count / 2] + } + + static func milliseconds(_ duration: Duration) -> Double { + Double(duration.components.seconds) * 1_000 + Double(duration.components.attoseconds) / 1e15 + } + + static func ratio(_ medians: (reference: Duration, shipped: Duration)) -> Double { + milliseconds(medians.reference) / milliseconds(medians.shipped) + } + + /// Appends one measurement to the bench log when running under `make bench`. + /// JSON lines keyed by git SHA keep the series across commits comparable on + /// one machine; nothing in the test assertions ever reads this file back. + static func report(suite: String, name: String, medians: (reference: Duration, shipped: Duration)) { + guard isFullMode else { return } + let environment = ProcessInfo.processInfo.environment + let record = Record( + date: Date.now.ISO8601Format(), + suite: suite, + name: name, + referenceMilliseconds: milliseconds(medians.reference), + shippedMilliseconds: milliseconds(medians.shipped), + ratio: ratio(medians), + iterations: iterations, + gitSHA: environment["PROWL_BENCH_GIT_SHA"] + ) + guard let line = try? JSONEncoder().encode(record) else { return } + + let directory = + environment["PROWL_BENCH_LOG_DIR"].map { URL(fileURLWithPath: $0) } + ?? FileManager.default.homeDirectoryForCurrentUser + .appending(path: "Library/Logs/Prowl/measurements/bench") + let logURL = directory.appending(path: "bench.jsonl") + try? FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + if !FileManager.default.fileExists(atPath: logURL.path(percentEncoded: false)) { + FileManager.default.createFile(atPath: logURL.path(percentEncoded: false), contents: nil) + } + guard let handle = try? FileHandle(forWritingTo: logURL) else { return } + defer { try? handle.close() } + _ = try? handle.seekToEnd() + try? handle.write(contentsOf: line + Data("\n".utf8)) + } + + private struct Record: Encodable { + let date: String + let suite: String + let name: String + let referenceMilliseconds: Double + let shippedMilliseconds: Double + let ratio: Double + let iterations: Int + let gitSHA: String? + } +} diff --git a/supacodeTests/CLISocketServerTests.swift b/supacodeTests/CLISocketServerTests.swift index e58677a4..7e520d41 100644 --- a/supacodeTests/CLISocketServerTests.swift +++ b/supacodeTests/CLISocketServerTests.swift @@ -55,16 +55,20 @@ struct CLISocketServerTests { #expect(canConnect(to: socketPath)) } - @Test func ownedDescriptorsAreClosedOnExec() throws { - let socketPath = temporarySocketPath(suffix: "cloexec") - let server = CLISocketServer(router: CLICommandRouter(), socketPath: socketPath) - try server.start() - defer { server.stop() } - - let descriptors = server.debugFileDescriptors - #expect(isCloseOnExec(descriptors.server)) - #expect(isCloseOnExec(descriptors.lock)) - } + // `debugFileDescriptors` exists only in Debug builds, and so does this test — + // `make bench` compiles this target under the Release configuration. + #if DEBUG + @Test func ownedDescriptorsAreClosedOnExec() throws { + let socketPath = temporarySocketPath(suffix: "cloexec") + let server = CLISocketServer(router: CLICommandRouter(), socketPath: socketPath) + try server.start() + defer { server.stop() } + + let descriptors = server.debugFileDescriptors + #expect(isCloseOnExec(descriptors.server)) + #expect(isCloseOnExec(descriptors.lock)) + } + #endif @Test func socketFilesAreOwnerOnly() throws { let socketPath = temporarySocketPath(suffix: "permissions") diff --git a/supacodeTests/CommandIconMapTests.swift b/supacodeTests/CommandIconMapTests.swift index 821998b3..18801f24 100644 --- a/supacodeTests/CommandIconMapTests.swift +++ b/supacodeTests/CommandIconMapTests.swift @@ -93,18 +93,22 @@ struct CommandIconMapTests { // MARK: - Debug catalog - @Test func debugAllEntriesIsSorted() { - let tokens = CommandIconMap.debugAllEntries.map(\.token) - #expect(tokens == tokens.sorted()) - } + // `debugAllEntries` exists only in Debug builds, and so do these tests — + // `make bench` compiles this target under the Release configuration. + #if DEBUG + @Test func debugAllEntriesIsSorted() { + let tokens = CommandIconMap.debugAllEntries.map(\.token) + #expect(tokens == tokens.sorted()) + } - @Test func debugAllEntriesCoversWellKnownTokens() { - let tokens = Set(CommandIconMap.debugAllEntries.map(\.token)) - // Spot-check that the debug surface actually exposes the tokens - // a user is most likely to hunt for. - let mustHave: Set = [ - "git", "docker", "claude", "vim", "ssh", "npm", "swift", - ] - #expect(mustHave.isSubset(of: tokens)) - } + @Test func debugAllEntriesCoversWellKnownTokens() { + let tokens = Set(CommandIconMap.debugAllEntries.map(\.token)) + // Spot-check that the debug surface actually exposes the tokens + // a user is most likely to hunt for. + let mustHave: Set = [ + "git", "docker", "claude", "vim", "ssh", "npm", "swift", + ] + #expect(mustHave.isSubset(of: tokens)) + } + #endif } diff --git a/supacodeTests/FingerprintNormalizeBenchmarks.swift b/supacodeTests/FingerprintNormalizeBenchmarks.swift new file mode 100644 index 00000000..4854c149 --- /dev/null +++ b/supacodeTests/FingerprintNormalizeBenchmarks.swift @@ -0,0 +1,77 @@ +import Foundation +import Testing + +@testable import supacode + +extension PerformanceBenchmarks { + /// Pins the #650/#657 escape-absence guard: nearly every transcript fragment + /// holds no ESC byte, so proving absence with a byte scan must stay cheaper + /// than letting the regex engine walk the whole string to conclude nothing. + /// + /// The corpus mirrors the measured workload shape from docs-ai/032.004 — + /// mostly non-ASCII bytes, a small minority of fragments carrying escapes — + /// which is also why no separate ASCII-fast-path ratio is asserted: on this + /// byte mix its measured gain is marginal, and the equivalence tests in + /// `AgentSessionFingerprintNormalizeTests` already pin its semantics. + @Suite + struct FingerprintNormalizeBenchmarks { + /// Optimized builds only: under `-Onone` the guard's own byte scan is an + /// unspecialized `Sequence.contains` and its advantage over the regex + /// collapses to ~1.13x — see `BenchmarkMeasurement.isOptimizedBuild`. + @Test(.enabled(if: BenchmarkMeasurement.isOptimizedBuild)) + func escapeAbsenceGuardOutpacesTheRegexOnlyFormulation() { + let corpus = Self.corpus() + + for fragment in corpus { + #expect(AgentSessionFingerprintMatcher.normalize(fragment) == Self.referenceNormalize(fragment)) + } + + let medians = BenchmarkMeasurement.interleavedMedians( + reference: { + for fragment in corpus { _ = Self.referenceNormalize(fragment) } + }, + shipped: { + for fragment in corpus { _ = AgentSessionFingerprintMatcher.normalize(fragment) } + } + ) + BenchmarkMeasurement.report(suite: "FingerprintNormalize", name: "mixed-corpus", medians: medians) + #expect( + BenchmarkMeasurement.ratio(medians) >= 1.3, + "shipped normalize was only \(BenchmarkMeasurement.ratio(medians))x the regex-only formulation" + ) + } + + /// The pre-#650 formulation: the escape-stripping regex runs whether or not + /// an ESC byte exists. Kept verbatim so the benchmark measures exactly the + /// path the guard replaced. + private static func referenceNormalize(_ value: String) -> String { + value + .replacing(#/\u{001B}\[[0-?]*[ -\/]*[@-~]/#, with: " ") + .lowercased() + .split(whereSeparator: \Character.isWhitespace) + .joined(separator: " ") + } + + /// 240 fragments, ~80% non-ASCII bytes, 2 carrying real CSI sequences — + /// the "238 of 240 fragments contained no ESC" shape the guard was built + /// for. Deterministic so every run measures the same bytes. + private static func corpus() -> [String] { + let fragmentCount = BenchmarkMeasurement.isFullMode ? 240 : 60 + let cjkLine = "エージェントが長い応答を生成しています。コードの説明と修正の提案を含む本文のテキストです。" + let asciiLine = "The agent produced MIXED Case output with collapsing\twhitespace and code: let x = f(y)" + var fragments: [String] = [] + for index in 0..= minimumRatio, + "shipped scanner was only \(BenchmarkMeasurement.ratio(medians))x the reduce reader on \(name)" + ) + } + + private static var inputByteCount: Int { + (BenchmarkMeasurement.isFullMode ? 2_048 : 256) * 1_024 + } + + /// Text shaped like a `sample(1)` capture — the workload #644 was written + /// against: moderate lines, leading indentation, pure ASCII. + private static func sparseText(byteCount: Int) -> Data { + var out = Data(capacity: byteCount + 128) + var lineNumber = 0 + while out.count < byteCount { + let indent = String(repeating: " ", count: 4 + (lineNumber % 5) * 2) + let line = "\(indent)\(1_200 + lineNumber % 800) Thread_\(90_000 + lineNumber) com.apple.main-thread\n" + out.append(Data(line.utf8)) + lineNumber += 1 + } + return out.prefix(byteCount) + } + + /// The pre-#644 reader, verbatim: 64 KiB chunks walked through + /// `Data.Iterator` with a value-witness call per byte. + private static func referenceCountLines(in fileURL: URL) -> Int? { + guard let handle = try? FileHandle(forReadingFrom: fileURL) else { return nil } + defer { try? handle.close() } + + let binaryProbeByteCount = 8_192 + let chunkByteCount = 64 * 1_024 + var probedByteCount = 0 + var lineCount = 0 + var isEmpty = true + var lastByte: UInt8? + + while true { + guard let chunk = try? handle.read(upToCount: chunkByteCount), !chunk.isEmpty else { break } + isEmpty = false + if probedByteCount < binaryProbeByteCount { + let remainingProbeCount = binaryProbeByteCount - probedByteCount + let probe = chunk.prefix(remainingProbeCount) + if probe.contains(0x00) { return nil } + probedByteCount += probe.count + } + lineCount += chunk.reduce(0) { $0 + ($1 == 0x0A ? 1 : 0) } + lastByte = chunk.last + } + + if !isEmpty, lastByte != 0x0A { + lineCount += 1 + } + return lineCount + } + } +} diff --git a/supacodeTests/SessionScoringBenchmarks.swift b/supacodeTests/SessionScoringBenchmarks.swift new file mode 100644 index 00000000..1ba36e07 --- /dev/null +++ b/supacodeTests/SessionScoringBenchmarks.swift @@ -0,0 +1,90 @@ +import Foundation +import Testing + +@testable import supacode + +extension PerformanceBenchmarks { + /// Pins the #650/#657 fragment cache: a poll whose transcript tails are + /// unchanged must not pay tail reads, JSON parsing, and normalization again. + /// Cold rounds rebuild a fresh `TranscriptFragmentCache` per call — the + /// pre-#650 per-poll cost — while warm rounds reuse one cache the way + /// `AgentSessionResolver` does across its 300 ms polls. + @Suite + struct SessionScoringBenchmarks { + @Test func warmFragmentCacheOutpacesColdReparsingPerPoll() throws { + let fileManager = FileManager.default + let tempRoot = fileManager.temporaryDirectory.appending(path: UUID().uuidString) + defer { try? fileManager.removeItem(at: tempRoot) } + try fileManager.createDirectory(at: tempRoot, withIntermediateDirectories: true) + + let marker = "the quick brown benchmark fox jumps over the lazy resolver dog" + let sessionCount = 6 + let linesPerTranscript = BenchmarkMeasurement.isFullMode ? 400 : 100 + var candidates: [AgentSessionCandidate] = [] + for sessionIndex in 0..= 3, + "warm fragment cache was only \(BenchmarkMeasurement.ratio(medians))x cold re-parsing" + ) + } + } +} + +/// `bestMatch` takes the cache `inout`; a closure cannot capture `inout` state, +/// so the warm cache lives in a reference box instead. +private final class FragmentCacheBox { + var cache = TranscriptFragmentCache() +} diff --git a/supacodeTests/WorktreeDirectoryIndexBenchmarks.swift b/supacodeTests/WorktreeDirectoryIndexBenchmarks.swift new file mode 100644 index 00000000..7712e10c --- /dev/null +++ b/supacodeTests/WorktreeDirectoryIndexBenchmarks.swift @@ -0,0 +1,99 @@ +import Foundation +import IdentifiedCollections +import Testing + +@testable import supacode + +extension PerformanceBenchmarks { + /// Pins the #648/#655 directory index against the pre-#648 shape it replaced: + /// every agent row scanning every worktree, with `PathPolicy` normalizing both + /// sides of each containment test — filesystem round-trips per (row, worktree) + /// pair. The index normalizes each worktree once at build time and each query + /// once at lookup, so even the worst case (build plus a full query batch) + /// must beat one naive batch. + @Suite + struct WorktreeDirectoryIndexBenchmarks { + @Test func indexBuildPlusQueryBatchOutpacesThePerRowScan() throws { + let fileManager = FileManager.default + let tempRoot = fileManager.temporaryDirectory.appending(path: UUID().uuidString) + defer { try? fileManager.removeItem(at: tempRoot) } + + let repositoryCount = BenchmarkMeasurement.isFullMode ? 6 : 3 + let worktreesPerRepository = 4 + var repositories: IdentifiedArrayOf = [] + var queries: [URL] = [] + for repositoryIndex in 0.. = [] + for worktreeIndex in 0..= 2, + "index build plus batch was only \(BenchmarkMeasurement.ratio(medians))x the per-row scan" + ) + } + + /// The pre-#648 resolution shape: every query walks every worktree, and each + /// containment test normalizes both sides again via `PathPolicy`. + private static func referenceResolve(_ query: URL, in worktrees: [Worktree]) -> Worktree.ID? { + var best: (id: Worktree.ID, depth: Int)? + for worktree in worktrees where PathPolicy.contains(query, in: worktree.workingDirectory) { + let depth = PathPolicy.normalizeURL(worktree.workingDirectory).pathComponents.count + if depth > (best?.depth ?? -1) { + best = (worktree.id, depth) + } + } + return best?.id + } + } +} -- 2.51.2