diff --git a/.gitattributes b/.gitattributes new file mode 100644 index 00000000..b20d14f4 --- /dev/null +++ b/.gitattributes @@ -0,0 +1,2 @@ +# Agent-screen fixtures preserve trailing blank terminal rows as detector input. +supacodeTests/Fixtures/AgentScreenDetection/**/*.txt -whitespace diff --git a/supacode.xcodeproj/project.pbxproj b/supacode.xcodeproj/project.pbxproj index 3c38bb5f..69af0dfb 100644 --- a/supacode.xcodeproj/project.pbxproj +++ b/supacode.xcodeproj/project.pbxproj @@ -68,6 +68,9 @@ }; D6F821022F1FCBC1004B4174 /* supacodeTests */ = { isa = PBXFileSystemSynchronizedRootGroup; + explicitFolders = ( + Fixtures, + ); path = supacodeTests; sourceTree = ""; }; diff --git a/supacodeTests/AgentScreenFixtureCorpusTests.swift b/supacodeTests/AgentScreenFixtureCorpusTests.swift new file mode 100644 index 00000000..b0df437a --- /dev/null +++ b/supacodeTests/AgentScreenFixtureCorpusTests.swift @@ -0,0 +1,258 @@ +import Foundation +import Testing + +@testable import supacode + +struct AgentScreenFixtureCorpusTests { + @Test func initialCorpusCoversClaudeAndCodexLifecycleStates() throws { + let fixtures = try AgentScreenFixtureCorpus.load() + + for agent in [DetectedAgent.claude, .codex] { + let states = + fixtures + .filter { $0.agent == agent && !$0.isQuarantined } + .map(\.expectedState) + #expect(states.contains(.blocked)) + #expect(states.contains(.working)) + #expect(states.contains(.idle)) + } + + #expect( + fixtures.contains { + $0.agent == .claude + && $0.isQuarantined + && $0.expectedState == .unknown + && $0.currentState == .idle + } + ) + } + + @Test func capturedFixturesMatchCurrentDetector() throws { + let fixtures = try AgentScreenFixtureCorpus.load() + + #expect(!fixtures.isEmpty, "The captured screen corpus must not be empty.") + for fixture in fixtures { + #expect( + fixture.text == AgentScreenFixtureCorpus.canonicalTail(fixture.text), + "Fixture is not the canonical 24-non-empty-line detector tail: \(fixture.relativePath)" + ) + + let actualState = fixture.agent.detectState(in: fixture.text) + let mismatchMessage = + "Fixture \(fixture.relativePath) expected \(fixture.currentState.rawValue), got \(actualState.rawValue)" + #expect(actualState == fixture.currentState, Comment(rawValue: mismatchMessage)) + + if fixture.isQuarantined { + #expect( + fixture.expectedState != fixture.currentState, + "Quarantined fixture no longer describes a misdetection: \(fixture.relativePath)" + ) + #expect( + fixture.metadata.issue != nil, + "Quarantined fixture must link an issue: \(fixture.relativePath)" + ) + } else { + #expect(fixture.expectedState == fixture.currentState) + #expect(fixture.metadata.issue == nil) + } + } + } +} + +private enum AgentScreenFixtureCorpus { + static let recentNonEmptyLineLimit = 24 + static let root = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .appending(path: "Fixtures/AgentScreenDetection", directoryHint: .isDirectory) + + static func load() throws -> [AgentScreenFixture] { + let fileManager = FileManager.default + guard + let enumerator = fileManager.enumerator( + at: root, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles] + ) + else { + throw CorpusError("Fixture root is unavailable: \(root.path())") + } + + let urls = enumerator.compactMap { $0 as? URL } + let screenURLs = + urls + .filter { $0.pathExtension == "txt" } + .sorted { $0.path() < $1.path() } + let metadataURLs = Set(urls.filter { $0.lastPathComponent.hasSuffix(".metadata.json") }) + + var consumedMetadataURLs: Set = [] + let fixtures = try screenURLs.map { screenURL in + let fixture = try loadFixture(at: screenURL) + consumedMetadataURLs.insert(fixture.metadataURL) + return fixture + } + + let orphanedMetadata = metadataURLs.subtracting(consumedMetadataURLs) + guard orphanedMetadata.isEmpty else { + throw CorpusError( + "Metadata without a matching screen fixture: \(orphanedMetadata.map(\.lastPathComponent).sorted())" + ) + } + return fixtures + } + + static func canonicalTail(_ content: String) -> String { + let lines = content.split(separator: "\n", omittingEmptySubsequences: false).map(String.init) + var remainingNonEmptyLines = recentNonEmptyLineLimit + var startIndex = lines.startIndex + + for index in lines.indices.reversed() { + guard !lines[index].trimmingCharacters(in: .whitespaces).isEmpty else { continue } + remainingNonEmptyLines -= 1 + if remainingNonEmptyLines == 0 { + startIndex = index + break + } + } + return lines[startIndex...].joined(separator: "\n") + } + + private static func loadFixture(at screenURL: URL) throws -> AgentScreenFixture { + let relativePath = relativePath(for: screenURL) + let components = relativePath.split(separator: "/").map(String.init) + let layout = try FixtureLayout(components: components, relativePath: relativePath) + let metadataURL = screenURL.deletingPathExtension().appendingPathExtension("metadata.json") + + guard FileManager.default.fileExists(atPath: metadataURL.path()) else { + throw CorpusError("Missing metadata for \(relativePath)") + } + + let metadata = try JSONDecoder().decode( + AgentScreenFixtureMetadata.self, + from: Data(contentsOf: metadataURL) + ) + try validate(metadata: metadata, layout: layout, relativePath: relativePath) + + return AgentScreenFixture( + relativePath: relativePath, + metadataURL: metadataURL, + agent: layout.agent, + expectedState: layout.expectedState, + currentState: layout.currentState, + isQuarantined: layout.isQuarantined, + text: try String(contentsOf: screenURL, encoding: .utf8), + metadata: metadata + ) + } + + private static func validate( + metadata: AgentScreenFixtureMetadata, + layout: FixtureLayout, + relativePath: String + ) throws { + guard metadata.schemaVersion == 1 else { + throw CorpusError("Unsupported metadata schema for \(relativePath): \(metadata.schemaVersion)") + } + guard metadata.cliVersion == layout.cliVersion else { + throw CorpusError("CLI version metadata does not match path for \(relativePath)") + } + guard metadata.captureSource == "prowl-read-detection" else { + throw CorpusError("Invalid capture source for \(relativePath): \(metadata.captureSource)") + } + guard ISO8601DateFormatter().date(from: metadata.capturedAt) != nil else { + throw CorpusError("Invalid capture timestamp for \(relativePath): \(metadata.capturedAt)") + } + guard metadata.terminal.columns > 0, metadata.terminal.rows > 0 else { + throw CorpusError("Invalid terminal geometry for \(relativePath)") + } + guard !metadata.redactions.isEmpty else { + throw CorpusError("Missing redaction summary for \(relativePath)") + } + } + + private static func relativePath(for url: URL) -> String { + let path = String(url.path().dropFirst(root.path().count)) + return path.hasPrefix("/") ? String(path.dropFirst()) : path + } +} + +private struct AgentScreenFixture { + let relativePath: String + let metadataURL: URL + let agent: DetectedAgent + let expectedState: AgentRawState + let currentState: AgentRawState + let isQuarantined: Bool + let text: String + let metadata: AgentScreenFixtureMetadata +} + +private struct AgentScreenFixtureMetadata: Decodable { + struct Terminal: Decodable { + let columns: Int + let rows: Int + } + + let schemaVersion: Int + let capturedAt: String + let cliVersion: String + let captureSource: String + let terminal: Terminal + let redactions: [String] + let issue: String? + + private enum CodingKeys: String, CodingKey { + case schemaVersion = "schema_version" + case capturedAt = "captured_at" + case cliVersion = "cli_version" + case captureSource = "capture_source" + case terminal + case redactions + case issue + } +} + +private struct FixtureLayout { + let agent: DetectedAgent + let cliVersion: String + let expectedState: AgentRawState + let currentState: AgentRawState + let isQuarantined: Bool + + init(components: [String], relativePath: String) throws { + guard components.count == 4 || components.count == 6 else { + throw CorpusError("Invalid fixture path layout: \(relativePath)") + } + guard let agent = DetectedAgent(rawValue: components[0]) else { + throw CorpusError("Unknown runtime in fixture path: \(relativePath)") + } + self.agent = agent + self.cliVersion = components[1] + + if components.count == 4 { + guard let state = AgentRawState(rawValue: components[2]) else { + throw CorpusError("Invalid expected state in fixture path: \(relativePath)") + } + self.expectedState = state + self.currentState = state + self.isQuarantined = false + } else { + guard components[2] == "known-misdetection", + let expectedState = AgentRawState(rawValue: components[3]), + let currentState = AgentRawState(rawValue: components[4]) + else { + throw CorpusError("Invalid quarantine path layout: \(relativePath)") + } + self.expectedState = expectedState + self.currentState = currentState + self.isQuarantined = true + } + } +} + +private struct CorpusError: Error, CustomStringConvertible { + let description: String + + init(_ description: String) { + self.description = description + } +} diff --git a/supacodeTests/BenchmarkMeasurement.swift b/supacodeTests/BenchmarkMeasurement.swift index 7f999714..f1e73999 100644 --- a/supacodeTests/BenchmarkMeasurement.swift +++ b/supacodeTests/BenchmarkMeasurement.swift @@ -67,6 +67,11 @@ nonisolated enum BenchmarkMeasurement { return (median(referenceTimes), median(shippedTimes)) } + static func repeatedMedian(_ body: () -> Void) -> Duration { + body() + return median((0.. Void) -> Duration { let start = ContinuousClock.now body() @@ -85,6 +90,20 @@ nonisolated enum BenchmarkMeasurement { milliseconds(medians.reference) / milliseconds(medians.shipped) } + static func reportAbsolute(suite: String, name: String, median: Duration) { + guard isFullMode else { return } + let environment = ProcessInfo.processInfo.environment + let record = AbsoluteRecord( + date: Date.now.ISO8601Format(), + suite: suite, + name: name, + medianMilliseconds: milliseconds(median), + iterations: iterations, + gitSHA: environment["PROWL_BENCH_GIT_SHA"] + ) + write(record) + } + /// Appends one measurement to the bench log when running under `make bench`. /// JSON lines keyed by git SHA keep the series across commits comparable on /// one machine; nothing in the test assertions ever reads this file back. @@ -101,8 +120,12 @@ nonisolated enum BenchmarkMeasurement { iterations: iterations, gitSHA: environment["PROWL_BENCH_GIT_SHA"] ) - guard let line = try? JSONEncoder().encode(record) else { return } + write(record) + } + private static func write(_ record: some Encodable) { + guard let line = try? JSONEncoder().encode(record) else { return } + let environment = ProcessInfo.processInfo.environment let directory = environment["PROWL_BENCH_LOG_DIR"].map { URL(fileURLWithPath: $0) } ?? FileManager.default.homeDirectoryForCurrentUser @@ -118,6 +141,15 @@ nonisolated enum BenchmarkMeasurement { try? handle.write(contentsOf: line + Data("\n".utf8)) } + private struct AbsoluteRecord: Encodable { + let date: String + let suite: String + let name: String + let medianMilliseconds: Double + let iterations: Int + let gitSHA: String? + } + private struct Record: Encodable { let date: String let suite: String diff --git a/supacodeTests/Fixtures/AgentScreenDetection/README.md b/supacodeTests/Fixtures/AgentScreenDetection/README.md new file mode 100644 index 00000000..cd7e3b68 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/README.md @@ -0,0 +1,83 @@ +# Agent Screen Detection Fixture Corpus + +This directory contains sanitized, capture-derived inputs for the production screen +classifier. It is a regression suite, not a transcript archive and not a replacement for +focused inline predicate tests. + +## Layout + +Normal fixture: + +```text +///.txt +///.metadata.json +``` + +Known current misdetection: + +```text +//known-misdetection///-.txt +//known-misdetection///-.metadata.json +``` + +The harness accepts every `DetectedAgent`; the initial corpus contains only `claude` and +`codex`. States are `working`, `blocked`, `idle`, or `unknown`; `done` is display state +derived from `idle + unseen` and is never a fixture state. + +## Capture and promotion + +1. Run a current Debug build of Prowl and the matching bundled `prowl` CLI. +2. Capture the exact production input with `prowl read --source detection --json`. +3. Require `.data.source == "detection"`; never substitute a viewport read. +4. Keep the raw response under the ignored `.local/agent-screen-captures/` directory. +5. Record exact CLI version, capture timestamp, terminal rows/columns, and the redaction + summary in a same-basename metadata file. +6. Reduce the capture to the exact canonical tail the detector receives: start at the + 24th non-empty line from the bottom, retaining blank lines and trailing screen rows + inside that window. +7. Redact paths, repositories, account identifiers, prompts, and model output without + changing runtime chrome, line ordering, markers, wrapping, or blank-line boundaries. +8. Run `AgentScreenFixtureCorpusTests` before committing. + +The loader resolves this tree through `#filePath`, so tests intentionally run from a +source checkout rather than relying on test-bundle resource flattening. + +Required metadata shape: + +```json +{ + "schema_version": 1, + "captured_at": "2026-08-06T12:34:56Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 120, + "rows": 40 + }, + "redactions": [ + "working directory replaced with ", + "user prompt replaced with " + ], + "issue": null +} +``` + +`issue` is required and non-null only in `known-misdetection`. + +## Quarantine + +A fresh capture that the current detector misclassifies must not be omitted or made into a +failing infrastructure PR. Put it under `known-misdetection`, encode both intended and +current states in the path, and link the tracking issue in metadata. The corpus test +asserts the current behavior, so a later detector fix makes the quarantine fail until the +fixture is promoted to the normal expected-state path. + +## Retention and privacy + +- Keep the newest verified fixture for each scenario/UI shape. +- Retain an older CLI version only when its distinct shape remains intentionally supported. +- Remove byte-equivalent history. +- Never commit raw captures, credentials, user prompts, model output, account names, home + paths, or repository names. +- Reconstructed and synthetic screens stay inline in `ScreenHeuristicsTests.swift`; they + must not be version-stamped as captured evidence here. diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/command-permission.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/command-permission.metadata.json new file mode 100644 index 00000000..1f7649f2 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/command-permission.metadata.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:27:45Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "account and organization identifiers replaced with synthetic values", + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/command-permission.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/command-permission.txt new file mode 100644 index 00000000..677eb984 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/command-permission.txt @@ -0,0 +1,46 @@ + +╭─── Claude Code v2.1.223 ───────────────────────────────────────────────────────────────────────────────╮ +│ │ Tips for getting started │ +│ Welcome back fixture! │ Run /init to create a CLAUDE.md file with instru… │ +│ │ ───────────────────────────────────────────────── │ +│ ▐▛███▜▌ │ What's new │ +│ ▝▜█████▛▘ │ Added owner wildcard entries (`"owner/*"`) to th… │ +│ ▘▘ ▝▝ │ Added a warning when workflow agents, forked ski… │ +│ Fable 5 with xhigh effort · Claude XXX · │ Added a `/teleport` hint in cloud sessions showi… │ +│ 's Organization │ /release-notes for more │ +│ /…/prowl-fixture-workspace-live-00000/claude-main │ │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +❯ Run the shell command `touch permission-probe.txt` now. Do not explain. + + Running 1 shell command… + ⎿ $ touch permission-probe.txt + +────────────────────────────────────────────────────────────────────────────────────────────────────────── + Bash command + + touch permission-probe.txt + Create empty file permission-probe.txt + + Do you want to proceed? + ❯ 1. Yes + 2. Yes, and always allow access to claude-main/ from this project + 3. No + + Esc to cancel · Tab to amend · ctrl+e to explain + + + + + + + + + + + + + + + diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/workspace-trust.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/workspace-trust.metadata.json new file mode 100644 index 00000000..1318e034 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/workspace-trust.metadata.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:27:15Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "shell login record replaced with " + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/workspace-trust.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/workspace-trust.txt new file mode 100644 index 00000000..62d5fb53 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/blocked/workspace-trust.txt @@ -0,0 +1,21 @@ + +➜ claude-main git:(master) stty size > '/tmp/prowl-fixture-workspace-live-00000/claude-main/.terminal-size' +➜ claude-main git:(master) ✗ claude --permission-mode manual + +────────────────────────────────────────────────────────────────────────────────────────────────────────── + Accessing workspace: + + /private/tmp/prowl-fixture-workspace-live-00000/claude-main + + Quick safety check: Is this a project you created or one you trust? (Like your own code, a well-known + open source project, or work from your team). If not, take a moment to review what's in this folder + first. + + Claude Code'll be able to read, edit, and execute files here. + + Security guide + + ❯ 1. Yes, I trust this folder + 2. No, exit + + Enter to confirm · Esc to cancel \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/composer.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/composer.metadata.json new file mode 100644 index 00000000..626416e1 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/composer.metadata.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:27:27Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "account and organization identifiers replaced with synthetic values", + "usage balances and plan labels replaced with placeholders" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/composer.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/composer.txt new file mode 100644 index 00000000..e1929987 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/composer.txt @@ -0,0 +1,52 @@ + +╭─── Claude Code v2.1.223 ───────────────────────────────────────────────────────────────────────────────╮ +│ │ Tips for getting started │ +│ Welcome back fixture! │ Run /init to create a CLAUDE.md file with instru… │ +│ │ ───────────────────────────────────────────────── │ +│ ▐▛███▜▌ │ What's new │ +│ ▝▜█████▛▘ │ Added owner wildcard entries (`"owner/*"`) to th… │ +│ ▘▘ ▝▝ │ Added a warning when workflow agents, forked ski… │ +│ Fable 5 with xhigh effort · Claude XXX · │ Added a `/teleport` hint in cloud sessions showi… │ +│ 's Organization │ /release-notes for more │ +│ /…/prowl-fixture-workspace-live-00000/claude-main │ │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + ◉ xhigh · /effort +────────────────────────────────────────────────────────────────────────────────────────────────────────── +❯ Try "write a test for " +────────────────────────────────────────────────────────────────────────────────────────────────────────── + [Fable 5 | XXX] ██░░░░░░░░ 23% | claude-main | 1 CLAUDE.md | 1 MCPs | 3 hooks | 5h: XX% (XXm) /rc + ⏸ manual mode on · ← for agents \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.metadata.json new file mode 100644 index 00000000..338ec745 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.metadata.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:30:40Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "usage balances and plan labels replaced with placeholders", + "session identifier replaced with a placeholder", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.txt new file mode 100644 index 00000000..f7371dd8 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.txt @@ -0,0 +1,32 @@ +⏺ Agent(Run sleep 8, return DONE) + ⎿  Backgrounded agent (↓ to manage · ctrl+o to expand) + +⏺ Task Output + ⎿  Read output (ctrl+o to expand) + +⏺ Agent "Run sleep 8, return DONE" finished · 25s + +⏺ Subagent 已跑完喵～ 它执行了 sleep 8，返回结果是：DONE + +✻ Brewed for 32s + +❯ Reply with exactly these lines and nothing else: + Do you want to proceed? + ❯ 1. Yes + 2. No + Esc to cancel · Tab to amend + +⏺ Do you want to proceed? + ❯ 1. Yes + 2. No + Esc to cancel · Tab to amend + +✻ Crunched for 7s + +────────────────────────────────────────────────────────────────────────────────────────────────────────── +❯ +────────────────────────────────────────────────────────────────────────────────────────────────────────── + [Fable 5 | XXX] ░░░░░░░░░░ 4% | claude-main | 1 CLAUDE.md | 1 MCPs | 3 hooks | 5h: XX% (XXm) | ⏱️  2m + ✓ Bash ×2 | ✓ Agent ×1 | ✓ ToolSearch ×1 | ✓ TaskOutput ×1 + ⏸ manual mode on · ← for agents + /rc \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.metadata.json new file mode 100644 index 00000000..43979cbc --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.metadata.json @@ -0,0 +1,18 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:29:13Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "account and organization identifiers replaced with synthetic values", + "usage balances and plan labels replaced with placeholders", + "prompt history entries replaced with ", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": "https://github.com/onevcat/Prowl/issues/676" +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.txt new file mode 100644 index 00000000..a77db369 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.txt @@ -0,0 +1,42 @@ +│ /…/prowl-fixture-workspace-live-00000/claude-main │ │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +❯ Run the shell command `touch permission-probe.txt` now. Do not explain. + + Ran 1 shell command + +⏺ 好的喵～ permission-probe.txt 已创建。 + +✻ Baked for 10s + +❯ Run the shell command `sleep 8`, then reply exactly DONE. + + Ran 1 shell command + +⏺ DONE + +✻ Brewed for 19s + + + + + + + + + +────────────────────────────────────────────────────────────────────────────────────────────────────────── + Search prompts · everywhere + ↑ 2h ago + 1h ago + 1h ago + 50m ago + 38m ago + 38m ago + 1m ago + ❯ 1m ago + ╭────────────────────────────────────────────────────────────────────────────────────────────────────╮ + │ ⌕ Filter history… │ + ╰────────────────────────────────────────────────────────────────────────────────────────────────────╯ + ↑/↓ to nav · Enter to use · Esc to cancel · ctrl+s to scope \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.metadata.json new file mode 100644 index 00000000..4e555e2c --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.metadata.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:29:55Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "account and organization identifiers replaced with synthetic values", + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.txt new file mode 100644 index 00000000..b56349fc --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.txt @@ -0,0 +1,41 @@ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +❯ Run the shell command `touch permission-probe.txt` now. Do not explain. + + Ran 1 shell command + +⏺ 好的喵～ permission-probe.txt 已创建。 + +✻ Baked for 10s + +❯ Run the shell command `sleep 8`, then reply exactly DONE. + + Ran 1 shell command + +⏺ DONE + +✻ Brewed for 19s + +❯ Launch exactly one general-purpose subagent. Have it run sleep 8 and then return DONE. Wait for it + before responding. + +⏺ Agent(Run sleep 8, return DONE) + ⎿  Backgrounded agent (↓ to manage · ctrl+o to expand) + + + + +✢ Slithering… (16s · ↓ ) + ⎿  Tip: Use ctrl+v to paste images from your clipboard + +────────────────────────────────────────────────────────────────────────────────────────────────────────── +❯ +────────────────────────────────────────────────────────────────────────────────────────────────────────── + [Fable 5 | XXX] ░░░░░░░░░░ 4% | claude-main | 1 CLAUDE.md | 1 MCPs | 3 hooks | 5h: XX% (XXm) | ⏱️  2m + ◐ Agent | ✓ Bash ×2 + ⏸ manual mode on · 1 shell · ← for agents + /rc + + ⏺ main + ◯ general-purpose Run sleep 8, return DONE 11s · ↓ \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/foreground-spinner.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/foreground-spinner.metadata.json new file mode 100644 index 00000000..ded1d7c9 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/foreground-spinner.metadata.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:28:04Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "account and organization identifiers replaced with synthetic values", + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/foreground-spinner.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/foreground-spinner.txt new file mode 100644 index 00000000..27ac29d2 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/foreground-spinner.txt @@ -0,0 +1,51 @@ +╭─── Claude Code v2.1.223 ───────────────────────────────────────────────────────────────────────────────╮ +│ │ Tips for getting started │ +│ Welcome back fixture! │ Run /init to create a CLAUDE.md file with instru… │ +│ │ ───────────────────────────────────────────────── │ +│ ▐▛███▜▌ │ What's new │ +│ ▝▜█████▛▘ │ Added owner wildcard entries (`"owner/*"`) to th… │ +│ ▘▘ ▝▝ │ Added a warning when workflow agents, forked ski… │ +│ Fable 5 with xhigh effort · Claude XXX · │ Added a `/teleport` hint in cloud sessions showi… │ +│ 's Organization │ /release-notes for more │ +│ /…/prowl-fixture-workspace-live-00000/claude-main │ │ +╰────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +❯ Run the shell command `touch permission-probe.txt` now. Do not explain. + +⏺ Running 1 shell command… + ⎿ $ touch permission-probe.txt + + + + + + + + + + + + + + + + + + + + + + + +· Infusing… (8s · ↓ ) + + ❯ Run the shell command `sleep 8`, then reply exactly DONE. + ◉ xhigh · /effort +────────────────────────────────────────────────────────────────────────────────────────────────────────── +❯ Press up to edit queued messages +────────────────────────────────────────────────────────────────────────────────────────────────────────── + [Fable 5 | XXX] ░░░░░░░░░░ 4% | claude-main | 1 CLAUDE.md | 1 MCPs | 3 hooks | 5h: XX% (XXm) | ⏱️  <1m + ✓ Bash ×1 + ⏸ manual mode on · ← for agents + /rc \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.metadata.json new file mode 100644 index 00000000..a6c9f6bb --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.metadata.json @@ -0,0 +1,17 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:29:48Z", + "cli_version": "2.1.223", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 106, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "account and organization identifiers replaced with synthetic values", + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.txt new file mode 100644 index 00000000..cb4c7398 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.txt @@ -0,0 +1,38 @@ +❯ Run the shell command `touch permission-probe.txt` now. Do not explain. + + Ran 1 shell command + +⏺ 好的喵～ permission-probe.txt 已创建。 + +✻ Baked for 10s + +❯ Run the shell command `sleep 8`, then reply exactly DONE. + + Ran 1 shell command + +⏺ DONE + +✻ Brewed for 19s + +❯ Launch exactly one general-purpose subagent. Have it run sleep 8 and then return DONE. Wait for it + before responding. + +⏺ Agent(Run sleep 8, return DONE) + ⎿  Initializing… + (ctrl+b to run in background) + + + +✢ Slithering… (8s · ↓ ) + ⎿  Tip: Use ctrl+v to paste images from your clipboard + +────────────────────────────────────────────────────────────────────────────────────────────────────────── +❯ +────────────────────────────────────────────────────────────────────────────────────────────────────────── + [Fable 5 | XXX] ░░░░░░░░░░ 4% | claude-main | 1 CLAUDE.md | 1 MCPs | 3 hooks | 5h: XX% (XXm) | ⏱️  2m + ◐ Agent | ✓ Bash ×2 + ⏸ manual mode on · ← for agents + /rc + + ⏺ main + ◯ general-purpose Run sleep 8, return DONE 4s \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/command-permission.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/command-permission.metadata.json new file mode 100644 index 00000000..94340200 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/command-permission.metadata.json @@ -0,0 +1,15 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:26:33Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 110, + "rows": 52 + }, + "redactions": [ + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/command-permission.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/command-permission.txt new file mode 100644 index 00000000..28afba49 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/command-permission.txt @@ -0,0 +1,33 @@ +╭────────────────────────────────────────────╮ +│ >_ OpenAI Codex (v0.146.1) │ +│ │ +│ model: gpt-5.6-sol /model to change │ +│ directory: /private/tmp/…/codex-permission │ +╰────────────────────────────────────────────╯ + + Tip: New Use /fast to enable our fastest inference with increased plan usage. + +• + + +› Run touch permission-probe.txt using the shell now. + + +• I’ll run that command in the workspace. + +• Running touch permission-probe.txt + + + Would you like to run the following command? + + Environment: local + + Reason: Do you want to allow creating permission-probe.txt in the workspace? + + $ touch permission-probe.txt + +› 1. Yes, proceed (y) + 2. Yes, and don't ask again for commands that start with `touch permission-probe.txt` (p) + 3. No, and tell Codex what to do differently (esc) + + Press enter to confirm or esc to cancel \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/directory-trust.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/directory-trust.metadata.json new file mode 100644 index 00000000..def11926 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/directory-trust.metadata.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:23:05Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 132, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/directory-trust.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/directory-trust.txt new file mode 100644 index 00000000..b5967110 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/directory-trust.txt @@ -0,0 +1,9 @@ +> You are in /private/tmp/prowl-fixture-workspace-live-00000/codex-main + + Do you trust the contents of this directory? Working with untrusted contents comes with higher risk of prompt injection. Trusting + the directory allows project-local config, hooks, and exec policies to load. + +› 1. Yes, continue + 2. No, quit + + Press enter to continue \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/hook-review.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/hook-review.metadata.json new file mode 100644 index 00000000..2bca6421 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/hook-review.metadata.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:23:53Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 127, + "rows": 52 + }, + "redactions": [ + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/hook-review.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/hook-review.txt new file mode 100644 index 00000000..4e2ba137 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/hook-review.txt @@ -0,0 +1,10 @@ + + Hooks need review + 1 hook is new or changed. + Hooks can run outside the sandbox after you trust them. + +› 1. Review hooks + 2. Trust all and continue + 3. Continue without trusting (hooks won't run) + + Press enter to confirm or esc to go back \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/sign-in-selection.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/sign-in-selection.metadata.json new file mode 100644 index 00000000..ddcc9a2c --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/sign-in-selection.metadata.json @@ -0,0 +1,14 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:24:46Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 115, + "rows": 52 + }, + "redactions": [ + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/sign-in-selection.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/sign-in-selection.txt new file mode 100644 index 00000000..5fb1c334 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/blocked/sign-in-selection.txt @@ -0,0 +1,31 @@ +             *'`+*+\~/_*, +            ^_,||/~~-~+\,, +           |__/\|;_.\,''\\, +           / ;||"|^  /_/|/ +          |` '|*~//\   `_"| +          \  ~*"*||~|*   |/, +          "  ||\+/+||-_ .\|| +          "  ~\ \\|;~~+\+;|| +          |  ,|\,|_/_*___|*` +           , "|||||""!\,"\|` +           \`',\,*"  "",// +            |' |||~*,:,/|/` +             ;`**/|+;_!//' +              *, _*\_,;* + + + Welcome to Codex, OpenAI's command-line coding agent + + Sign in with ChatGPT to use Codex as part of your paid plan + or connect an API key for usage-based billing + +> 1. Sign in with ChatGPT + Usage included with Plus, Pro, Business, and Enterprise plans + + 2. Sign in with Device Code + Sign in from another device with a one-time code + + 3. Provide your own API key + Pay for what you use + + Press enter to continue \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/composer.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/composer.metadata.json new file mode 100644 index 00000000..4d3091bc --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/composer.metadata.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:24:54Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 132, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/composer.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/composer.txt new file mode 100644 index 00000000..8419208d --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/composer.txt @@ -0,0 +1,16 @@ +╭─────────────────────────────────────────────────────╮ +│ >_ OpenAI Codex (v0.146.1) │ +│ │ +│ model: gpt-5.6-terra xhigh /model to change │ +│ directory: /private/tmp/…/codex-main │ +│ permissions: YOLO mode │ +╰─────────────────────────────────────────────────────╯ + + Tip: New Use /fast to enable our fastest inference with increased plan usage. + +• + + +› Run /review on my current changes + + gpt-5.6-terra xhigh · Context 0% used · weekly XX% left · /private/tmp/prowl-fixture-workspace-live-00000/codex-main \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.metadata.json new file mode 100644 index 00000000..96fcc8b7 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.metadata.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:27:00Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 132, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.txt new file mode 100644 index 00000000..ba7f2982 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.txt @@ -0,0 +1,41 @@ +│ >_ OpenAI Codex (v0.146.1) │ +│ │ +│ model: gpt-5.6-terra xhigh /model to change │ +│ directory: /private/tmp/…/codex-main │ +│ permissions: YOLO mode │ +╰─────────────────────────────────────────────────────╯ + + Tip: New Use /fast to enable our fastest inference with increased plan usage. + +• + + +› Run a shell command that sleeps for 8 seconds, then reply exactly DONE. + + +• Starting the 8-second wait now喵～ + +• Ran sleep 8 + └ (no output) + +────────────────────────────────────────────────────────────────────────────────────────────────────────────── + +• DONE + +────────────────────────────────────────────────────────────────────────────────────────────────────────────── + + +› Reply with exactly these four lines and nothing else: + Do you trust the contents of this directory? + › 1. Yes, continue + 2. No, quit + Press enter to continue + + +• Do you trust the contents of this directory? + › 1. Yes, continue + + +› Run /review on my current changes + + gpt-5.6-terra xhigh · Context 4% used · weekly XX% left · XX.XK used · /private/tmp/prowl-fixture-workspace… \ No newline at end of file diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/working/foreground-footer.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/working/foreground-footer.metadata.json new file mode 100644 index 00000000..e6251102 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/working/foreground-footer.metadata.json @@ -0,0 +1,16 @@ +{ + "schema_version": 1, + "captured_at": "2026-08-06T16:25:06Z", + "cli_version": "0.146.1", + "capture_source": "prowl-read-detection", + "terminal": { + "columns": 132, + "rows": 52 + }, + "redactions": [ + "temporary capture root replaced with a synthetic equal-width path", + "usage balances and plan labels replaced with placeholders", + "terminal right-edge padding removed without changing line wrapping" + ], + "issue": null +} diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/working/foreground-footer.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/working/foreground-footer.txt new file mode 100644 index 00000000..96d43a60 --- /dev/null +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/working/foreground-footer.txt @@ -0,0 +1,22 @@ +╭─────────────────────────────────────────────────────╮ +│ >_ OpenAI Codex (v0.146.1) │ +│ │ +│ model: gpt-5.6-terra xhigh /model to change │ +│ directory: /private/tmp/…/codex-main │ +│ permissions: YOLO mode │ +╰─────────────────────────────────────────────────────╯ + + Tip: New Use /fast to enable our fastest inference with increased plan usage. + +• + + +› Run a shell command that sleeps for 8 seconds, then reply exactly DONE. + + +• Working (0s • esc to interrupt) + + +› Run /review on my current changes + + gpt-5.6-terra xhigh · Context 0% used · weekly XX% left · /private/tmp/prowl-fixture-workspace-live-00000/codex-main \ No newline at end of file diff --git a/supacodeTests/ScreenHeuristicsBenchmarks.swift b/supacodeTests/ScreenHeuristicsBenchmarks.swift new file mode 100644 index 00000000..fdfa230c --- /dev/null +++ b/supacodeTests/ScreenHeuristicsBenchmarks.swift @@ -0,0 +1,69 @@ +import Foundation +import Testing + +@testable import supacode + +extension PerformanceBenchmarks { + /// Tracks absolute changed-screen classification cost over the same sanitized + /// Claude/Codex capture corpus used by the end-to-end regression test. + @Suite + struct ScreenHeuristicsBenchmarks { + @Test func capturedCorpusChangedFrameCost() throws { + let fixtures = try Self.loadFixtures() + let repeats = BenchmarkMeasurement.isFullMode ? 2_000 : 20 + var checksum = 0 + + let median = BenchmarkMeasurement.repeatedMedian { + var iterationChecksum = 0 + for _ in 0.. 0) + BenchmarkMeasurement.reportAbsolute( + suite: "ScreenHeuristics", + name: "captured-claude-codex-corpus", + median: median + ) + } + + private static func loadFixtures() throws -> [(agent: DetectedAgent, text: String)] { + let root = URL(fileURLWithPath: #filePath) + .deletingLastPathComponent() + .appending(path: "Fixtures/AgentScreenDetection", directoryHint: .isDirectory) + guard + let enumerator = FileManager.default.enumerator( + at: root, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles] + ) + else { + throw ScreenHeuristicsBenchmarkError.fixtureRootUnavailable + } + + return + try enumerator + .compactMap { $0 as? URL } + .filter { $0.pathExtension == "txt" } + .sorted { $0.path() < $1.path() } + .map { url in + let relativePath = String(url.path().dropFirst(root.path().count)) + guard let runtime = relativePath.split(separator: "/").first, + let agent = DetectedAgent(rawValue: String(runtime)) + else { + throw ScreenHeuristicsBenchmarkError.invalidFixturePath(relativePath) + } + return (agent, try String(contentsOf: url, encoding: .utf8)) + } + } + } +} + +private enum ScreenHeuristicsBenchmarkError: Error { + case fixtureRootUnavailable + case invalidFixturePath(String) +} -- 2.51.2 From b5420f175d06399658856e30cbd5653e5ff7b78a Mon Sep 17 00:00:00 2001 From: onevcat Date: Fri, 7 Aug 2026 01:57:00 +0900 Subject: [PATCH 2/6] Bound screen corpus benchmark workload --- supacodeTests/BenchmarkMeasurement.swift | 11 +++++++++-- supacodeTests/ScreenHeuristicsBenchmarks.swift | 5 +++-- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/supacodeTests/BenchmarkMeasurement.swift b/supacodeTests/BenchmarkMeasurement.swift index f1e73999..9cf42999 100644 --- a/supacodeTests/BenchmarkMeasurement.swift +++ b/supacodeTests/BenchmarkMeasurement.swift @@ -90,14 +90,20 @@ nonisolated enum BenchmarkMeasurement { milliseconds(medians.reference) / milliseconds(medians.shipped) } - static func reportAbsolute(suite: String, name: String, median: Duration) { + static func reportAbsolute( + suite: String, + name: String, + median: Duration, + normalizingBy workloadCount: Int = 1 + ) { guard isFullMode else { return } let environment = ProcessInfo.processInfo.environment let record = AbsoluteRecord( date: Date.now.ISO8601Format(), suite: suite, name: name, - medianMilliseconds: milliseconds(median), + medianMilliseconds: milliseconds(median) / Double(workloadCount), + normalizationDivisor: workloadCount, iterations: iterations, gitSHA: environment["PROWL_BENCH_GIT_SHA"] ) @@ -146,6 +152,7 @@ nonisolated enum BenchmarkMeasurement { let suite: String let name: String let medianMilliseconds: Double + let normalizationDivisor: Int let iterations: Int let gitSHA: String? } diff --git a/supacodeTests/ScreenHeuristicsBenchmarks.swift b/supacodeTests/ScreenHeuristicsBenchmarks.swift index fdfa230c..50c68311 100644 --- a/supacodeTests/ScreenHeuristicsBenchmarks.swift +++ b/supacodeTests/ScreenHeuristicsBenchmarks.swift @@ -10,7 +10,7 @@ extension PerformanceBenchmarks { struct ScreenHeuristicsBenchmarks { @Test func capturedCorpusChangedFrameCost() throws { let fixtures = try Self.loadFixtures() - let repeats = BenchmarkMeasurement.isFullMode ? 2_000 : 20 + let repeats = BenchmarkMeasurement.isFullMode ? 20 : 2 var checksum = 0 let median = BenchmarkMeasurement.repeatedMedian { @@ -27,7 +27,8 @@ extension PerformanceBenchmarks { BenchmarkMeasurement.reportAbsolute( suite: "ScreenHeuristics", name: "captured-claude-codex-corpus", - median: median + median: median, + normalizingBy: repeats ) } -- 2.51.2 From da900ab649c3f6fde7dd2af967c9fa59dbfb613b Mon Sep 17 00:00:00 2001 From: onevcat Date: Fri, 7 Aug 2026 02:04:29 +0900 Subject: [PATCH 3/6] Record captured screen corpus baseline --- .../030-agent-status-detection/000-plan.md | 3 + .../009-captured-screen-fixture-corpus.md | 106 ++++++++++++++++++ 2 files changed, 109 insertions(+) create mode 100644 docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md diff --git a/docs-ai/030-agent-status-detection/000-plan.md b/docs-ai/030-agent-status-detection/000-plan.md index 1eb543e2..e93c3255 100644 --- a/docs-ai/030-agent-status-detection/000-plan.md +++ b/docs-ai/030-agent-status-detection/000-plan.md @@ -128,3 +128,6 @@ the failed attempt to extend the first signal to plain commands is - Updated 2026-08-06: `prowl read --source detection` now captures the exact active-screen buffer used by production state detection — see [008-detector-faithful-cli-capture.md](008-detector-faithful-cli-capture.md) +- Updated 2026-08-07: a 15-screen Claude/Codex captured corpus, executable quarantine, + provenance validation, and Release classifier baseline are in place — see + [009-captured-screen-fixture-corpus.md](009-captured-screen-fixture-corpus.md) diff --git a/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md new file mode 100644 index 00000000..df3c24c3 --- /dev/null +++ b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md @@ -0,0 +1,106 @@ +# 030.009 — Captured Screen Fixture Corpus and Baseline + +| | | +| --- | --- | +| **Status** | Implemented | +| **Anchor date** | 2026-08-07 | +| **Primary PR** | TBD | +| **Plan** | [007-screen-profile-migration-plan.md](007-screen-profile-migration-plan.md), Phase 2 | +| **Prerequisite** | [008-detector-faithful-cli-capture.md](008-detector-faithful-cli-capture.md) | + +## Context + +Inline screen literals test focused predicates well but cannot prove their CLI version, +capture source, terminal geometry, or redaction history. Phase 2 adds a capture-derived +regression corpus before changing production classification behavior. + +## Corpus + +The committed corpus contains 15 sanitized `prowl read --source detection` captures: + +| Runtime | Version | Blocked | Working | Idle | Quarantine | +| --- | --- | --- | --- | --- | --- | +| Codex | 0.146.1 | directory trust, hook review, sign-in selection, command permission | foreground footer | composer, quoted directory-trust transcript | — | +| Claude | 2.1.223 | workspace trust, command permission | foreground spinner, active subagent, backgrounded subagent | composer, quoted permission transcript | history-search viewer | + +All captures came from authenticated current CLIs in an isolated Debug Prowl instance and +private temporary workspaces. The terminal was 52 rows high and 106–132 columns wide. +Raw captures stayed under ignored `.local/agent-screen-captures/`; copied authentication +and state databases were removed after capture. + +Each `.txt` is the canonical production tail: it starts at the 24th non-empty line from +the bottom while preserving internal/trailing blank terminal rows. Same-basename metadata +records schema version, exact CLI version, UTC capture timestamp from the raw file, +`prowl-read-detection` source, terminal geometry, and an explicit redaction summary. +Paths, account/organization identifiers, usage balances, prompt history, and session IDs +were replaced with synthetic placeholders. Right-edge terminal padding was removed without +changing leading columns or wraps. A corpus-specific `.gitattributes` rule preserves +intentional trailing blank screen rows without weakening whitespace checks elsewhere. + +Reconstructed and synthetic screens remain inline in `ScreenHeuristicsTests`; no existing +literal was relabeled as a captured/versioned fixture. + +## Harness and quarantine + +`AgentScreenFixtureCorpusTests`: + +- discovers fixtures relative to `#filePath` and asserts at least one ran; +- validates path layout, runtime/state tokens, metadata schema/source/version/timestamp, + terminal geometry, redaction summary, matching sidecars, and no orphan metadata; +- verifies each committed text is already the canonical 24-non-empty-line tail; +- runs every normal fixture through the current detector; +- asserts initial Claude/Codex blocked/working/idle lifecycle coverage; +- treats `known-misdetection///...` as an executable quarantine. + +Fresh capture exposed one current drift: Claude 2.1.223 history search now renders +`Search prompts · everywhere`, `⌕ Filter history…`, and `↑/↓ to nav ...` instead of the +older viewer hints. Product semantics require `.unknown`, but the current detector returns +`.idle`. The capture is committed under `known-misdetection/unknown/idle/` with issue #676. +Phase 2 asserts current behavior and does not hide a production fix inside test +infrastructure; the Claude profile phase must either promote it with a capture-backed fix +or leave the quarantine explicit. + +The synchronized Xcode test group treats `Fixtures/` as an explicit folder so nested +runtime/version filenames retain hierarchy rather than flattening into duplicate resource +outputs. Tests intentionally read the source checkout; bundle resources are not an +alternate classifier path. + +## Release baseline + +`ScreenHeuristicsBenchmarks` classifies the complete 15-fixture corpus as changed-screen +input. `make bench` runs 20 corpora per timed sample, reports the median normalized to one +corpus, and applies no absolute CI threshold. + +Baseline at `b5420f17` on this M-series host: + +- complete 15-fixture corpus: **3.146 ms median**; +- arithmetic per-fixture cost: **0.210 ms**; +- 15 Release timing samples; benchmark suite completed in 6.16 s after the initial build. + +The first attempted 2,000-corpus sample was deliberately aborted after proving unsuitable +for a per-PR gate; the committed workload uses 20 and remains well above clock noise. +Future Codex/Claude profile PRs compare the same absolute metric and preserve the scan +cache, so this measures changed-frame classifier cost rather than steady-state polling. + +## Validation + +- Corpus tests: 2 passed; 15 screen fixtures and 15 metadata sidecars executed. +- Screen benchmark Debug smoke: 1 passed. +- Full app suite: xcsift reported 2,275 passed; xcresult independently verified 2,278 + tests and zero failures. +- `make bench`: 6 benchmark tests passed; four existing Ghostty symbol-index warnings, + zero errors/failures; screen corpus record appended successfully. +- `make check` and `make build-app` passed. +- Privacy audit found no user email/account, home path, OAuth URL/state, usage number, + copied credential, or unredacted high-entropy session identifier. The only user-related + URL is the intentional public #676 quarantine link. +- Metadata JSON parsed, fixture hashes were all distinct, every fixture held at most 24 + non-empty lines, raw staging was confirmed ignored, and `git diff --check` passed with + the scoped fixture whitespace attribute. + +## Result + +Phase 2's exit condition is met. Current Claude/Codex shapes are versioned and explainable, +newly observed drift is visible rather than dropped, and profile migrations now have both +a behavior corpus and a reproducible Release performance baseline. Production detector +behavior remains unchanged. -- 2.51.2 From 0499e4c13c3cc7825060832c28be1ae649b873c5 Mon Sep 17 00:00:00 2001 From: onevcat Date: Fri, 7 Aug 2026 02:05:02 +0900 Subject: [PATCH 4/6] Link screen corpus implementation record --- .../009-captured-screen-fixture-corpus.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md index df3c24c3..b49164e4 100644 --- a/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md +++ b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md @@ -4,7 +4,7 @@ | --- | --- | | **Status** | Implemented | | **Anchor date** | 2026-08-07 | -| **Primary PR** | TBD | +| **Primary PR** | [#685](https://github.com/onevcat/Prowl/pull/685) | | **Plan** | [007-screen-profile-migration-plan.md](007-screen-profile-migration-plan.md), Phase 2 | | **Prerequisite** | [008-detector-faithful-cli-capture.md](008-detector-faithful-cli-capture.md) | -- 2.51.2 From cf6e6f04d7df4096c85962c9550f21d9dbb27a41 Mon Sep 17 00:00:00 2001 From: onevcat Date: Fri, 7 Aug 2026 02:32:05 +0900 Subject: [PATCH 5/6] Harden screen fixture corpus validation --- .../009-captured-screen-fixture-corpus.md | 25 ++-- .../AgentDetection/ScreenHeuristics.swift | 8 +- .../AgentScreenFixtureCorpusTests.swift | 132 +++++++++++++----- .../Fixtures/AgentScreenDetection/README.md | 17 ++- .../idle/quoted-permission.metadata.json | 1 + .../claude/2.1.223/idle/quoted-permission.txt | 2 +- .../676-history-search-viewer.metadata.json | 1 + .../idle/676-history-search-viewer.txt | 2 +- .../backgrounded-subagent.metadata.json | 1 + .../2.1.223/working/backgrounded-subagent.txt | 2 +- .../working/subagent-active.metadata.json | 1 + .../2.1.223/working/subagent-active.txt | 2 +- .../idle/quoted-directory-trust.metadata.json | 1 + .../0.146.1/idle/quoted-directory-trust.txt | 2 +- .../ScreenHeuristicsBenchmarks.swift | 34 +---- 15 files changed, 138 insertions(+), 93 deletions(-) diff --git a/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md index b49164e4..177c972e 100644 --- a/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md +++ b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md @@ -28,13 +28,17 @@ private temporary workspaces. The terminal was 52 rows high and 106–132 column Raw captures stayed under ignored `.local/agent-screen-captures/`; copied authentication and state databases were removed after capture. -Each `.txt` is the canonical production tail: it starts at the 24th non-empty line from -the bottom while preserving internal/trailing blank terminal rows. Same-basename metadata +Each `.txt` is the canonical production tail produced by the shared +`agentDetectionRecentText` helper: it starts at the 24th non-empty line from the bottom +when at least 24 exist, otherwise retaining the whole screen, while preserving +internal/trailing blank terminal rows. Same-basename metadata records schema version, exact CLI version, UTC capture timestamp from the raw file, `prowl-read-detection` source, terminal geometry, and an explicit redaction summary. -Paths, account/organization identifiers, usage balances, prompt history, and session IDs -were replaced with synthetic placeholders. Right-edge terminal padding was removed without -changing leading columns or wraps. A corpus-specific `.gitattributes` rule preserves +Paths, account/organization identifiers, usage balances, real prompt history, private +persona output, and session IDs were replaced with synthetic placeholders. Purpose-built +probe prompts/output remain only where they provide conversational regression structure. +Right-edge terminal padding was removed without changing leading columns or wraps. A +corpus-specific `.gitattributes` rule preserves intentional trailing blank screen rows without weakening whitespace checks elsewhere. Reconstructed and synthetic screens remain inline in `ScreenHeuristicsTests`; no existing @@ -46,8 +50,10 @@ literal was relabeled as a captured/versioned fixture. - discovers fixtures relative to `#filePath` and asserts at least one ran; - validates path layout, runtime/state tokens, metadata schema/source/version/timestamp, - terminal geometry, redaction summary, matching sidecars, and no orphan metadata; -- verifies each committed text is already the canonical 24-non-empty-line tail; + explicit issue key, terminal geometry, redaction summary, matching sidecars, no orphan + metadata, and rejects every unexpected regular file; +- verifies each committed text is already the canonical detector tail through the same + production helper used by `detectState(in:)`; - runs every normal fixture through the current detector; - asserts initial Claude/Codex blocked/working/idle lifecycle coverage; - treats `known-misdetection///...` as an executable quarantine. @@ -84,9 +90,10 @@ cache, so this measures changed-frame classifier cost rather than steady-state p ## Validation -- Corpus tests: 2 passed; 15 screen fixtures and 15 metadata sidecars executed. +- Corpus tests: 4 passed; 15 screen fixtures and 15 metadata sidecars executed, plus + unexpected-file and missing-issue-key failure paths. - Screen benchmark Debug smoke: 1 passed. -- Full app suite: xcsift reported 2,275 passed; xcresult independently verified 2,278 +- Full app suite: xcsift reported 2,278 passed; xcresult independently verified 2,280 tests and zero failures. - `make bench`: 6 benchmark tests passed; four existing Ghostty symbol-index warnings, zero errors/failures; screen corpus record appended successfully. diff --git a/supacode/Infrastructure/AgentDetection/ScreenHeuristics.swift b/supacode/Infrastructure/AgentDetection/ScreenHeuristics.swift index 54506e16..b57e7740 100644 --- a/supacode/Infrastructure/AgentDetection/ScreenHeuristics.swift +++ b/supacode/Infrastructure/AgentDetection/ScreenHeuristics.swift @@ -1,10 +1,10 @@ import Foundation -nonisolated private let agentDetectionRecentLineLimit = 24 +nonisolated let agentDetectionRecentLineLimit = 24 extension DetectedAgent { nonisolated func detectState(in screen: String) -> AgentRawState { - let screen = recentLines(screen, limit: agentDetectionRecentLineLimit) + let screen = agentDetectionRecentText(screen) switch self { case .pi: return detectPi(screen) @@ -40,6 +40,10 @@ extension DetectedAgent { } } +nonisolated func agentDetectionRecentText(_ content: String) -> String { + recentLines(content, limit: agentDetectionRecentLineLimit) +} + nonisolated private func recentLines(_ content: String, limit: Int) -> String { let lines = content.split(separator: "\n", omittingEmptySubsequences: false).map(String.init) var remainingNonBlankLines = limit diff --git a/supacodeTests/AgentScreenFixtureCorpusTests.swift b/supacodeTests/AgentScreenFixtureCorpusTests.swift index b0df437a..1d189ac4 100644 --- a/supacodeTests/AgentScreenFixtureCorpusTests.swift +++ b/supacodeTests/AgentScreenFixtureCorpusTests.swift @@ -16,15 +16,49 @@ struct AgentScreenFixtureCorpusTests { #expect(states.contains(.working)) #expect(states.contains(.idle)) } + } - #expect( - fixtures.contains { - $0.agent == .claude - && $0.isQuarantined - && $0.expectedState == .unknown - && $0.currentState == .idle - } + @Test func corpusRejectsUnexpectedRegularFiles() throws { + let temporaryRoot = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString, directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: temporaryRoot, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + try Data().write(to: temporaryRoot.appending(path: "README.md")) + try Data().write(to: temporaryRoot.appending(path: "raw-capture.json")) + + #expect(throws: CorpusError.self) { + try AgentScreenFixtureCorpus.load(from: temporaryRoot) + } + } + + @Test func corpusRequiresExplicitIssueMetadataKey() throws { + let temporaryRoot = FileManager.default.temporaryDirectory + .appending(path: UUID().uuidString, directoryHint: .isDirectory) + let fixtureDirectory = temporaryRoot.appending(path: "claude/1.0/idle", directoryHint: .isDirectory) + try FileManager.default.createDirectory(at: fixtureDirectory, withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: temporaryRoot) } + try "idle".write(to: fixtureDirectory.appending(path: "composer.txt"), atomically: true, encoding: .utf8) + try """ + { + "schema_version": 1, + "captured_at": "2026-08-07T00:00:00Z", + "cli_version": "1.0", + "capture_source": "prowl-read-detection", + "terminal": { "columns": 80, "rows": 24 }, + "redactions": ["none required"] + } + """.write( + to: fixtureDirectory.appending(path: "composer.metadata.json"), + atomically: true, + encoding: .utf8 ) + + do { + _ = try AgentScreenFixtureCorpus.load(from: temporaryRoot) + } catch is DecodingError { + return + } + Issue.record("Fixture metadata without an explicit issue key was accepted.") } @Test func capturedFixturesMatchCurrentDetector() throws { @@ -59,34 +93,48 @@ struct AgentScreenFixtureCorpusTests { } } -private enum AgentScreenFixtureCorpus { - static let recentNonEmptyLineLimit = 24 +enum AgentScreenFixtureCorpus { static let root = URL(fileURLWithPath: #filePath) .deletingLastPathComponent() .appending(path: "Fixtures/AgentScreenDetection", directoryHint: .isDirectory) - static func load() throws -> [AgentScreenFixture] { + static func load(from fixtureRoot: URL = root) throws -> [AgentScreenFixture] { + let fixtureRoot = fixtureRoot.standardizedFileURL.resolvingSymlinksInPath() let fileManager = FileManager.default guard let enumerator = fileManager.enumerator( - at: root, + at: fixtureRoot, includingPropertiesForKeys: [.isRegularFileKey], options: [.skipsHiddenFiles] ) else { - throw CorpusError("Fixture root is unavailable: \(root.path())") + throw CorpusError("Fixture root is unavailable: \(fixtureRoot.path(percentEncoded: false))") + } + + let urls = enumerator.compactMap { ($0 as? URL)?.standardizedFileURL.resolvingSymlinksInPath() } + let regularFileURLs = try urls.filter { + try $0.resourceValues(forKeys: [.isRegularFileKey]).isRegularFile == true + } + let readmeURL = fixtureRoot.appending(path: "README.md") + let unexpectedURLs = regularFileURLs.filter { url in + url != readmeURL + && url.pathExtension != "txt" + && !url.lastPathComponent.hasSuffix(".metadata.json") + } + guard unexpectedURLs.isEmpty else { + let unexpectedPaths = unexpectedURLs.map { relativePath(for: $0, root: fixtureRoot) }.sorted() + throw CorpusError("Unexpected files in fixture corpus: \(unexpectedPaths)") } - let urls = enumerator.compactMap { $0 as? URL } let screenURLs = - urls + regularFileURLs .filter { $0.pathExtension == "txt" } .sorted { $0.path() < $1.path() } - let metadataURLs = Set(urls.filter { $0.lastPathComponent.hasSuffix(".metadata.json") }) + let metadataURLs = Set(regularFileURLs.filter { $0.lastPathComponent.hasSuffix(".metadata.json") }) var consumedMetadataURLs: Set = [] let fixtures = try screenURLs.map { screenURL in - let fixture = try loadFixture(at: screenURL) + let fixture = try loadFixture(at: screenURL, root: fixtureRoot) consumedMetadataURLs.insert(fixture.metadataURL) return fixture } @@ -101,28 +149,16 @@ private enum AgentScreenFixtureCorpus { } static func canonicalTail(_ content: String) -> String { - let lines = content.split(separator: "\n", omittingEmptySubsequences: false).map(String.init) - var remainingNonEmptyLines = recentNonEmptyLineLimit - var startIndex = lines.startIndex - - for index in lines.indices.reversed() { - guard !lines[index].trimmingCharacters(in: .whitespaces).isEmpty else { continue } - remainingNonEmptyLines -= 1 - if remainingNonEmptyLines == 0 { - startIndex = index - break - } - } - return lines[startIndex...].joined(separator: "\n") + agentDetectionRecentText(content) } - private static func loadFixture(at screenURL: URL) throws -> AgentScreenFixture { - let relativePath = relativePath(for: screenURL) + private static func loadFixture(at screenURL: URL, root: URL) throws -> AgentScreenFixture { + let relativePath = relativePath(for: screenURL, root: root) let components = relativePath.split(separator: "/").map(String.init) let layout = try FixtureLayout(components: components, relativePath: relativePath) let metadataURL = screenURL.deletingPathExtension().appendingPathExtension("metadata.json") - guard FileManager.default.fileExists(atPath: metadataURL.path()) else { + guard FileManager.default.fileExists(atPath: metadataURL.path(percentEncoded: false)) else { throw CorpusError("Missing metadata for \(relativePath)") } @@ -169,13 +205,15 @@ private enum AgentScreenFixtureCorpus { } } - private static func relativePath(for url: URL) -> String { - let path = String(url.path().dropFirst(root.path().count)) + private static func relativePath(for url: URL, root: URL) -> String { + let rootPath = root.standardizedFileURL.resolvingSymlinksInPath().path(percentEncoded: false) + let urlPath = url.standardizedFileURL.resolvingSymlinksInPath().path(percentEncoded: false) + let path = String(urlPath.dropFirst(rootPath.count)) return path.hasPrefix("/") ? String(path.dropFirst()) : path } } -private struct AgentScreenFixture { +struct AgentScreenFixture { let relativePath: String let metadataURL: URL let agent: DetectedAgent @@ -186,7 +224,7 @@ private struct AgentScreenFixture { let metadata: AgentScreenFixtureMetadata } -private struct AgentScreenFixtureMetadata: Decodable { +struct AgentScreenFixtureMetadata: Decodable { struct Terminal: Decodable { let columns: Int let rows: Int @@ -209,6 +247,26 @@ private struct AgentScreenFixtureMetadata: Decodable { case redactions case issue } + + init(from decoder: Decoder) throws { + let container = try decoder.container(keyedBy: CodingKeys.self) + guard container.contains(.issue) else { + throw DecodingError.keyNotFound( + CodingKeys.issue, + DecodingError.Context( + codingPath: container.codingPath, + debugDescription: "Fixture metadata must include issue as a URL or explicit null." + ) + ) + } + self.schemaVersion = try container.decode(Int.self, forKey: .schemaVersion) + self.capturedAt = try container.decode(String.self, forKey: .capturedAt) + self.cliVersion = try container.decode(String.self, forKey: .cliVersion) + self.captureSource = try container.decode(String.self, forKey: .captureSource) + self.terminal = try container.decode(Terminal.self, forKey: .terminal) + self.redactions = try container.decode([String].self, forKey: .redactions) + self.issue = try container.decodeIfPresent(String.self, forKey: .issue) + } } private struct FixtureLayout { @@ -249,7 +307,7 @@ private struct FixtureLayout { } } -private struct CorpusError: Error, CustomStringConvertible { +struct CorpusError: Error, CustomStringConvertible { let description: String init(_ description: String) { diff --git a/supacodeTests/Fixtures/AgentScreenDetection/README.md b/supacodeTests/Fixtures/AgentScreenDetection/README.md index cd7e3b68..01dfd8a3 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/README.md +++ b/supacodeTests/Fixtures/AgentScreenDetection/README.md @@ -32,11 +32,13 @@ derived from `idle + unseen` and is never a fixture state. 4. Keep the raw response under the ignored `.local/agent-screen-captures/` directory. 5. Record exact CLI version, capture timestamp, terminal rows/columns, and the redaction summary in a same-basename metadata file. -6. Reduce the capture to the exact canonical tail the detector receives: start at the - 24th non-empty line from the bottom, retaining blank lines and trailing screen rows - inside that window. -7. Redact paths, repositories, account identifiers, prompts, and model output without - changing runtime chrome, line ordering, markers, wrapping, or blank-line boundaries. +6. Reduce the capture with the production `agentDetectionRecentText` helper: start at + the 24th non-empty line from the bottom when at least 24 exist; otherwise retain the + whole screen. Keep blank lines and trailing screen rows inside that window. +7. Redact paths, repositories, account identifiers, and all real-session prompts/model + output without changing runtime chrome, line ordering, markers, wrapping, or blank-line + boundaries. Deliberately scripted probe interactions from disposable workspaces may + remain verbatim and are preferred for conversational fixtures. 8. Run `AgentScreenFixtureCorpusTests` before committing. The loader resolves this tree through `#filePath`, so tests intentionally run from a @@ -77,7 +79,8 @@ fixture is promoted to the normal expected-state path. - Keep the newest verified fixture for each scenario/UI shape. - Retain an older CLI version only when its distinct shape remains intentionally supported. - Remove byte-equivalent history. -- Never commit raw captures, credentials, user prompts, model output, account names, home - paths, or repository names. +- Never commit raw captures, credentials, real-session user prompts/model output, account + names, home paths, or repository names. Purpose-built scripted probes are allowed as + described above; replace any output that exposes private configuration or persona text. - Reconstructed and synthetic screens stay inline in `ScreenHeuristicsTests.swift`; they must not be version-stamped as captured evidence here. diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.metadata.json index 338ec745..6078a59c 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.metadata.json +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.metadata.json @@ -10,6 +10,7 @@ "redactions": [ "usage balances and plan labels replaced with placeholders", "session identifier replaced with a placeholder", + "private persona-styled model output replaced with ", "terminal right-edge padding removed without changing line wrapping" ], "issue": null diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.txt index f7371dd8..386727f2 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.txt +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/idle/quoted-permission.txt @@ -6,7 +6,7 @@ ⏺ Agent "Run sleep 8, return DONE" finished · 25s -⏺ Subagent 已跑完喵～ 它执行了 sleep 8，返回结果是：DONE +⏺ ✻ Brewed for 32s diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.metadata.json index 43979cbc..0ff9c7eb 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.metadata.json +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.metadata.json @@ -11,6 +11,7 @@ "temporary capture root replaced with a synthetic equal-width path", "account and organization identifiers replaced with synthetic values", "usage balances and plan labels replaced with placeholders", + "private persona-styled model output replaced with ", "prompt history entries replaced with ", "terminal right-edge padding removed without changing line wrapping" ], diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.txt index a77db369..53ada3e8 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.txt +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/known-misdetection/unknown/idle/676-history-search-viewer.txt @@ -6,7 +6,7 @@ Ran 1 shell command -⏺ 好的喵～ permission-probe.txt 已创建。 +⏺ ✻ Baked for 10s diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.metadata.json index 4e555e2c..9d723641 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.metadata.json +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.metadata.json @@ -11,6 +11,7 @@ "temporary capture root replaced with a synthetic equal-width path", "account and organization identifiers replaced with synthetic values", "usage balances and plan labels replaced with placeholders", + "private persona-styled model output replaced with ", "terminal right-edge padding removed without changing line wrapping" ], "issue": null diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.txt index b56349fc..c99053db 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.txt +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/backgrounded-subagent.txt @@ -5,7 +5,7 @@ Ran 1 shell command -⏺ 好的喵～ permission-probe.txt 已创建。 +⏺ ✻ Baked for 10s diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.metadata.json index a6c9f6bb..079d883d 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.metadata.json +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.metadata.json @@ -11,6 +11,7 @@ "temporary capture root replaced with a synthetic equal-width path", "account and organization identifiers replaced with synthetic values", "usage balances and plan labels replaced with placeholders", + "private persona-styled model output replaced with ", "terminal right-edge padding removed without changing line wrapping" ], "issue": null diff --git a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.txt b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.txt index cb4c7398..fad67c5c 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.txt +++ b/supacodeTests/Fixtures/AgentScreenDetection/claude/2.1.223/working/subagent-active.txt @@ -2,7 +2,7 @@ Ran 1 shell command -⏺ 好的喵～ permission-probe.txt 已创建。 +⏺ ✻ Baked for 10s diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.metadata.json b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.metadata.json index 96fcc8b7..e93656cb 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.metadata.json +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.metadata.json @@ -10,6 +10,7 @@ "redactions": [ "temporary capture root replaced with a synthetic equal-width path", "usage balances and plan labels replaced with placeholders", + "private persona-styled model output replaced with ", "terminal right-edge padding removed without changing line wrapping" ], "issue": null diff --git a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.txt b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.txt index ba7f2982..0a95df1f 100644 --- a/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.txt +++ b/supacodeTests/Fixtures/AgentScreenDetection/codex/0.146.1/idle/quoted-directory-trust.txt @@ -13,7 +13,7 @@ › Run a shell command that sleeps for 8 seconds, then reply exactly DONE. -• Starting the 8-second wait now喵～ +• • Ran sleep 8 └ (no output) diff --git a/supacodeTests/ScreenHeuristicsBenchmarks.swift b/supacodeTests/ScreenHeuristicsBenchmarks.swift index 50c68311..825c4b38 100644 --- a/supacodeTests/ScreenHeuristicsBenchmarks.swift +++ b/supacodeTests/ScreenHeuristicsBenchmarks.swift @@ -1,4 +1,3 @@ -import Foundation import Testing @testable import supacode @@ -33,38 +32,7 @@ extension PerformanceBenchmarks { } private static func loadFixtures() throws -> [(agent: DetectedAgent, text: String)] { - let root = URL(fileURLWithPath: #filePath) - .deletingLastPathComponent() - .appending(path: "Fixtures/AgentScreenDetection", directoryHint: .isDirectory) - guard - let enumerator = FileManager.default.enumerator( - at: root, - includingPropertiesForKeys: [.isRegularFileKey], - options: [.skipsHiddenFiles] - ) - else { - throw ScreenHeuristicsBenchmarkError.fixtureRootUnavailable - } - - return - try enumerator - .compactMap { $0 as? URL } - .filter { $0.pathExtension == "txt" } - .sorted { $0.path() < $1.path() } - .map { url in - let relativePath = String(url.path().dropFirst(root.path().count)) - guard let runtime = relativePath.split(separator: "/").first, - let agent = DetectedAgent(rawValue: String(runtime)) - else { - throw ScreenHeuristicsBenchmarkError.invalidFixturePath(relativePath) - } - return (agent, try String(contentsOf: url, encoding: .utf8)) - } + try AgentScreenFixtureCorpus.load().map { ($0.agent, $0.text) } } } } - -private enum ScreenHeuristicsBenchmarkError: Error { - case fixtureRootUnavailable - case invalidFixturePath(String) -} -- 2.51.2 From 255921e427e7beb972af34f52a68f666a1ef8cfb Mon Sep 17 00:00:00 2001 From: onevcat Date: Fri, 7 Aug 2026 02:39:24 +0900 Subject: [PATCH 6/6] Update reviewed screen corpus baseline --- .../009-captured-screen-fixture-corpus.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md index 177c972e..47d0fcb0 100644 --- a/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md +++ b/docs-ai/030-agent-status-detection/009-captured-screen-fixture-corpus.md @@ -77,11 +77,11 @@ alternate classifier path. input. `make bench` runs 20 corpora per timed sample, reports the median normalized to one corpus, and applies no absolute CI threshold. -Baseline at `b5420f17` on this M-series host: +Final reviewed baseline at `cf6e6f04` on this M-series host: -- complete 15-fixture corpus: **3.146 ms median**; -- arithmetic per-fixture cost: **0.210 ms**; -- 15 Release timing samples; benchmark suite completed in 6.16 s after the initial build. +- complete 15-fixture corpus: **3.142 ms median**; +- arithmetic per-fixture cost: **0.209 ms**; +- 15 Release timing samples; benchmark suite completed in 6.17 s on the warm build. The first attempted 2,000-corpus sample was deliberately aborted after proving unsuitable for a per-PR gate; the committed workload uses 20 and remains well above clock noise.