A cbor+zstd log management tool (grep, extract, merge, split, repack, redact)
cbor zstd aperture logs
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555//! Tool-call failure inspection for the `toolfailures` subcommand.//!//! Each record's `capture.requestBody` echoes the full conversation history, so//! a single record contains BOTH the assistant turn that issued tool calls AND//! the `role:"tool"` / `tool_result` messages carrying their results. We pair//! calls to results by id (`tool_calls[].id` <-> `tool_call_id` for OpenAI,//! `tool_use.id` <-> `tool_result.tool_use_id` for Anthropic) and inspect only//! the NEW turn (messages after the last assistant message). That way each//! call/result pair is emitted exactly once, in the record where the result//! first appears — no double-counting as a thread grows.//!//! A result is a FAILURE when ANY of://! - an Anthropic `tool_result` block carries `is_error: true`,//! - a line in the result reports a nonzero exit code, e.g.//! "Command exited with code 1" (shell tools), or//! - the result text starts with one of a known set of error prefixes, e.g.//! "error: HTTP 404", "ENOENT: no such file...", "old_string not found",//! "Permission denied for `edit` (...)".//!//! Prefix matching is ANCHORED to the start of the (left-trimmed) result so it//! can't false-positive on file content that merely mentions "error" — tools//! that return arbitrary content (`read`, `grep`, `glob`) prefix it with line//! numbers or paths, so a file whose first line reads "error: ..." arrives as//! "1: error: ..." and does not match. "No files found" and similar empty-but-//! successful results are deliberately NOT failures: the tool ran fine.
use crate::format;use regex::{Regex, RegexBuilder};use serde_json::Value as Json;use std::collections::BTreeMap;
/// Exit-code phrasing seen in real captures, e.g. "Command exited with code 1"./// Group 1 captures the numeric code. Matched case-insensitively. Like the/// error prefixes, ANCHORED to a line start (one leading word such as/// "Command" tolerated): file content that merely mentions "exit code 1"/// arrives line-numbered or path-prefixed from `read`/`grep` results and must/// not decide the verdict. The harness summary is usually the last line, so/// every match is scanned and a nonzero code wins (an earlier "code 0"/// mention must not mask a later failure).pub const EXIT_CODE_PATTERN: &str = r"(?m)^[ \t]*(?:[A-Za-z]+[ \t]+)?(?:exited with|exit) code[:\s]+(-?\d+)";
/// Anchored error-prefix alternatives, applied to the left-trimmed result./// Case-insensitive. Each alternative is a real failure shape observed in/// Tailscale Aperture captures; see the module doc for why anchoring is safe.pub const ERROR_PREFIX_PATTERN: &str = concat!( r"^(?:(?:read )?error:", r"|ENOENT:", r"|invalid regex pattern", r"|invalid parameter '", r"|invalid tool input:", r"|Permission denied for ", r"|runtime error:", r"|parse error:", r"|old_string not found", r"|file changed since last read", r"|hash \S+ matches multiple lines", r")",);
/// Compiled failure detectors (exit-code + anchored error-prefix). Build once,/// reuse across records.pub struct Detector { exit: Regex, prefix: Regex,}
impl Detector { pub fn new() -> Result<Self, regex::Error> { Ok(Self { exit: RegexBuilder::new(EXIT_CODE_PATTERN) .case_insensitive(true) .build()?, prefix: RegexBuilder::new(ERROR_PREFIX_PATTERN) .case_insensitive(true) .build()?, }) }
fn exit_code(&self, text: &str) -> Option<i64> { let mut zero = None; for c in self .exit .captures_iter(text) .filter_map(|c| c.get(1)?.as_str().parse::<i64>().ok()) { if c != 0 { return Some(c); } zero = Some(c); } zero }
/// Decide (is_failure, exit_code). An Anthropic `is_error: true` flag is /// authoritative; otherwise a nonzero exit code is a failure; otherwise the /// anchored error-prefix is. A zero exit code is success (code surfaced). fn classify(&self, text: &str, is_error_flag: Option<bool>) -> (bool, Option<i64>) { let code = self.exit_code(text); let fail = match is_error_flag { Some(true) => true, _ => match code { Some(c) if c != 0 => true, _ => self.prefix.is_match(text.trim_start()), }, }; (fail, code) }}
/// One tool call paired with its result, ready to emit.pub struct ToolRow { pub tool_call_id: String, pub name: String, pub input: Json, pub result: String, pub is_error: bool, pub exit_code: Option<i64>,}
impl ToolRow { pub fn is_failure(&self) -> bool { self.is_error }}
/// Parse `capture.requestBody` (a JSON-encoded string in these captures) to a/// Json object. Mirrors `thread::record_request_body` but lives here so the/// tool-failure path stays decoupled from the thread module.fn request_body(rec: &ciborium::Value) -> Option<Json> { match format::path_get(rec, "capture.requestBody")? { ciborium::Value::Text(s) if !s.is_empty() => serde_json::from_str::<Json>(s).ok(), _ => None, }}
/// Normalize a tool call's arguments/input to a JSON value. OpenAI `arguments`/// is a JSON string (parse it); Anthropic `input` is already a value.fn parse_input(v: &Json) -> Json { match v { Json::String(s) => serde_json::from_str::<Json>(s).unwrap_or_else(|_| v.clone()), _ => v.clone(), }}
/// Flatten an Anthropic `tool_result.content` (string or array of text blocks)/// into a single string. OpenAI results are already strings and pass through.fn result_text(content: &Json) -> String { match content { Json::String(s) => s.clone(), Json::Array(arr) => arr .iter() .filter_map(|b| match b { Json::String(s) => Some(s.clone()), Json::Object(o) => o.get("text").and_then(|t| t.as_str()).map(String::from), _ => None, }) .collect::<Vec<_>>() .join("\n"), _ => String::new(), }}
/// Decide (is_error, exit_code) for a result via the [`Detector`].fn classify(det: &Detector, text: &str, is_error_flag: Option<bool>) -> (bool, Option<i64>) { det.classify(text, is_error_flag)}
/// Inspect one record's request history and return a `ToolRow` per tool result/// in the NEW turn (after the last assistant message), each paired with its/// call. Returns empty when the record has no tool results.pub fn collect(rec: &ciborium::Value, det: &Detector) -> Vec<ToolRow> { let mut rows = Vec::new(); let body = match request_body(rec) { Some(b) => b, None => return rows, }; let msgs = match body.get("messages").and_then(|m| m.as_array()) { Some(a) => a, None => return rows, }; // Last assistant message; everything after it is the new user/tool turn. let last_asst = msgs .iter() .rposition(|m| m.get("role").and_then(|v| v.as_str()) == Some("assistant")); let am_idx = match last_asst { Some(i) => i, None => return rows, }; if am_idx + 1 >= msgs.len() { return rows; }
// Build id -> (name, input) from the last assistant message's tool calls. let mut calls: BTreeMap<String, (String, Json)> = BTreeMap::new(); let am = &msgs[am_idx]; if let Some(tcs) = am.get("tool_calls").and_then(|v| v.as_array()) { for c in tcs { let id = c .get("id") .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); let f = c.get("function"); let name = f .and_then(|f| f.get("name")) .and_then(|v| v.as_str()) .unwrap_or("tool") .to_string(); let input = f .and_then(|f| f.get("arguments")) .map(parse_input) .unwrap_or(Json::Null); calls.insert(id, (name, input)); } } if let Some(blocks) = am.get("content").and_then(|v| v.as_array()) { for b in blocks { if b.get("type").and_then(|v| v.as_str()) == Some("tool_use") { let id = b .get("id") .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); let name = b .get("name") .and_then(|v| v.as_str()) .unwrap_or("tool") .to_string(); let input = b.get("input").map(parse_input).unwrap_or(Json::Null); calls.insert(id, (name, input)); } } }
// Iterate the new turn; pair results to calls by id. for m in &msgs[am_idx + 1..] { let role = m.get("role").and_then(|v| v.as_str()).unwrap_or(""); // OpenAI role:"tool" result. if role == "tool" { push_row( &mut rows, &calls, det, m.get("tool_call_id").and_then(|v| v.as_str()).unwrap_or(""), m.get("content").unwrap_or(&Json::Null), None, ); continue; } // Anthropic tool_result blocks live in a user message's content array. if let Some(blocks) = m.get("content").and_then(|v| v.as_array()) { for b in blocks { if b.get("type").and_then(|v| v.as_str()) == Some("tool_result") { let flag = b.get("is_error").and_then(|v| v.as_bool()); push_row( &mut rows, &calls, det, b.get("tool_use_id").and_then(|v| v.as_str()).unwrap_or(""), b.get("content").unwrap_or(&Json::Null), flag, ); } } } } rows}
fn push_row( rows: &mut Vec<ToolRow>, calls: &BTreeMap<String, (String, Json)>, det: &Detector, id: &str, content: &Json, is_error_flag: Option<bool>,) { let text = result_text(content); let (name, input) = match calls.get(id) { Some((n, i)) => (n.clone(), i.clone()), None => ("?".to_string(), Json::Null), }; let (is_error, exit_code) = classify(det, &text, is_error_flag); rows.push(ToolRow { tool_call_id: id.to_string(), name, input, result: text, is_error, exit_code, });}
#[cfg(test)]mod tests { use super::*;
fn rec(body: &str) -> ciborium::Value { // Build a minimal CBOR record with capture.requestBody = body (text). let json = serde_json::json!({ "id": 1, "model": "m", "capture": { "requestBody": body } }); crate::format::json_to_cbor(&json) }
#[test] fn pairs_openai_call_to_result_and_flags_nonzero_exit() { let body = serde_json::json!({ "messages": [ {"role":"system","content":"s"}, {"role":"user","content":"run it"}, {"role":"assistant","content":null,"tool_calls":[ {"id":"call_1","type":"function","function":{"name":"bash","arguments":"{\"command\":\"ls\"}"}} ]}, {"role":"tool","tool_call_id":"call_1","content":"(no output)\n\nCommand exited with code 1"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert_eq!(rows[0].name, "bash"); assert_eq!(rows[0].tool_call_id, "call_1"); assert_eq!(rows[0].exit_code, Some(1)); assert!(rows[0].is_failure()); // input parsed from the JSON arguments string. assert_eq!(rows[0].input["command"], "ls"); }
#[test] fn zero_exit_code_is_not_a_failure() { let body = serde_json::json!({ "messages": [ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"bash","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"ok\nCommand exited with code 0"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert_eq!(rows[0].exit_code, Some(0)); assert!(!rows[0].is_failure()); }
#[test] fn no_exit_code_is_success() { let body = serde_json::json!({ "messages": [ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"read","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"file contents here"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(!rows[0].is_failure()); assert_eq!(rows[0].exit_code, None); }
#[test] fn anthropic_is_error_flag_marks_failure() { let body = serde_json::json!({ "messages": [ {"role":"user","content":"go"}, {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"search","input":{"q":"x"}}]}, {"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"not found","is_error":true}]} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert_eq!(rows[0].name, "search"); assert!(rows[0].is_failure()); assert_eq!(rows[0].exit_code, None); }
#[test] fn only_new_turn_results_are_emitted() { // History has a prior tool_result; only the one after the last // assistant must be emitted. let body = serde_json::json!({ "messages": [ {"role":"assistant","tool_calls":[{"id":"a","function":{"name":"bash","arguments":"{}"}}]}, {"role":"tool","tool_call_id":"a","content":"old\nCommand exited with code 1"}, {"role":"assistant","content":"done"}, {"role":"tool","tool_call_id":"a","content":"new\nCommand exited with code 2"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1, "only the new-turn result is emitted"); assert_eq!(rows[0].exit_code, Some(2)); }
#[test] fn empty_body_yields_nothing() { let det = Detector::new().unwrap(); let v = crate::format::json_to_cbor(&serde_json::json!({"id":1,"capture":{}})); assert!(collect(&v, &det).is_empty()); let v2 = crate::format::json_to_cbor( &serde_json::json!({"id":1,"capture":{"requestBody":"not json"}}), ); assert!(collect(&v2, &det).is_empty()); }
// --- anchored error-prefix detection (non-shell tools) ---
#[test] fn edit_old_string_not_found_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"edit","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"old_string not found in file"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure(), "edit no-match is a failure"); assert_eq!(rows[0].exit_code, None); }
#[test] fn enoent_prefix_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"read","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"ENOENT: no such file or directory, access '/nope.txt'"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure()); }
#[test] fn permission_denied_prefix_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"write","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"Permission denied for `write` (/tmp/x). Do not retry."} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure(), "runtime rejection is a failure"); }
#[test] fn exit_code_phrasing_inside_quoted_content_is_not_a_failure() { // Regression: the pattern matched anywhere in the result and took the // first match, so a successful `read` whose file content mentioned // "exit code 1" was judged a failure — and an early "code 0" mention // masked a later nonzero summary line. let det = Detector::new().unwrap(); let quoted = "Contents of NOTES.md:\n1: Command exited with code 1 when run\n2: done"; let (fail, code) = det.classify(quoted, None); assert!(!fail, "line-numbered mention flagged as failure"); assert_eq!(code, None); // A summary line stays authoritative, even past a zero mention. let (fail, code) = det.classify("exit code 0\nrunning...\nCommand exited with code 1", None); assert!(fail); assert_eq!(code, Some(1)); }
#[test] fn file_content_starting_with_error_is_not_a_failure() { // A successful `read` of a file whose first line is literally "error: ..." // arrives prefixed with a line number ("1: error: ..."), so the anchored // prefix must NOT match — this is the load-bearing false-positive guard. let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"read","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"1: error: this is just source code\n2: fn main(){}"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!( !rows[0].is_failure(), "line-number-prefixed content is not an error" ); }
#[test] fn empty_result_is_not_a_failure() { // "No files found" / empty glob is a successful query, not a tool error. let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"glob","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"No files found"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(!rows[0].is_failure()); }
#[test] fn http_error_prefix_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"webfetch","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"error: HTTP 404"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure()); }}