//! Tool-call failure inspection for the `toolfailures` subcommand. //! //! Each record's `capture.requestBody` echoes the full conversation history, so //! a single record contains BOTH the assistant turn that issued tool calls AND //! the `role:"tool"` / `tool_result` messages carrying their results. We pair //! calls to results by id (`tool_calls[].id` <-> `tool_call_id` for OpenAI, //! `tool_use.id` <-> `tool_result.tool_use_id` for Anthropic) and inspect only //! the NEW turn (messages after the last assistant message). That way each //! call/result pair is emitted exactly once, in the record where the result //! first appears — no double-counting as a thread grows. //! //! A result is a FAILURE when ANY of: //! - an Anthropic `tool_result` block carries `is_error: true`, //! - a line in the result reports a nonzero exit code, e.g. //! "Command exited with code 1" (shell tools), or //! - the result text starts with one of a known set of error prefixes, e.g. //! "error: HTTP 404", "ENOENT: no such file...", "old_string not found", //! "Permission denied for `edit` (...)". //! //! Prefix matching is ANCHORED to the start of the (left-trimmed) result so it //! can't false-positive on file content that merely mentions "error" — tools //! that return arbitrary content (`read`, `grep`, `glob`) prefix it with line //! numbers or paths, so a file whose first line reads "error: ..." arrives as //! "1: error: ..." and does not match. "No files found" and similar empty-but- //! successful results are deliberately NOT failures: the tool ran fine. use crate::format; use regex::{Regex, RegexBuilder}; use serde_json::Value as Json; use std::collections::BTreeMap; /// Exit-code phrasing seen in real captures, e.g. "Command exited with code 1". /// Group 1 captures the numeric code. Matched case-insensitively. Like the /// error prefixes, ANCHORED to a line start (one leading word such as /// "Command" tolerated): file content that merely mentions "exit code 1" /// arrives line-numbered or path-prefixed from `read`/`grep` results and must /// not decide the verdict. The harness summary is usually the last line, so /// every match is scanned and a nonzero code wins (an earlier "code 0" /// mention must not mask a later failure). pub const EXIT_CODE_PATTERN: &str = r"(?m)^[ \t]*(?:[A-Za-z]+[ \t]+)?(?:exited with|exit) code[:\s]+(-?\d+)"; /// Anchored error-prefix alternatives, applied to the left-trimmed result. /// Case-insensitive. Each alternative is a real failure shape observed in /// Tailscale Aperture captures; see the module doc for why anchoring is safe. pub const ERROR_PREFIX_PATTERN: &str = concat!( r"^(?:(?:read )?error:", r"|ENOENT:", r"|invalid regex pattern", r"|invalid parameter '", r"|invalid tool input:", r"|Permission denied for ", r"|runtime error:", r"|parse error:", r"|old_string not found", r"|file changed since last read", r"|hash \S+ matches multiple lines", r")", ); /// Compiled failure detectors (exit-code + anchored error-prefix). Build once, /// reuse across records. pub struct Detector { exit: Regex, prefix: Regex, } impl Detector { pub fn new() -> Result { Ok(Self { exit: RegexBuilder::new(EXIT_CODE_PATTERN) .case_insensitive(true) .build()?, prefix: RegexBuilder::new(ERROR_PREFIX_PATTERN) .case_insensitive(true) .build()?, }) } fn exit_code(&self, text: &str) -> Option { let mut zero = None; for c in self .exit .captures_iter(text) .filter_map(|c| c.get(1)?.as_str().parse::().ok()) { if c != 0 { return Some(c); } zero = Some(c); } zero } /// Decide (is_failure, exit_code). An Anthropic `is_error: true` flag is /// authoritative; otherwise a nonzero exit code is a failure; otherwise the /// anchored error-prefix is. A zero exit code is success (code surfaced). fn classify(&self, text: &str, is_error_flag: Option) -> (bool, Option) { let code = self.exit_code(text); let fail = match is_error_flag { Some(true) => true, _ => match code { Some(c) if c != 0 => true, _ => self.prefix.is_match(text.trim_start()), }, }; (fail, code) } } /// One tool call paired with its result, ready to emit. pub struct ToolRow { pub tool_call_id: String, pub name: String, pub input: Json, pub result: String, pub is_error: bool, pub exit_code: Option, } impl ToolRow { pub fn is_failure(&self) -> bool { self.is_error } } /// Parse `capture.requestBody` (a JSON-encoded string in these captures) to a /// Json object. Mirrors `thread::record_request_body` but lives here so the /// tool-failure path stays decoupled from the thread module. fn request_body(rec: &ciborium::Value) -> Option { match format::path_get(rec, "capture.requestBody")? { ciborium::Value::Text(s) if !s.is_empty() => serde_json::from_str::(s).ok(), _ => None, } } /// Normalize a tool call's arguments/input to a JSON value. OpenAI `arguments` /// is a JSON string (parse it); Anthropic `input` is already a value. fn parse_input(v: &Json) -> Json { match v { Json::String(s) => serde_json::from_str::(s).unwrap_or_else(|_| v.clone()), _ => v.clone(), } } /// Flatten an Anthropic `tool_result.content` (string or array of text blocks) /// into a single string. OpenAI results are already strings and pass through. fn result_text(content: &Json) -> String { match content { Json::String(s) => s.clone(), Json::Array(arr) => arr .iter() .filter_map(|b| match b { Json::String(s) => Some(s.clone()), Json::Object(o) => o.get("text").and_then(|t| t.as_str()).map(String::from), _ => None, }) .collect::>() .join("\n"), _ => String::new(), } } /// Decide (is_error, exit_code) for a result via the [`Detector`]. fn classify(det: &Detector, text: &str, is_error_flag: Option) -> (bool, Option) { det.classify(text, is_error_flag) } /// Inspect one record's request history and return a `ToolRow` per tool result /// in the NEW turn (after the last assistant message), each paired with its /// call. Returns empty when the record has no tool results. pub fn collect(rec: &ciborium::Value, det: &Detector) -> Vec { let mut rows = Vec::new(); let body = match request_body(rec) { Some(b) => b, None => return rows, }; let msgs = match body.get("messages").and_then(|m| m.as_array()) { Some(a) => a, None => return rows, }; // Last assistant message; everything after it is the new user/tool turn. let last_asst = msgs .iter() .rposition(|m| m.get("role").and_then(|v| v.as_str()) == Some("assistant")); let am_idx = match last_asst { Some(i) => i, None => return rows, }; if am_idx + 1 >= msgs.len() { return rows; } // Build id -> (name, input) from the last assistant message's tool calls. let mut calls: BTreeMap = BTreeMap::new(); let am = &msgs[am_idx]; if let Some(tcs) = am.get("tool_calls").and_then(|v| v.as_array()) { for c in tcs { let id = c .get("id") .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); let f = c.get("function"); let name = f .and_then(|f| f.get("name")) .and_then(|v| v.as_str()) .unwrap_or("tool") .to_string(); let input = f .and_then(|f| f.get("arguments")) .map(parse_input) .unwrap_or(Json::Null); calls.insert(id, (name, input)); } } if let Some(blocks) = am.get("content").and_then(|v| v.as_array()) { for b in blocks { if b.get("type").and_then(|v| v.as_str()) == Some("tool_use") { let id = b .get("id") .and_then(|v| v.as_str()) .unwrap_or("") .to_string(); let name = b .get("name") .and_then(|v| v.as_str()) .unwrap_or("tool") .to_string(); let input = b.get("input").map(parse_input).unwrap_or(Json::Null); calls.insert(id, (name, input)); } } } // Iterate the new turn; pair results to calls by id. for m in &msgs[am_idx + 1..] { let role = m.get("role").and_then(|v| v.as_str()).unwrap_or(""); // OpenAI role:"tool" result. if role == "tool" { push_row( &mut rows, &calls, det, m.get("tool_call_id").and_then(|v| v.as_str()).unwrap_or(""), m.get("content").unwrap_or(&Json::Null), None, ); continue; } // Anthropic tool_result blocks live in a user message's content array. if let Some(blocks) = m.get("content").and_then(|v| v.as_array()) { for b in blocks { if b.get("type").and_then(|v| v.as_str()) == Some("tool_result") { let flag = b.get("is_error").and_then(|v| v.as_bool()); push_row( &mut rows, &calls, det, b.get("tool_use_id").and_then(|v| v.as_str()).unwrap_or(""), b.get("content").unwrap_or(&Json::Null), flag, ); } } } } rows } fn push_row( rows: &mut Vec, calls: &BTreeMap, det: &Detector, id: &str, content: &Json, is_error_flag: Option, ) { let text = result_text(content); let (name, input) = match calls.get(id) { Some((n, i)) => (n.clone(), i.clone()), None => ("?".to_string(), Json::Null), }; let (is_error, exit_code) = classify(det, &text, is_error_flag); rows.push(ToolRow { tool_call_id: id.to_string(), name, input, result: text, is_error, exit_code, }); } #[cfg(test)] mod tests { use super::*; fn rec(body: &str) -> ciborium::Value { // Build a minimal CBOR record with capture.requestBody = body (text). let json = serde_json::json!({ "id": 1, "model": "m", "capture": { "requestBody": body } }); crate::format::json_to_cbor(&json) } #[test] fn pairs_openai_call_to_result_and_flags_nonzero_exit() { let body = serde_json::json!({ "messages": [ {"role":"system","content":"s"}, {"role":"user","content":"run it"}, {"role":"assistant","content":null,"tool_calls":[ {"id":"call_1","type":"function","function":{"name":"bash","arguments":"{\"command\":\"ls\"}"}} ]}, {"role":"tool","tool_call_id":"call_1","content":"(no output)\n\nCommand exited with code 1"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert_eq!(rows[0].name, "bash"); assert_eq!(rows[0].tool_call_id, "call_1"); assert_eq!(rows[0].exit_code, Some(1)); assert!(rows[0].is_failure()); // input parsed from the JSON arguments string. assert_eq!(rows[0].input["command"], "ls"); } #[test] fn zero_exit_code_is_not_a_failure() { let body = serde_json::json!({ "messages": [ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"bash","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"ok\nCommand exited with code 0"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert_eq!(rows[0].exit_code, Some(0)); assert!(!rows[0].is_failure()); } #[test] fn no_exit_code_is_success() { let body = serde_json::json!({ "messages": [ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"read","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"file contents here"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(!rows[0].is_failure()); assert_eq!(rows[0].exit_code, None); } #[test] fn anthropic_is_error_flag_marks_failure() { let body = serde_json::json!({ "messages": [ {"role":"user","content":"go"}, {"role":"assistant","content":[{"type":"tool_use","id":"t1","name":"search","input":{"q":"x"}}]}, {"role":"user","content":[{"type":"tool_result","tool_use_id":"t1","content":"not found","is_error":true}]} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert_eq!(rows[0].name, "search"); assert!(rows[0].is_failure()); assert_eq!(rows[0].exit_code, None); } #[test] fn only_new_turn_results_are_emitted() { // History has a prior tool_result; only the one after the last // assistant must be emitted. let body = serde_json::json!({ "messages": [ {"role":"assistant","tool_calls":[{"id":"a","function":{"name":"bash","arguments":"{}"}}]}, {"role":"tool","tool_call_id":"a","content":"old\nCommand exited with code 1"}, {"role":"assistant","content":"done"}, {"role":"tool","tool_call_id":"a","content":"new\nCommand exited with code 2"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1, "only the new-turn result is emitted"); assert_eq!(rows[0].exit_code, Some(2)); } #[test] fn empty_body_yields_nothing() { let det = Detector::new().unwrap(); let v = crate::format::json_to_cbor(&serde_json::json!({"id":1,"capture":{}})); assert!(collect(&v, &det).is_empty()); let v2 = crate::format::json_to_cbor( &serde_json::json!({"id":1,"capture":{"requestBody":"not json"}}), ); assert!(collect(&v2, &det).is_empty()); } // --- anchored error-prefix detection (non-shell tools) --- #[test] fn edit_old_string_not_found_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"edit","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"old_string not found in file"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure(), "edit no-match is a failure"); assert_eq!(rows[0].exit_code, None); } #[test] fn enoent_prefix_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"read","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"ENOENT: no such file or directory, access '/nope.txt'"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure()); } #[test] fn permission_denied_prefix_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"write","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"Permission denied for `write` (/tmp/x). Do not retry."} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure(), "runtime rejection is a failure"); } #[test] fn exit_code_phrasing_inside_quoted_content_is_not_a_failure() { // Regression: the pattern matched anywhere in the result and took the // first match, so a successful `read` whose file content mentioned // "exit code 1" was judged a failure — and an early "code 0" mention // masked a later nonzero summary line. let det = Detector::new().unwrap(); let quoted = "Contents of NOTES.md:\n1: Command exited with code 1 when run\n2: done"; let (fail, code) = det.classify(quoted, None); assert!(!fail, "line-numbered mention flagged as failure"); assert_eq!(code, None); // A summary line stays authoritative, even past a zero mention. let (fail, code) = det.classify("exit code 0\nrunning...\nCommand exited with code 1", None); assert!(fail); assert_eq!(code, Some(1)); } #[test] fn file_content_starting_with_error_is_not_a_failure() { // A successful `read` of a file whose first line is literally "error: ..." // arrives prefixed with a line number ("1: error: ..."), so the anchored // prefix must NOT match — this is the load-bearing false-positive guard. let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"read","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"1: error: this is just source code\n2: fn main(){}"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!( !rows[0].is_failure(), "line-number-prefixed content is not an error" ); } #[test] fn empty_result_is_not_a_failure() { // "No files found" / empty glob is a successful query, not a tool error. let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"glob","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"No files found"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(!rows[0].is_failure()); } #[test] fn http_error_prefix_is_a_failure() { let body = serde_json::json!({ "messages":[ {"role":"user","content":"go"}, {"role":"assistant","tool_calls":[ {"id":"c","function":{"name":"webfetch","arguments":"{}"}} ]}, {"role":"tool","tool_call_id":"c","content":"error: HTTP 404"} ] }) .to_string(); let det = Detector::new().unwrap(); let rows = collect(&rec(&body), &det); assert_eq!(rows.len(), 1); assert!(rows[0].is_failure()); } }