Something went wrong. Try again.
fork of prime agent with changes i want but are not upstreamable probqbly (and some fixes)
Something went wrong. Try again.
33 kB · 834 lines
JavaScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835#!/usr/bin/env node
import { createReadStream } from "node:fs";import { promises as fs } from "node:fs";import { homedir } from "node:os";import path from "node:path";import { createInterface } from "node:readline";
const DEFAULT_SESSIONS_DIR = path.join(homedir(), ".pi/agent/sessions");const DEFAULT_ACTIVE_EDIT_EXTENSION_PATH = path.join(homedir(), ".pi/agent/extensions/edit.ts");const DEFAULT_TOP = 20;
function parseArgs(argv) { const options = { sessionsDir: DEFAULT_SESSIONS_DIR, json: false, includeRecords: false, failedOnly: false, modelFilter: undefined, extFilter: undefined, top: DEFAULT_TOP, help: false, allSessions: false, since: undefined, autoSincePath: DEFAULT_ACTIVE_EDIT_EXTENSION_PATH, };
for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === "--help" || arg === "-h") { options.help = true; } else if (arg === "--json") { options.json = true; } else if (arg === "--include-records") { options.includeRecords = true; } else if (arg === "--failed-only") { options.failedOnly = true; } else if (arg === "--model") { options.modelFilter = argv[++i]; } else if (arg === "--ext") { options.extFilter = argv[++i]?.toLowerCase(); } else if (arg === "--top") { const value = Number.parseInt(argv[++i] ?? "", 10); if (!Number.isFinite(value) || value <= 0) { throw new Error("--top must be a positive integer"); } options.top = value; } else if (arg === "--sessions-dir") { options.sessionsDir = argv[++i]; } else if (arg === "--all-sessions") { options.allSessions = true; } else if (arg === "--since") { options.since = argv[++i]; } else if (arg === "--auto-since-path") { options.autoSincePath = argv[++i]; } else { throw new Error(`Unknown argument: ${arg}`); } }
return options;}
function printHelp() { console.log(`Usage: node scripts/edit-tool-stats.mjs [options]
Options: --sessions-dir <path> Sessions directory (default: ~/.pi/agent/sessions) --model <substring> Filter provider/model by substring --ext <extension> Filter by file extension, e.g. .ts --failed-only Include only failed edit calls --top <n> Number of examples to show (default: ${DEFAULT_TOP}) --since <iso> Only scan session files created at or after this ISO time --all-sessions Disable the automatic since filter --auto-since-path <p> Use birth time of this file for the automatic since filter --json Print JSON summary instead of human report --include-records Include raw records in JSON output -h, --help Show this help`);}
function parseSessionFileTimestamp(sessionFile) { const base = path.basename(sessionFile); const rawTimestamp = base.split("_")[0]; if (!rawTimestamp) return null; const isoTimestamp = rawTimestamp.replace(/T(\d{2})-(\d{2})-(\d{2})-(\d{3})Z$/, "T$1:$2:$3.$4Z"); const ms = Date.parse(isoTimestamp); if (!Number.isFinite(ms)) return null; return ms;}
function formatIso(ms) { return new Date(ms).toISOString();}
async function resolveAutoSinceMs(options) { if (options.allSessions) return null; if (options.since) { const ms = Date.parse(options.since); if (!Number.isFinite(ms)) { throw new Error(`Invalid --since value: ${options.since}`); } return { ms, source: `--since ${options.since}` }; } if (!options.autoSincePath) return null; try { const stats = await fs.stat(options.autoSincePath); const birthtimeMs = Number.isFinite(stats.birthtimeMs) && stats.birthtimeMs > 0 ? stats.birthtimeMs : stats.mtimeMs; if (!Number.isFinite(birthtimeMs) || birthtimeMs <= 0) return null; return { ms: birthtimeMs, source: `birth time of ${options.autoSincePath}` }; } catch { return null; }}
async function* walkJsonlFiles(dir) { const entries = await fs.readdir(dir, { withFileTypes: true }); entries.sort((a, b) => a.name.localeCompare(b.name));
for (const entry of entries) { const fullPath = path.join(dir, entry.name); if (entry.isDirectory()) { yield* walkJsonlFiles(fullPath); } else if (entry.isFile() && entry.name.endsWith(".jsonl")) { yield fullPath; } }}
function getPathExtension(filePath) { if (typeof filePath !== "string" || filePath.length === 0) return "[unknown]"; const ext = path.extname(filePath).toLowerCase(); if (ext) return ext; const base = path.basename(filePath); if (base.startsWith(".") && !base.slice(1).includes(".")) return base.toLowerCase(); return "[no_ext]";}
function utf8Bytes(value) { return Buffer.byteLength(value ?? "", "utf8");}
function longestCommonPrefixLength(a, b) { const max = Math.min(a.length, b.length); let index = 0; while (index < max && a[index] === b[index]) index++; return index;}
function longestCommonSuffixLength(a, b) { const max = Math.min(a.length, b.length); let index = 0; while (index < max && a[a.length - 1 - index] === b[b.length - 1 - index]) index++; return index;}
function analyzeReplacement(oldText, newText) { const prefixChars = longestCommonPrefixLength(oldText, newText); const oldRemainder = oldText.slice(prefixChars); const newRemainder = newText.slice(prefixChars); const suffixChars = longestCommonSuffixLength(oldRemainder, newRemainder); const oldCore = suffixChars > 0 ? oldRemainder.slice(0, -suffixChars) : oldRemainder; const newCore = suffixChars > 0 ? newRemainder.slice(0, -suffixChars) : newRemainder; const prefix = oldText.slice(0, prefixChars); const suffix = suffixChars > 0 ? oldRemainder.slice(-suffixChars) : "";
const oldBytes = utf8Bytes(oldText); const newBytes = utf8Bytes(newText); const sharedPrefixBytes = utf8Bytes(prefix); const sharedSuffixBytes = utf8Bytes(suffix); const sharedContextBytes = sharedPrefixBytes + sharedSuffixBytes; const coreOldBytes = utf8Bytes(oldCore); const coreNewBytes = utf8Bytes(newCore); const coreBytes = coreOldBytes + coreNewBytes; const totalEditBytes = oldBytes + newBytes; const wrapperPayloadBytes = totalEditBytes - coreBytes; const inflationRatio = coreBytes === 0 ? null : totalEditBytes / coreBytes;
return { oldBytes, newBytes, totalEditBytes, sharedPrefixBytes, sharedSuffixBytes, sharedContextBytes, coreOldBytes, coreNewBytes, coreBytes, wrapperPayloadBytes, inflationRatio, noCoreChange: coreBytes === 0, };}
function median(numbers) { return quantile(numbers, 0.5);}
function quantile(numbers, q) { const finite = numbers.filter((value) => Number.isFinite(value)).sort((a, b) => a - b); if (finite.length === 0) return null; if (finite.length === 1) return finite[0]; const position = (finite.length - 1) * q; const lower = Math.floor(position); const upper = Math.ceil(position); if (lower === upper) return finite[lower]; const weight = position - lower; return finite[lower] * (1 - weight) + finite[upper] * weight;}
function formatInt(value) { return new Intl.NumberFormat("en-US").format(value);}
function formatPercent(part, total) { if (total === 0) return "n/a"; return `${((part / total) * 100).toFixed(1)}%`;}
function formatRatio(value) { if (value === null) return "no-core-change"; if (!Number.isFinite(value)) return "∞"; if (value >= 100) return `${value.toFixed(0)}x`; if (value >= 10) return `${value.toFixed(1)}x`; return `${value.toFixed(2)}x`;}
function formatBytes(value) { if (value < 1024) return `${value}B`; if (value < 1024 * 1024) return `${(value / 1024).toFixed(value >= 10 * 1024 ? 0 : 1)}KB`; return `${(value / (1024 * 1024)).toFixed(1)}MB`;}
function extractTextContent(content) { if (typeof content === "string") return content; if (!Array.isArray(content)) return ""; return content .filter((block) => block?.type === "text" && typeof block.text === "string") .map((block) => block.text) .join("\n");}
function classifyErrorKind(text, isError, matchedResult) { if (!matchedResult) return "missing_result"; if (!isError) return null; const normalized = text.toLowerCase(); if (normalized.includes("file not found")) return "file_not_found"; if (normalized.includes("could not find the exact text")) return "not_found_exact_text"; if (normalized.includes("found multiple occurrences") || /^found \d+ occurrences/m.test(normalized)) { return "multiple_occurrences"; } if (normalized.includes("no changes made")) return "no_changes_made"; if (normalized.includes("input is invalid")) return "invalid_input"; if (normalized.includes("must not overlap")) return "overlapping_edits"; if (normalized.includes("aborted")) return "aborted"; return "other";}
function getArgStyle(args) { const hasEdits = Array.isArray(args?.edits); const hasOldText = typeof args?.oldText === "string" || typeof args?.newText === "string"; const hasOldString = typeof args?.old_string === "string" || typeof args?.new_string === "string"; if (hasEdits) return "edits"; if (hasOldText && hasOldString) return "mixed"; if (hasOldText) return "oldText/newText"; if (hasOldString) return "old_string/new_string"; return "unknown";}
function analyzeToolArguments(args) { const normalizedArgs = args && typeof args === "object" ? args : {}; const filePath = typeof normalizedArgs.path === "string" ? normalizedArgs.path : ""; const extension = getPathExtension(filePath); const argStyle = getArgStyle(normalizedArgs);
if (Array.isArray(normalizedArgs.edits)) { const perEdit = normalizedArgs.edits.map((edit) => analyzeReplacement(typeof edit?.oldText === "string" ? edit.oldText : "", typeof edit?.newText === "string" ? edit.newText : "") ); const inflations = perEdit.map((edit) => edit.inflationRatio).filter((value) => value !== null); const totals = perEdit.reduce( (acc, edit) => ({ oldBytes: acc.oldBytes + edit.oldBytes, newBytes: acc.newBytes + edit.newBytes, totalEditBytes: acc.totalEditBytes + edit.totalEditBytes, sharedPrefixBytes: acc.sharedPrefixBytes + edit.sharedPrefixBytes, sharedSuffixBytes: acc.sharedSuffixBytes + edit.sharedSuffixBytes, sharedContextBytes: acc.sharedContextBytes + edit.sharedContextBytes, coreOldBytes: acc.coreOldBytes + edit.coreOldBytes, coreNewBytes: acc.coreNewBytes + edit.coreNewBytes, coreBytes: acc.coreBytes + edit.coreBytes, wrapperPayloadBytes: acc.wrapperPayloadBytes + edit.wrapperPayloadBytes, noCoreChangeCount: acc.noCoreChangeCount + (edit.noCoreChange ? 1 : 0), }), { oldBytes: 0, newBytes: 0, totalEditBytes: 0, sharedPrefixBytes: 0, sharedSuffixBytes: 0, sharedContextBytes: 0, coreOldBytes: 0, coreNewBytes: 0, coreBytes: 0, wrapperPayloadBytes: 0, noCoreChangeCount: 0, } ); return { path: filePath, extension, mode: "multi", argStyle, editsCount: normalizedArgs.edits.length, ...totals, inflationRatio: totals.coreBytes === 0 ? null : totals.totalEditBytes / totals.coreBytes, medianEditInflationRatio: median(inflations), maxEditInflationRatio: inflations.length > 0 ? Math.max(...inflations) : null, perEdit, }; }
const oldText = typeof normalizedArgs.oldText === "string" ? normalizedArgs.oldText : typeof normalizedArgs.old_string === "string" ? normalizedArgs.old_string : ""; const newText = typeof normalizedArgs.newText === "string" ? normalizedArgs.newText : typeof normalizedArgs.new_string === "string" ? normalizedArgs.new_string : ""; const replacement = analyzeReplacement(oldText, newText); return { path: filePath, extension, mode: "single", argStyle, editsCount: 1, ...replacement, medianEditInflationRatio: replacement.inflationRatio, maxEditInflationRatio: replacement.inflationRatio, perEdit: [replacement], };}
function groupCounts(records, keyFn) { const counts = new Map(); for (const record of records) { const key = keyFn(record); counts.set(key, (counts.get(key) ?? 0) + 1); } return counts;}
function collectInflations(records) { return records.map((record) => record.inflationRatio).filter((value) => value !== null);}
function summarizeGroups(records, keyFn) { const groups = new Map(); for (const record of records) { const key = keyFn(record); if (!groups.has(key)) groups.set(key, []); groups.get(key).push(record); }
return [...groups.entries()] .map(([key, group]) => { const resolved = group.filter((record) => record.success !== null); const success = resolved.filter((record) => record.success).length; const failed = resolved.filter((record) => record.success === false).length; const multi = group.filter((record) => record.mode === "multi").length; const inflations = collectInflations(group); return { key, calls: group.length, resolved: resolved.length, success, failed, unresolved: group.length - resolved.length, multi, multiRate: group.length === 0 ? null : multi / group.length, successRate: resolved.length === 0 ? null : success / resolved.length, medianInflation: quantile(inflations, 0.5), p95Inflation: quantile(inflations, 0.95), }; }) .sort((a, b) => b.calls - a.calls || a.key.localeCompare(b.key));}
function buildSameFileClusterStats(records) { const groups = new Map(); for (const record of records) { const key = `${record.sessionFile}::${record.assistantEntryId}::${record.path}`; if (!groups.has(key)) groups.set(key, []); groups.get(key).push(record); }
const clusters = [...groups.values()].filter((group) => group.length >= 2); const assistantMessagesWithCluster = new Set(clusters.map((group) => `${group[0].sessionFile}::${group[0].assistantEntryId}`)); const assistantMessagesWithMultiEdit = new Set( records .filter((record) => record.mode === "multi" && record.editsCount > 1) .map((record) => `${record.sessionFile}::${record.assistantEntryId}`) ); const callsInsideClusters = clusters.reduce((sum, group) => sum + group.length, 0);
return { clustersCount: clusters.length, assistantMessagesWithCluster: assistantMessagesWithCluster.size, assistantMessagesWithMultiEdit: assistantMessagesWithMultiEdit.size, callsInsideClusters, averageCallsPerCluster: clusters.length === 0 ? null : callsInsideClusters / clusters.length, ratioClusterAssistantMessagesToMultiEditAssistantMessages: assistantMessagesWithMultiEdit.size === 0 ? null : assistantMessagesWithCluster.size / assistantMessagesWithMultiEdit.size, };}
function buildInflationBuckets(records) { const buckets = [ { key: "no_core_change", label: "no-core-change", test: (record) => record.inflationRatio === null }, { key: "lt4", label: "<4x", test: (record) => record.inflationRatio !== null && record.inflationRatio < 4 }, { key: "4to10", label: "4-10x", test: (record) => record.inflationRatio !== null && record.inflationRatio >= 4 && record.inflationRatio < 10 }, { key: "10to25", label: "10-25x", test: (record) => record.inflationRatio !== null && record.inflationRatio >= 10 && record.inflationRatio < 25 }, { key: "gte25", label: "25x+", test: (record) => record.inflationRatio !== null && record.inflationRatio >= 25 }, ];
return buckets.map((bucket) => { const bucketRecords = records.filter(bucket.test); const resolved = bucketRecords.filter((record) => record.success !== null); const failed = resolved.filter((record) => record.success === false).length; return { key: bucket.key, label: bucket.label, count: bucketRecords.length, resolved: resolved.length, failed, failureRate: resolved.length === 0 ? null : failed / resolved.length, }; });}
function buildHugeReplacementStats(records) { const thresholds = [1024, 4096, 16384, 65536]; return thresholds.map((threshold) => ({ threshold, count: records.filter((record) => record.totalEditBytes > threshold).length, }));}
function buildWorstExamples(records, top) { const scored = [...records].sort((a, b) => { const aScore = a.inflationRatio === null ? Number.POSITIVE_INFINITY : a.inflationRatio; const bScore = b.inflationRatio === null ? Number.POSITIVE_INFINITY : b.inflationRatio; if (aScore !== bScore) return bScore - aScore; if (a.totalEditBytes !== b.totalEditBytes) return b.totalEditBytes - a.totalEditBytes; return a.path.localeCompare(b.path); });
return scored.slice(0, top).map((record) => ({ providerModel: record.providerModel, extension: record.extension, path: record.path, inflationRatio: record.inflationRatio, totalEditBytes: record.totalEditBytes, coreBytes: record.coreBytes, mode: record.mode, editsCount: record.editsCount, failed: record.success === false, errorKind: record.errorKind, sessionFile: record.sessionFile, }));}
function buildSummary(records, meta, options) { const uniqueAssistantMessages = new Set(records.map((record) => `${record.sessionFile}::${record.assistantEntryId}`)).size; const resolved = records.filter((record) => record.success !== null); const success = resolved.filter((record) => record.success).length; const failed = resolved.filter((record) => record.success === false).length; const unresolved = records.length - resolved.length; const single = records.filter((record) => record.mode === "single").length; const multi = records.filter((record) => record.mode === "multi").length; const modeStats = ["single", "multi"].map((mode) => { const modeRecords = records.filter((record) => record.mode === mode); const modeResolved = modeRecords.filter((record) => record.success !== null); const modeSuccess = modeResolved.filter((record) => record.success).length; const modeFailed = modeResolved.filter((record) => record.success === false).length; return { mode, calls: modeRecords.length, resolved: modeResolved.length, success: modeSuccess, failed: modeFailed, unresolved: modeRecords.length - modeResolved.length, successRate: modeResolved.length === 0 ? null : modeSuccess / modeResolved.length, failureRate: modeResolved.length === 0 ? null : modeFailed / modeResolved.length, }; }); const multiEditLengthBuckets = [ { key: "1", label: "edits.length === 1", test: (record) => record.mode === "multi" && record.editsCount === 1 }, { key: "2", label: "edits.length === 2", test: (record) => record.mode === "multi" && record.editsCount === 2 }, { key: "3plus", label: "edits.length >= 3", test: (record) => record.mode === "multi" && record.editsCount >= 3 }, ].map((bucket) => { const bucketRecords = records.filter(bucket.test); const bucketResolved = bucketRecords.filter((record) => record.success !== null); const bucketSuccess = bucketResolved.filter((record) => record.success).length; const bucketFailed = bucketResolved.filter((record) => record.success === false).length; return { key: bucket.key, label: bucket.label, calls: bucketRecords.length, resolved: bucketResolved.length, success: bucketSuccess, failed: bucketFailed, unresolved: bucketRecords.length - bucketResolved.length, successRate: bucketResolved.length === 0 ? null : bucketSuccess / bucketResolved.length, failureRate: bucketResolved.length === 0 ? null : bucketFailed / bucketResolved.length, }; }); const argStyles = [...groupCounts(records, (record) => record.argStyle).entries()] .map(([style, count]) => ({ style, count })) .sort((a, b) => b.count - a.count || a.style.localeCompare(b.style)); const providerStats = summarizeGroups(records, (record) => record.providerModel); const extensionStats = summarizeGroups(records, (record) => record.extension); const inflations = collectInflations(records); const noCoreChange = records.filter((record) => record.inflationRatio === null).length; const pathologicalThresholds = [10, 25, 100].map((threshold) => ({ threshold, count: records.filter((record) => record.inflationRatio !== null && record.inflationRatio >= threshold).length, })); const failureKinds = [...groupCounts(records.filter((record) => record.success === false), (record) => record.errorKind ?? "other").entries()] .map(([kind, count]) => ({ kind, count })) .sort((a, b) => b.count - a.count || a.kind.localeCompare(b.kind));
return { filters: { model: options.modelFilter ?? null, extension: options.extFilter ?? null, failedOnly: options.failedOnly, }, scan: { sessionsDir: meta.sessionsDir, sessionFilesScanned: meta.sessionFilesScanned, sessionFilesIncluded: meta.sessionFilesIncluded, sessionFilesSkippedOlderThanSince: meta.sessionFilesSkippedOlderThanSince, sessionFilesWithEditCalls: meta.sessionFilesWithEditCalls, since: meta.since ? { ms: meta.since.ms, iso: formatIso(meta.since.ms), source: meta.since.source } : null, malformedLines: meta.malformedLines, unmatchedToolResults: meta.unmatchedToolResults, }, counts: { assistantMessagesWithEditCalls: uniqueAssistantMessages, totalEditCalls: records.length, resolvedEditCalls: resolved.length, success, failed, unresolved, single, multi, noCoreChange, }, modeStats, multiEditLengthBuckets, argStyles, providerStats, extensionStats, inflation: { median: quantile(inflations, 0.5), p90: quantile(inflations, 0.9), p95: quantile(inflations, 0.95), p99: quantile(inflations, 0.99), pathologicalThresholds, hugeReplacements: buildHugeReplacementStats(records), failureByBucket: buildInflationBuckets(records), }, sameFileClusters: buildSameFileClusterStats(records), failureKinds, worstExamples: buildWorstExamples(records, options.top), };}
function printGroupTable(title, groups, formatter) { if (groups.length === 0) return; console.log(`\n${title}`); for (const group of groups) { console.log(formatter(group)); }}
function printHumanReport(summary) { const { scan, counts, modeStats, multiEditLengthBuckets, argStyles, providerStats, extensionStats, inflation, sameFileClusters, failureKinds, worstExamples, filters } = summary; console.log(`Scanned ${formatInt(scan.sessionFilesIncluded)} session files in ${scan.sessionsDir}`); if (scan.since) { console.log(`Session filter: files created at or after ${scan.since.iso} (${scan.since.source})`); console.log(`Skipped older session files: ${formatInt(scan.sessionFilesSkippedOlderThanSince)} of ${formatInt(scan.sessionFilesScanned)}`); } console.log(`Found ${formatInt(counts.totalEditCalls)} edit tool calls in ${formatInt(counts.assistantMessagesWithEditCalls)} assistant messages`); if (filters.model || filters.extension || filters.failedOnly) { const filterParts = []; if (filters.model) filterParts.push(`model contains \"${filters.model}\"`); if (filters.extension) filterParts.push(`extension = ${filters.extension}`); if (filters.failedOnly) filterParts.push("failed only"); console.log(`Filters: ${filterParts.join(", ")}`); }
console.log("\nSuccess rate"); console.log(` success: ${formatInt(counts.success)} ${formatPercent(counts.success, counts.resolvedEditCalls)}`); console.log(` failed: ${formatInt(counts.failed)} ${formatPercent(counts.failed, counts.resolvedEditCalls)}`); console.log(` unresolved: ${formatInt(counts.unresolved)}`);
console.log("\nMode usage"); console.log(` single replacement: ${formatInt(counts.single)} ${formatPercent(counts.single, counts.totalEditCalls)}`); console.log(` multi-edit (edits): ${formatInt(counts.multi)} ${formatPercent(counts.multi, counts.totalEditCalls)}`);
console.log("\nFailures by edit type"); for (const mode of modeStats) { console.log( ` ${mode.mode.padEnd(6)} calls=${formatInt(mode.calls).padStart(4)} success=${mode.successRate === null ? "n/a" : formatPercent(mode.success, mode.resolved).padStart(6)} failed=${mode.failureRate === null ? "n/a" : formatPercent(mode.failed, mode.resolved).padStart(6)} unresolved=${formatInt(mode.unresolved)}` ); }
console.log("\nMulti-edit bucket split"); for (const bucket of multiEditLengthBuckets) { console.log( ` ${bucket.label.padEnd(20)} ${formatInt(bucket.calls).padStart(4)} calls success=${bucket.successRate === null ? "n/a" : formatPercent(bucket.success, bucket.resolved).padStart(6)} failed=${bucket.failureRate === null ? "n/a" : formatPercent(bucket.failed, bucket.resolved).padStart(6)}` ); }
console.log("\nArgument style"); for (const entry of argStyles) { console.log(` ${entry.style.padEnd(22)} ${formatInt(entry.count).padStart(8)} ${formatPercent(entry.count, counts.totalEditCalls)}`); }
printGroupTable("By provider/model", providerStats, (group) => { return [ ` ${group.key}`, ` calls: ${formatInt(group.calls)}`, ` success: ${group.successRate === null ? "n/a" : formatPercent(group.success, group.resolved)}`, ` multi-edit: ${group.multiRate === null ? "n/a" : formatPercent(group.multi, group.calls)}`, ` median inflation: ${formatRatio(group.medianInflation)}`, ` p95 inflation: ${formatRatio(group.p95Inflation)}`, ].join("\n"); });
printGroupTable("By file extension", extensionStats, (group) => { return ` ${group.key.padEnd(10)} calls=${formatInt(group.calls).padStart(6)} success=${group.successRate === null ? "n/a" : formatPercent(group.success, group.resolved).padStart(6)} medianInflation=${formatRatio(group.medianInflation)}`; });
console.log("\nContext inflation"); console.log(` median inflation: ${formatRatio(inflation.median)}`); console.log(` p90 inflation: ${formatRatio(inflation.p90)}`); console.log(` p95 inflation: ${formatRatio(inflation.p95)}`); console.log(` p99 inflation: ${formatRatio(inflation.p99)}`); console.log(` no-core-change: ${formatInt(counts.noCoreChange)}`);
console.log("\nHuge replacements"); for (const entry of inflation.hugeReplacements) { console.log(` >${formatBytes(entry.threshold).padEnd(6)} ${formatInt(entry.count).padStart(8)}`); }
console.log("\nPathological wrappers"); for (const entry of inflation.pathologicalThresholds) { console.log(` inflation >= ${String(entry.threshold).padEnd(3)}x ${formatInt(entry.count).padStart(8)}`); }
console.log("\nSame-file multi-call behavior"); console.log(` assistant msgs with 2+ edit calls to same file: ${formatInt(sameFileClusters.assistantMessagesWithCluster)}`); console.log(` total same-file clusters: ${formatInt(sameFileClusters.clustersCount)}`); console.log(` calls inside those clusters: ${formatInt(sameFileClusters.callsInsideClusters)}`); console.log(` average calls per cluster: ${sameFileClusters.averageCallsPerCluster === null ? "n/a" : sameFileClusters.averageCallsPerCluster.toFixed(2)}`); console.log(` assistant msgs using one multi-edit call: ${formatInt(sameFileClusters.assistantMessagesWithMultiEdit)}`); console.log(` ratio multi-call / multi-edit assistant msgs: ${sameFileClusters.ratioClusterAssistantMessagesToMultiEditAssistantMessages === null ? "n/a" : sameFileClusters.ratioClusterAssistantMessagesToMultiEditAssistantMessages.toFixed(2)}`);
console.log("\nFailures by kind"); if (failureKinds.length === 0) { console.log(" none"); } else { for (const failure of failureKinds) { console.log(` ${failure.kind.padEnd(22)} ${formatInt(failure.count).padStart(8)}`); } }
console.log("\nFailure rate by inflation bucket"); for (const bucket of inflation.failureByBucket) { console.log(` ${bucket.label.padEnd(14)} ${bucket.failureRate === null ? "n/a" : formatPercent(bucket.failed, bucket.resolved).padStart(6)} (${formatInt(bucket.count)} calls)`); }
console.log(`\nWorst ${formatInt(worstExamples.length)} examples`); for (let i = 0; i < worstExamples.length; i++) { const example = worstExamples[i]; console.log( ` ${i + 1}. ${example.providerModel} ${example.extension} inflation=${formatRatio(example.inflationRatio)} failed=${example.failed ? example.errorKind : "false"}` ); console.log(` path: ${example.path}`); console.log(` totalEditBytes=${formatBytes(example.totalEditBytes)} coreBytes=${formatBytes(example.coreBytes)} mode=${example.mode} edits=${example.editsCount}`); }
if (scan.malformedLines > 0 || scan.unmatchedToolResults > 0) { console.log("\nParser notes"); if (scan.malformedLines > 0) console.log(` malformed lines skipped: ${formatInt(scan.malformedLines)}`); if (scan.unmatchedToolResults > 0) console.log(` unmatched edit tool results: ${formatInt(scan.unmatchedToolResults)}`); }}
async function scanSessions(sessionsDir, since) { const records = []; const meta = { sessionsDir, sessionFilesScanned: 0, sessionFilesIncluded: 0, sessionFilesSkippedOlderThanSince: 0, sessionFilesWithEditCalls: 0, since, malformedLines: 0, unmatchedToolResults: 0, };
for await (const sessionFile of walkJsonlFiles(sessionsDir)) { meta.sessionFilesScanned++; const sessionTimestampMs = parseSessionFileTimestamp(sessionFile); if (since && sessionTimestampMs !== null && sessionTimestampMs < since.ms) { meta.sessionFilesSkippedOlderThanSince++; continue; } meta.sessionFilesIncluded++; const pending = new Map(); let fileHadEditCall = false; const input = createReadStream(sessionFile, { encoding: "utf8" }); const rl = createInterface({ input, crlfDelay: Infinity });
for await (const line of rl) { if (!line.trim()) continue; let entry; try { entry = JSON.parse(line); } catch { meta.malformedLines++; continue; }
if (entry?.type !== "message" || !entry.message) continue; const message = entry.message;
if (message.role === "assistant" && Array.isArray(message.content)) { for (const block of message.content) { if (block?.type !== "toolCall" || block.name !== "edit") continue; fileHadEditCall = true; const analysis = analyzeToolArguments(block.arguments); const record = { sessionFile, assistantEntryId: entry.id, toolCallId: typeof block.id === "string" ? block.id : "", timestamp: entry.timestamp, api: typeof message.api === "string" ? message.api : null, provider: typeof message.provider === "string" ? message.provider : "[unknown]", model: typeof message.model === "string" ? message.model : "[unknown]", providerModel: `${typeof message.provider === "string" ? message.provider : "[unknown]"}/${typeof message.model === "string" ? message.model : "[unknown]"}`, success: null, errorKind: null, errorText: "", resultSummary: "", matchedResult: false, ...analysis, }; records.push(record); if (record.toolCallId) pending.set(record.toolCallId, record); } }
if (message.role === "toolResult" && message.toolName === "edit") { const toolCallId = typeof message.toolCallId === "string" ? message.toolCallId : ""; const record = pending.get(toolCallId); if (!record) { meta.unmatchedToolResults++; continue; } const text = extractTextContent(message.content); record.matchedResult = true; record.success = message.isError === true ? false : true; record.resultSummary = text; record.errorText = message.isError === true ? text : ""; record.errorKind = classifyErrorKind(text, message.isError === true, true); pending.delete(toolCallId); } }
for (const record of pending.values()) { record.matchedResult = false; record.success = null; record.errorKind = classifyErrorKind("", false, false); }
if (fileHadEditCall) meta.sessionFilesWithEditCalls++; }
return { records, meta };}
function applyFilters(records, options) { return records.filter((record) => { if (options.modelFilter && !record.providerModel.toLowerCase().includes(options.modelFilter.toLowerCase())) { return false; } if (options.extFilter && record.extension !== options.extFilter) { return false; } if (options.failedOnly && record.success !== false) { return false; } return true; });}
async function main() { const options = parseArgs(process.argv.slice(2)); if (options.help) { printHelp(); return; }
const sessionsDir = path.resolve(options.sessionsDir); await fs.access(sessionsDir); const since = await resolveAutoSinceMs(options);
const { records, meta } = await scanSessions(sessionsDir, since); const filteredRecords = applyFilters(records, options); const summary = buildSummary(filteredRecords, meta, options);
if (options.json) { const output = options.includeRecords ? { summary, records: filteredRecords } : { summary }; console.log(JSON.stringify(output, null, 2)); return; }
printHumanReport(summary);}
main().catch((error) => { console.error(error instanceof Error ? error.message : String(error)); process.exit(1);});