Something went wrong. Try again.
I'm going to build my own OpenClaw, with blackjack... and bun!
Something went wrong. Try again.
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198/** * tool-output.ts – High-level API for persisting and querying large tool outputs. * * When a tool call (bash, read, etc.) produces output too large to include * inline in the conversation, this module: * 1. Writes the full content to a log file on disk. * 2. Stores metadata in the `tool_outputs` DB table. * 3. Indexes the content in FTS5 for later searching. * 4. Returns a short preview/summary for the agent's context window. * * Also manages automatic retention cleanup (pruneToolOutputs). * * Consumers: * - tools/tracked-bash.ts calls saveToolOutput() for large bash outputs. * - tools/context-tools.ts calls getToolOutput() / searchToolOutput(). * - runtime.ts calls startToolOutputCleanup() at startup. * - agent-pool.ts may call readToolOutputFile() to fetch full content. */
import { existsSync, mkdirSync, readFileSync, unlinkSync, writeFileSync } from "fs";import { join } from "path";import { DATA_DIR } from "./core/config.js";import { createUuid } from "./utils/ids.js";import { createLogger, debugSuppressedError } from "./utils/logger.js";import { storeToolOutput, insertToolOutputChunk, getToolOutputById, deleteToolOutputsBefore, searchToolOutputSnippets, type ToolOutputRecord,} from "./db.js";import { buildPreviewLines } from "./utils/preview.js";
/** Directory where tool output log files are stored on disk. */const TOOL_OUTPUT_DIR = join(DATA_DIR, "tool-output");const log = createLogger("tool-output");/** Default chunk size (characters) for FTS indexing. */const DEFAULT_CHUNK_SIZE = 4000;
/** Options for saveToolOutput(). */export interface ToolOutputSaveOptions { /** Override the generated UUID. */ id?: string; /** Tool name that produced this output (e.g. "bash", "read"). */ source?: string | null; /** Pre-computed summary (if not provided, one is generated). */ summary?: string | null; /** Override the created_at timestamp. */ createdAt?: string;}
/** Result returned after successfully saving a tool output. */export interface ToolOutputSaveResult { /** UUID of the stored output. */ id: string; /** Filesystem path to the full log file. */ path: string; /** Short preview/summary of the output content. */ summary: string; /** Total size of the output in bytes. */ sizeBytes: number; /** Total number of lines in the output. */ lineCount: number;}
/** * Generate a short text preview of the first N lines of content. * Used as the default summary when none is provided. */export function buildPreview(text: string, maxLines = 12, maxLineLength = 200): string { const { preview } = buildPreviewLines(text, { maxLines, maxLineLength, includeOmittedLine: true, }); return preview;}
/** * Split text into line-aware chunks for FTS indexing. * Avoids splitting mid-line when possible. */function chunkText(text: string, chunkSize = DEFAULT_CHUNK_SIZE): string[] { const lines = text.replace(/\r\n/g, "\n").split("\n"); const chunks: string[] = []; let buffer = ""; for (const line of lines) { const next = buffer ? `${buffer}\n${line}` : line; if (next.length > chunkSize && buffer) { chunks.push(buffer); buffer = line; continue; } buffer = next; } if (buffer) chunks.push(buffer); return chunks;}
/** * Persist a tool output: write to disk, store metadata in SQLite, and * index content chunks in FTS5. Returns a summary result. */export function saveToolOutput(text: string, options: ToolOutputSaveOptions = {}): ToolOutputSaveResult { mkdirSync(TOOL_OUTPUT_DIR, { recursive: true }); const id = options.id ?? createUuid("out"); const createdAt = options.createdAt ?? new Date().toISOString(); const path = join(TOOL_OUTPUT_DIR, `${id}.log`); writeFileSync(path, text ?? "", "utf8");
const sizeBytes = Buffer.byteLength(text ?? "", "utf8"); const lineCount = text ? text.replace(/\r\n/g, "\n").split("\n").length : 0; const summary = options.summary ?? buildPreview(text ?? "");
const record: ToolOutputRecord = { id, created_at: createdAt, source: options.source ?? null, size_bytes: sizeBytes, line_count: lineCount, summary, path, };
storeToolOutput(record);
// Index content in FTS5 in chunks for efficient full-text search. const chunks = chunkText(text ?? ""); for (const chunk of chunks) { if (chunk.trim()) insertToolOutputChunk(id, chunk); }
return { id, path, summary, sizeBytes, lineCount };}
/** Retrieve a tool output record by its UUID handle. */export function getToolOutput(handle: string): ToolOutputRecord | undefined { return getToolOutputById(handle);}
/** Search the FTS index for a tool output, returning snippet strings. */export function searchToolOutput(handle: string, query: string, limit = 5): string[] { const trimmed = query?.trim?.() ?? ""; if (!trimmed) return []; return searchToolOutputSnippets(handle, trimmed, limit);}
/** * Delete tool outputs older than `maxAgeDays`. Removes both the DB records * and the on-disk log files. Returns the number of records pruned. */export function pruneToolOutputs(maxAgeDays = 30): number { const cutoff = new Date(Date.now() - maxAgeDays * 24 * 60 * 60 * 1000).toISOString(); const rows = deleteToolOutputsBefore(cutoff); for (const row of rows) { if (row.path && existsSync(row.path)) { try { unlinkSync(row.path); } catch (err) { debugSuppressedError(log, "Failed to unlink a pruned tool-output file; it may already be gone.", err, { operation: "tool_output.prune.unlink", path: row.path, id: row.id, }); } } } return rows.length;}
/** Guard to ensure the cleanup interval is only started once. */let cleanupStarted = false;
/** * Start a periodic timer that prunes old tool outputs. * Called once by runtime.ts during startup. */export function startToolOutputCleanup(maxAgeDays = 30, intervalMs = 12 * 60 * 60 * 1000): void { if (cleanupStarted) return; cleanupStarted = true; // Run an initial prune immediately, then on a recurring interval. pruneToolOutputs(maxAgeDays); setInterval(() => pruneToolOutputs(maxAgeDays), intervalMs).unref();}
/** * Read the full content of a tool output log file from disk. * Returns null if the file doesn't exist or is unreadable. */export function readToolOutputFile(path: string): string | null { try { return readFileSync(path, "utf8"); } catch { return null; }}