diff --git a/captutor/README.md b/captutor/README.md index 05e9e2a4f3..aa64e7d43d 100644 --- a/captutor/README.md +++ b/captutor/README.md @@ -3,6 +3,11 @@ Screen-recorded, narrated, captioned software tutorials for fuser — assembled from tooling AC already had, plus one new primitive. +Native audit captures use Frame in process and run concurrently with post-action +DOM inspection. Custom `CAPTUTOR_FRAME` CLI overrides remain supported. See +[computer-use latency](../slab/COMPUTER-USE-PERFORMANCE.md) for measurements and +the isolated browser benchmark. + ``` node captutor.mjs narrate # just the voice — check pacing (cheap) node captutor.mjs render # narrate → record → drive → mp4 + vtt diff --git a/captutor/captutor.mjs b/captutor/captutor.mjs index 24166bc5d1..6a61f2145e 100644 --- a/captutor/captutor.mjs +++ b/captutor/captutor.mjs @@ -66,6 +66,7 @@ import { DirectorChannel, directorBeatState, resolveDirectorGoal, } from "./lib/director-channel.mjs"; import { masterPopDelivery } from "./lib/pop-audio-master.mjs"; +import { createFrameClient } from "./lib/frame-client.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); @@ -82,6 +83,7 @@ const FRAME = process.env.CAPTUTOR_FRAME || (existsSync(INSTALLED_FRAME) ? INSTALLED_FRAME : existsSync(REPO_FRAME) ? REPO_FRAME : join(resolve(HERE, "../../.."), "slab", "bin", "frame.mjs")); +const captureNativeFrame = createFrameClient(FRAME, { cli: Boolean(process.env.CAPTUTOR_FRAME) }); const FUSER = process.env.FUSER_REPO || `${process.env.HOME}/Developer/fuser`; const DOCS_PUBLIC = join(FUSER, "apps", "docs", "public"); @@ -116,11 +118,9 @@ const PASSIVE_FRAME_AUDIT = STAGE_MODE // capture can film one. Retire every Frame-owned transient immediately before // the reel starts. If Frame is installed, failure is a capture-safety failure: // it is better to abort a take than ship tooling UI inside the tutorial. -function clearFrameOverlays() { +async function clearFrameOverlays() { if (!existsSync(FRAME)) return; - execFileSync(process.execPath, [ - FRAME, "local", "--clear-overlays", "--quiet-overlay", "--no-ocr", "--json", - ], { encoding: "utf8", timeout: 10_000, stdio: ["ignore", "pipe", "pipe"] }); + await captureNativeFrame({ clearOverlays: true }); } // Some filming seats (notably clamshell Macs on native-only external panels) @@ -448,7 +448,7 @@ async function cmdRender(sp, workDir, locale, format, attempt = 1) { }; const frameAuditDir = join(workDir, "frame-audit", String(Date.now())); let frameAuditSequence = 0; - const passiveFrameAudit = (kind) => { + const passiveFrameAudit = async (kind) => { if (!PASSIVE_FRAME_AUDIT || !["click", "drag", "type"].includes(kind)) return null; if (!existsSync(FRAME)) { throw new Error("passive Frame audit is required in Stage Mode but Frame is unavailable"); @@ -458,10 +458,7 @@ async function cmdRender(sp, workDir, locale, format, attempt = 1) { const image = join(frameAuditDir, `${sequence}-${kind}.jpg`); let envelope; try { - envelope = JSON.parse(execFileSync(process.execPath, [ - FRAME, "local", "--screen", "--no-ocr", "--quiet-overlay", - "--out", image, "--json", - ], { encoding:"utf8", timeout:15_000, stdio:["ignore", "pipe", "pipe"] })); + envelope = await captureNativeFrame({ screen: true, out: image }); } catch (error) { throw new Error(`passive Frame audit failed after ${kind}: ${error.message}`); } @@ -483,8 +480,8 @@ async function cmdRender(sp, workDir, locale, format, attempt = 1) { let frameAudit = null; try { const result = await action(); - const after = await uiSnapshot(); - frameAudit = passiveFrameAudit(kind); + const [after, audit] = await Promise.all([uiSnapshot(), passiveFrameAudit(kind)]); + frameAudit = audit; const forbiddenRoute = (sp.forbiddenRouteFragments || []).find((fragment) => { try { return new URL(after.url).pathname.includes(fragment); } catch { return false; } @@ -732,7 +729,7 @@ async function cmdRender(sp, workDir, locale, format, attempt = 1) { await cdp.send("Page.bringToFront"); await sleep(600); - clearFrameOverlays(); + await clearFrameOverlays(); // /json retains the original Fuser URL and title after a renderer dies, so // those fields are not a health check. Require the page itself to answer just diff --git a/captutor/lib/frame-client.mjs b/captutor/lib/frame-client.mjs new file mode 100644 index 0000000000..44dd28f5c0 --- /dev/null +++ b/captutor/lib/frame-client.mjs @@ -0,0 +1,41 @@ +import { execFile } from "node:child_process"; +import { mkdir, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import { pathToFileURL } from "node:url"; +import { promisify } from "node:util"; + +const exec = promisify(execFile); + +// Keep custom CAPTUTOR_FRAME executables compatible. The house module is +// imported once and shares Frame's lease and transport without blocking CDP, +// pointer animation, or director messages behind a child process. +export function createFrameClient(path, { cli = false } = {}) { + let module; + return async function capture({ out, screen = false, clearOverlays = false } = {}) { + let env; + if (cli) { + const args = [path, "local", "--no-ocr", "--quiet-overlay", "--json"]; + if (screen) args.push("--screen"); + if (clearOverlays) args.push("--clear-overlays"); + if (out) args.push("--out", out); + const { stdout } = await exec(process.execPath, args, { timeout: 15000, maxBuffer: 4 * 1024 * 1024 }); + env = JSON.parse(stdout); + } else { + module ||= import(pathToFileURL(path).href); + const { captureFrame } = await module; + const result = await captureFrame("local", { + memory: true, noOCR: true, quietOverlay: true, screen, clearOverlays, + }); + env = result.env; + if (out && env.capture === "ok" && result.jpg?.length) { + await mkdir(dirname(out), { recursive: true }); + await writeFile(out, result.jpg); + } else if (out && env.capture === "ok") { + throw new Error("Frame returned no pixels"); + } + } + if (out && env.capture !== "ok") throw new Error(`Frame capture failed: ${env.capture}`); + if (clearOverlays && env.capture !== "action") throw new Error(`Frame overlay clear failed: ${env.capture}`); + return env; + }; +} diff --git a/captutor/test/frame-client.test.mjs b/captutor/test/frame-client.test.mjs new file mode 100644 index 0000000000..c9298265cd --- /dev/null +++ b/captutor/test/frame-client.test.mjs @@ -0,0 +1,59 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { createFrameClient } from "../lib/frame-client.mjs"; + +test("Captutor imports Frame once, yields during capture, and writes fresh audit pixels", async t => { + const dir = await mkdtemp(join(tmpdir(), "captutor-frame-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const path = join(dir, "frame.mjs"), out = join(dir, "audit", "frame.jpg"); + await writeFile(path, ` + let calls = 0; + export async function captureFrame(machine, options) { + await new Promise(resolve => setTimeout(resolve, 30)); + if (machine !== 'local' || !options.noOCR || !options.quietOverlay || !options.memory) + throw new Error('incorrect capture options'); + return { env: { capture: options.clearOverlays ? 'action' : 'ok', calls: ++calls }, jpg: Buffer.from('fresh') }; + } + `); + const capture = createFrameClient(path); + let ticks = 0; + const timer = setInterval(() => ticks++, 5); + try { + const first = await capture({ screen: true, out }); + assert.equal(first.calls, 1); + assert.equal(await readFile(out, "utf8"), "fresh"); + assert.ok(ticks > 0, "capture must not block CDP/director messages"); + assert.equal((await capture({ clearOverlays: true })).calls, 2); + } finally { clearInterval(timer); } +}); + +test("Captutor rejects denied or empty captures even when an old audit file exists", async t => { + const dir = await mkdtemp(join(tmpdir(), "captutor-frame-error-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const out = join(dir, "frame.jpg"); + await writeFile(out, "stale"); + for (const status of ["permission_needed", "ok"]) { + const path = join(dir, `${status}.mjs`); + await writeFile(path, `export async function captureFrame() { return { env: { capture: '${status}' } }; }`); + await assert.rejects(createFrameClient(path)({ out }), /capture failed|no pixels/); + assert.equal(await readFile(out, "utf8"), "stale"); + } +}); + +test("Captutor preserves custom Frame CLI overrides asynchronously", async t => { + const dir = await mkdtemp(join(tmpdir(), "captutor-frame-cli-")); + t.after(() => rm(dir, { recursive: true, force: true })); + const path = join(dir, "custom.mjs"), out = join(dir, "frame.jpg"); + await writeFile(path, ` + import {writeFile} from 'node:fs/promises'; + await new Promise(resolve => setTimeout(resolve, 20)); + await writeFile(process.argv[process.argv.indexOf('--out') + 1], 'cli-pixels'); + console.log(JSON.stringify({capture:'ok', args:process.argv.slice(2)})); + `); + const result = await createFrameClient(path, { cli: true })({ screen: true, out }); + assert.deepEqual(result.args, ["local", "--no-ocr", "--quiet-overlay", "--json", "--screen", "--out", out]); + assert.equal(await readFile(out, "utf8"), "cli-pixels"); +}); diff --git a/memory/memory-mcp.mjs b/memory/memory-mcp.mjs index 9b5cb0ca71..82caccbd62 100644 --- a/memory/memory-mcp.mjs +++ b/memory/memory-mcp.mjs @@ -24,6 +24,7 @@ import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { httpPort, serveHttp, serveStdio } from "../toolchain/mcp/http-front.mjs"; +import { shortWhen, toon } from "../shared/toon.mjs"; const HERE = dirname(fileURLToPath(import.meta.url)); const HOME = homedir(); @@ -269,15 +270,18 @@ async function toolSearch(a) { hits.sort((a, b) => new Date(b.when || 0) - new Date(a.when || 0)); - lines.push(`${hits.length} match${hits.length === 1 ? "" : "es"} for "${a.query}" across ${results.length - failures.length} machine(s)`); - if (failures.length) lines.push(`\nunreachable (their history is NOT in these results):\n${failures.join("\n")}`); - lines.push(""); - - for (const hit of hits) { - lines.push(`${hit.when} [${hit.machine}] ${hit.role || "?"} ${hit.session_id}#${hit.seq}`); - lines.push(` ${hit.snippet}`); - } - if (!hits.length) lines.push("(nothing — try fewer words, or --regex, or widen --since)"); + // One TOON row per hit; `ref` is session#seq, what memory_search takes back. + const rows = hits.map((hit) => ({ + when: shortWhen(hit.when), + machine: hit.machine, + role: hit.role || "?", + ref: `${hit.session_id}#${hit.seq}`, + snippet: hit.snippet, + })); + const notes = [`query "${a.query}" over ${results.length - failures.length} machine(s)`]; + if (failures.length) notes.push(`UNREACHABLE (not searched): ${failures.map((f) => f.trim()).join("; ")}`); + if (!hits.length) notes.push("nothing — try fewer words, regex:true, or a wider since"); + lines.push(toon("hits", rows, ["when", "machine", "role", "ref", "snippet"], { note: notes.join(" · ") })); return [{ type: "text", text: lines.join("\n") }]; } @@ -285,19 +289,24 @@ async function toolSearch(a) { async function toolSessions(a) { const args = ["list", ...flag("limit", a.limit ?? 15), ...flag("project", a.project)]; const results = await fanOut(a.machine, args); - const lines = []; + const rows = []; + const failures = []; for (const entry of results) { if (entry.error) { - lines.push(`[${entry.name}] unreachable: ${entry.error}`); + failures.push(`${entry.name}: ${entry.error}`); continue; } - lines.push(`[${entry.name}]`); + const machine = entry.name === "local" ? SELF : entry.name; for (const s of entry.result || []) { - lines.push(` ${s.updated_at} ${s.session_id} seq=${s.last_seq} project=${s.project} "${s.title}"`); + rows.push({ machine, updated: shortWhen(s.updated_at), sortTime: Date.parse(s.updated_at) || 0, id: s.session_id, seq: s.last_seq, project: s.project, title: s.title }); } } - return [{ type: "text", text: lines.join("\n") || "(no sessions)" }]; + rows.sort((x, y) => y.sortTime - x.sortTime); + const notes = []; + if (failures.length) notes.push(`UNREACHABLE (not listed): ${failures.join("; ")}`); + notes.push("search inside one: memory_search with query (+ project/since)"); + return [{ type: "text", text: toon("sessions", rows, ["machine", "updated", "id", "seq", "project", "title"], { note: notes.join(" · ") }) }]; } async function toolMachines() { diff --git a/shared/toon.mjs b/shared/toon.mjs new file mode 100644 index 0000000000..04ccf930e7 --- /dev/null +++ b/shared/toon.mjs @@ -0,0 +1,87 @@ +// toon.mjs — Token-Oriented Object Notation for tool replies. +// +// The house shape for any MCP tool that answers with a list. One header +// names the collection, its count and its columns; every row after it is +// bare comma-separated cells. Against the prose-and-bullet lists it replaces +// this reads at roughly two thirds the tokens, and the header makes an empty +// result definitive ("rocks[0]") instead of a sentence the model has to parse. +// +// machines[4]{name,role,live}: +// neo,control,up +// blueberry,control,self +// +// Rules the rows follow (kept small enough to hold in the head): +// • a cell is quoted only when it contains a comma, a quote, a newline, or +// an edge space; an inner quote doubles ("") +// • null and undefined print as nothing; numbers and booleans as themselves +// • `[N of T]` marks a capped list so the caller knows there is more +// +// Node builtins only — this file is imported by the resident MCP daemons. + +/** Escape one cell for a TOON row. */ +export function cell(value) { + if (value === null || value === undefined) return ""; + const s = typeof value === "string" ? value : String(value); + return /[",\n\r]|^\s|\s$/.test(s) ? `"${s.replace(/"/g, '""')}"` : s; +} + +/** + * Render rows as a TOON block. + * name collection name shown in the header ("papers") + * rows array of plain objects + * fields ordered column names; each row is read by these keys + * total optional true size when `rows` is a capped slice + * note optional trailing line (where to get more, what was hidden) + */ +export function toon(name, rows, fields, { total, note } = {}) { + const n = rows.length; + const count = total !== undefined && total !== n ? `${n} of ${total}` : String(n); + const lines = [`${name}[${count}]{${fields.join(",")}}:`]; + for (const row of rows) lines.push(" " + fields.map((f) => cell(row[f])).join(",")); + if (note) lines.push(note); + return lines.join("\n"); +} + +/** Clip a string with a size hint, so the reader knows how much was cut. */ +export function clip(text, max = 80) { + const s = String(text ?? "").replace(/\s+/g, " ").trim(); + return s.length <= max ? s : `${s.slice(0, max - 1)}…(+${s.length - max + 1}ch)`; +} + +/** + * Shorten an absolute path against named roots: { $AC: "/Users/x/ac" }. + * The longest matching root wins, so a vault inside the repo prints as + * $VAULT/… rather than $AC/aesthetic-computer-vault/…. + */ +export function shortPath(p, roots = {}) { + if (!p) return ""; + const s = String(p); + const hit = Object.entries(roots) + .filter(([, root]) => root && (s === root || s.startsWith(root.endsWith("/") ? root : root + "/"))) + .sort((a, b) => b[1].length - a[1].length)[0]; + if (!hit) return s; + const [label, root] = hit; + return label + s.slice(root.replace(/\/$/, "").length); +} + +/** Compact age: 42s, 7m, 3h, 5d. */ +export function ago(when, now = Date.now()) { + const t = typeof when === "number" ? when : Date.parse(when); + if (!Number.isFinite(t)) return "?"; + const s = Math.max(0, Math.round((now - t) / 1000)); + if (s < 90) return `${s}s`; + const m = Math.round(s / 60); + if (m < 90) return `${m}m`; + const h = Math.round(m / 60); + if (h < 48) return `${h}h`; + return `${Math.round(h / 24)}d`; +} + +/** ISO timestamp → "09-20 14:05" (drops the year and seconds), or "" */ +export function shortWhen(iso) { + if (!iso) return ""; + const s = String(iso); + const m = s.match(/^(\d{4})-(\d{2})-(\d{2})(?:[T ](\d{2}):(\d{2}))?/); + if (!m) return s; + return m[4] ? `${m[2]}-${m[3]} ${m[4]}:${m[5]}` : `${m[2]}-${m[3]}`; +} diff --git a/shared/toon.test.mjs b/shared/toon.test.mjs new file mode 100644 index 0000000000..0e3165ac6f --- /dev/null +++ b/shared/toon.test.mjs @@ -0,0 +1,48 @@ +import { test } from "node:test"; +import assert from "node:assert/strict"; +import { ago, cell, clip, shortPath, shortWhen, toon } from "./toon.mjs"; + +test("toon header, rows, cap and note", () => { + const out = toon("papers", [{ id: "a", title: "Hello, world", n: 2 }, { id: "b", title: null, n: false }], + ["id", "title", "n"], { total: 40, note: "more: full:true" }); + assert.equal(out, 'papers[2 of 40]{id,title,n}:\n a,"Hello, world",2\n b,,false\nmore: full:true'); +}); + +test("toon empty list is definitive", () => { + assert.equal(toon("rocks", [], ["host", "name"]), "rocks[0]{host,name}:"); +}); + +test("cell quotes only when it must", () => { + assert.equal(cell("plain"), "plain"); + assert.equal(cell('say "hi"'), '"say ""hi"""'); + assert.equal(cell(" edge"), '" edge"'); + assert.equal(cell("a\nb"), '"a\nb"'); + assert.equal(cell(undefined), ""); + assert.equal(cell(0), "0"); +}); + +test("clip keeps short text and hints at the cut", () => { + assert.equal(clip("short", 10), "short"); + assert.equal(clip("abcdefghijkl", 8), "abcdefg…(+5ch)"); + assert.equal(clip(" a b ", 10), "a b"); +}); + +test("shortPath picks the longest root", () => { + const roots = { $AC: "/u/ac", $VAULT: "/u/ac/vault" }; + assert.equal(shortPath("/u/ac/vault/p.tex", roots), "$VAULT/p.tex"); + assert.equal(shortPath("/u/ac/system/x.mjs", roots), "$AC/system/x.mjs"); + assert.equal(shortPath("/elsewhere/x", roots), "/elsewhere/x"); + assert.equal(shortPath("/u/acme/x", roots), "/u/acme/x"); +}); + +test("ago and shortWhen", () => { + const now = Date.parse("2026-09-20T12:00:00Z"); + assert.equal(ago(now - 30_000, now), "30s"); + assert.equal(ago(now - 5 * 60_000, now), "5m"); + assert.equal(ago(now - 3 * 3_600_000, now), "3h"); + assert.equal(ago(now - 5 * 86_400_000, now), "5d"); + assert.equal(ago("garbage", now), "?"); + assert.equal(shortWhen("2026-09-20T14:05:33Z"), "09-20 14:05"); + assert.equal(shortWhen("2026-09-20"), "09-20"); + assert.equal(shortWhen(""), ""); +}); diff --git a/slab/COMPUTER-USE-PERFORMANCE.md b/slab/COMPUTER-USE-PERFORMANCE.md new file mode 100644 index 0000000000..f3e0c0e7dc --- /dev/null +++ b/slab/COMPUTER-USE-PERFORMANCE.md @@ -0,0 +1,56 @@ +# Computer-use latency + +Measured on Blueberry, September 21, 2026. The browser fixture is an isolated +headless Chrome page with one input and one button. These are local tool +execution times; they exclude model inference, remote network latency, and +application work. Seven warm samples per browser operation, after one first call. + +| Operation | Before median | After median | +| --- | ---: | ---: | +| Puppet semantic snapshot | 2.22 ms | 1.67 ms | +| Puppet fill | 3.49 ms | 2.86 ms | +| Puppet click with verification | 32.02 ms | 33.74 ms | +| Frame transport through CLI / in process (mock capture) | 35.33 ms | 0.53 ms | + +Click timing remains around 33 ms: Playwright still waits for actionability and +checks the postcondition. The inspector connection is reused, but an exact +target lookup still runs on every operation. A cached page's `isClosed()` alone +is insufficient: another client's close event can arrive late. + +Captutor now imports Frame once, awaits capture without blocking its event +loop, and obtains post-action DOM evidence and the passive audit concurrently. +Each consequential action still waits for its audit. Custom `CAPTUTOR_FRAME` +CLI overrides remain supported through an asynchronous child process. + +Local Frame requests use the native request/done files directly under the same +machine lease. Requests are published atomically; timeouts fail without reading +old pixels or retrying input. Remote SSH transport is unchanged. The transport +comparison above uses a mock capture and measures process overhead, not native +screenshot latency. + +One initial live Frame call without OCR completed in 536 ms. Subsequent native +benchmarks encountered the login screen and `permission_needed`; they are +excluded. Successful native capture/OCR medians and a complete Captutor recording +remain unmeasured in this session. + +Puppet's sample page list fell from 357 to 230 bytes (36%). Exact page IDs and +URLs remain intact; `full:true` returns the original JSON state. These are byte +counts, not tokenizer measurements. Earlier list compaction also covers fleet, +prox, papers, calendar, and memory. All-day event end dates and memory ordering +across years are preserved. + +Reproduce browser timings (no user tabs or native input): + +```sh +node slab/bin/computer-use-bench.mjs +``` + +With an unlocked local display, add `--native` for five quiet captures per OCR +mode. The report includes native stage timings and excludes failed captures; +it stores no screen text or images. + +Focused regression checks: + +```sh +node --test --test-concurrency=1 shared/toon.test.mjs slab/test/frame-local.test.mjs slab/test/computer-use-transport.test.mjs slab/test/computer-use-semantic.test.mjs slab/test/computer-use-protocol.test.mjs slab/test/computer-use-isolation.test.mjs captutor/test/frame-client.test.mjs +``` diff --git a/slab/bin/cal-mcp.mjs b/slab/bin/cal-mcp.mjs index d0b3da0ba5..92033bedb1 100755 --- a/slab/bin/cal-mcp.mjs +++ b/slab/bin/cal-mcp.mjs @@ -23,6 +23,7 @@ // no SDK, only node builtins + the shared http-front. import { httpPort, serveHttp, serveStdio } from "../../toolchain/mcp/http-front.mjs"; import { UA, loadTokens } from "../../toolchain/mcp/ac-token.mjs"; +import { clip, toon } from "../../shared/toon.mjs"; const API = "https://aesthetic.computer/api/cal"; const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); @@ -81,7 +82,7 @@ function fmtEvent(ev) { } // ── tools ───────────────────────────────────────────────────────────────────── -async function toolList({ from, to } = {}) { +async function toolList({ from, to, full = false } = {}) { const range = new URLSearchParams(); if (from) range.set("from", from); if (to) range.set("to", to); @@ -96,10 +97,28 @@ async function toolList({ from, to } = {}) { const events = [...mine, ...fed]; if (!events.length) { const label = from || to ? ` in [${from || "…"} → ${to || "…"}]` : " this month"; - return [{ type: "text", text: `(no AesthetiCal events${label})` }]; + return [{ type: "text", text: `events[0]: no AesthetiCal events${label}` }]; } events.sort((a, b) => String(a.start).localeCompare(String(b.start))); - return [{ type: "text", text: `${events.length} event(s):\n\n${events.map(fmtEvent).join("\n")}` }]; + if (full) return [{ type: "text", text: `${events.length} event(s):\n\n${events.map(fmtEvent).join("\n")}` }]; + // Compact table: uid is what cal_update / cal_delete take; flags fold the + // rare columns (visibility other than private, repeat, read-only feed, note). + const rows = events.map((ev) => ({ + uid: ev.uid, + start: ev.allDay ? String(ev.start || "").slice(0, 10) : ev.start, + end: ev.allDay ? String(ev.end || "").slice(0, 10) : ev.end, + title: ev.title || "(untitled)", + flags: [ + ev.allDay ? "allday" : "", + ev.visibility && ev.visibility !== "private" ? ev.visibility : "", + ev.rrule ? `repeat:${ev.rrule}` : "", + ev.readOnly ? "feed" : "", + ev.note ? `note:${clip(ev.note, 40)}` : "", + ].filter(Boolean).join(" "), + })); + return [{ type: "text", text: toon("events", rows, ["uid", "start", "end", "title", "flags"], { + note: "full notes and per-event detail: full:true", + }) }]; } async function toolAdd(args = {}) { @@ -166,6 +185,7 @@ const TOOLS = [ properties: { from: { type: "string", description: "Range start, ISO 8601 (e.g. 2026-07-01). Omit for the current month." }, to: { type: "string", description: "Range end, ISO 8601. Omit for the current month." }, + full: { type: "boolean", description: "Per-event detail with whole notes. Default is a compact uid/start/end/title/flags table." }, }, }, }, diff --git a/slab/bin/computer-use-bench.mjs b/slab/bin/computer-use-bench.mjs new file mode 100644 index 0000000000..1f725a7544 --- /dev/null +++ b/slab/bin/computer-use-bench.mjs @@ -0,0 +1,66 @@ +#!/usr/bin/env node +// Bounded, local-only measurements. No model calls, user tabs, or native input. +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { chromium } from "playwright-core"; +import { SemanticBrowser } from "../lib/puppet-semantic.mjs"; +import { captureFrame } from "./frame.mjs"; + +const summarize = values => { + const sorted = [...values].sort((a, b) => a - b); + return { samples: sorted.length, medianMs: sorted[Math.floor(sorted.length / 2)], maxMs: sorted.at(-1) }; +}; +const round = n => +n.toFixed(2); +const report = { measuredAt: new Date().toISOString(), browser: {}, native: [] }; +const dir = await mkdtemp(join(tmpdir(), "computer-use-bench-")); +let context, service; +try { + context = await chromium.launchPersistentContext(dir, { + channel: "chrome", headless: true, args: ["--remote-debugging-port=0"], + }); + const port = (await readFile(join(dir, "DevToolsActivePort"), "utf8")).split("\n")[0]; + service = new SemanticBrowser(() => `http://127.0.0.1:${port}`); + const page = context.pages()[0]; + await page.setContent(`Pending`); + const cdp = await context.newCDPSession(page); + const { targetInfo } = await cdp.send("Target.getTargetInfo"); + await cdp.detach(); + const target = targetInfo.targetId; + for (const [action, args] of [ + ["snapshot", {}], + ["fill", { locator: { label: "Name" }, value: "sample" }], + ["click", { locator: { role: "button", name: "Save" }, after: { locator: { text: "Saved" } } }], + ]) { + const times = []; + for (let i = 0; i < 8; i++) { + const start = performance.now(); + const result = await service.run(action, { target, ...args }); + if (result.verification?.ok === false) throw new Error(`fixture ${action} did not verify`); + times.push(round(performance.now() - start)); + } + report.browser[action] = { firstMs: times.shift(), ...summarize(times) }; + } +} finally { + await service?.close(); + await context?.close(); + await rm(dir, { recursive: true, force: true }); +} + +// Only successful captures count. Locked displays and missing grants must not +// look like low-latency screenshots. This records no pixels or screen text. +if (process.argv.includes("--native")) { + for (const [mode, options] of [["no-ocr", { noOCR: true }], ["fast-ocr", { fast: true }], ["accurate-ocr", {}]]) { + const times = [], stages = []; + let skipped; + for (let i = 0; i < 5; i++) { + const start = performance.now(); + const { env, jpg } = await captureFrame("local", { ...options, memory: true, quietOverlay: true }); + if (env.capture !== "ok" || !jpg?.length) { skipped = env.capture; break; } + times.push(round(performance.now() - start)); + stages.push(env.timings_ms); + } + report.native.push({ mode, ...summarize(times), ...(skipped ? { skipped } : {}), stages }); + } +} +console.log(JSON.stringify(report, null, 2)); diff --git a/slab/bin/frame.mjs b/slab/bin/frame.mjs index 6adda12d47..0ba7b38c62 100755 --- a/slab/bin/frame.mjs +++ b/slab/bin/frame.mjs @@ -32,6 +32,7 @@ import { dirname, join, resolve } from "node:path"; import { tmpdir } from "node:os"; import { fileURLToPath } from "node:url"; import { withMachineLease } from "../lib/computer-use-lease.mjs"; +import { localFrame } from "../lib/frame-local.mjs"; const HOME = process.env.HOME; const CONFIG_PATH = @@ -486,7 +487,11 @@ async function captureFrameUnlocked(name, { memory = false, session, expectedTar } } if (!frame) { - try { frame = await directFrame(name, machines, mode); } catch (e) { + try { + frame = machines[name]?.local && !direct + ? await localFrame(join(HOME, ".local", "share", "slab", "state"), mode) + : await directFrame(name, machines, mode); + } catch (e) { throw new Error(`${name} unreachable: ${String(e.message || e).split("\n")[0]}`); } } diff --git a/slab/bin/paper-mcp.mjs b/slab/bin/paper-mcp.mjs index 5a378e5f05..03410ca652 100755 --- a/slab/bin/paper-mcp.mjs +++ b/slab/bin/paper-mcp.mjs @@ -20,6 +20,7 @@ import { fileURLToPath } from "node:url"; import { promisify } from "node:util"; import { httpPort, serveHttp, serveStdio } from "../../toolchain/mcp/http-front.mjs"; import { createSourceBundle } from "../../papers/source-bundle.mjs"; +import { toon } from "../../shared/toon.mjs"; const pexec = promisify(execFile); const SCRIPT_DIR = dirname(fileURLToPath(import.meta.url)); @@ -236,7 +237,10 @@ function fmtPaper(rec) { ].join("\n"); } -async function toolList({ query, scope, kind, limit = 30 } = {}) { +// The default list is a TOON table of id, kind, date, title — enough to pick a +// paper. Paths repeat the same long roots thirty times over, so they come only +// with `full` or from paper_find for one paper. +async function toolList({ query, scope, kind, limit = 30, full = false } = {}) { let rows = await catalog(); if (scope) rows = rows.filter((rec) => rec.scope === String(scope).toLowerCase()); if (kind) rows = rows.filter((rec) => rec.kind === String(kind).toLowerCase()); @@ -247,8 +251,21 @@ async function toolList({ query, scope, kind, limit = 30 } = {}) { .map((item) => item.rec); } const capped = rows.slice(0, Math.max(1, Math.min(100, Number(limit) || 30))); - if (!capped.length) return [{ type: "text", text: "(no papers match)" }]; - return [{ type: "text", text: `${capped.length}${rows.length > capped.length ? ` of ${rows.length}` : ""} paper(s):\n\n${capped.map(fmtPaper).join("\n\n")}` }]; + if (full) { + if (!capped.length) return [{ type: "text", text: "papers[0]: no papers match" }]; + return [{ type: "text", text: `${capped.length}${rows.length > capped.length ? ` of ${rows.length}` : ""} paper(s):\n\n${capped.map(fmtPaper).join("\n\n")}` }]; + } + const table = capped.map((rec) => ({ + id: rec.id, + kind: rec.kind, + pdf: rec.pdfPath ? "pdf" : "", + updated: fmtDate(rec.updatedAt), + title: rec.title, + })); + const note = capped.length + ? "paths for one paper: paper_find · every path: full:true" + : "no papers match — try a shorter query or drop the scope/kind filter"; + return [{ type: "text", text: toon("papers", table, ["id", "kind", "pdf", "updated", "title"], { total: rows.length, note }) }]; } async function toolFind({ paper } = {}) { @@ -620,6 +637,7 @@ const TOOLS = [ scope: { type: "string", enum: ["public", "private", "vault", "fuser", "configured"], description: "Optional paper-root filter." }, kind: { type: "string", enum: ["tex", "md", "pdf"], description: "Optional preferred-source kind." }, limit: { type: "integer", minimum: 1, maximum: 100, default: 30 }, + full: { type: "boolean", description: "Include source and PDF paths for every row. Default is a compact id/kind/pdf/date/title table; paper_find returns one paper's paths." }, }, }, }, diff --git a/slab/bin/prox-mcp.mjs b/slab/bin/prox-mcp.mjs index d33c994f6e..360358bd49 100755 --- a/slab/bin/prox-mcp.mjs +++ b/slab/bin/prox-mcp.mjs @@ -26,6 +26,7 @@ import { promisify } from "node:util"; import { join } from "node:path"; import { homedir, hostname } from "node:os"; import { httpPort, serveHttp, serveStdio } from "../../toolchain/mcp/http-front.mjs"; +import { clip, toon } from "../../shared/toon.mjs"; const pexec = promisify(execFile); const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); @@ -126,21 +127,6 @@ function age(ms) { return `${Math.round(m / 60)}h`; } -const STATUS_MARK = { - working: "●", awaiting: "◐", complete: "○", - rendering: "◍", blank: "·", interrupted: "✕", -}; - -function line(r) { - const mark = STATUS_MARK[r.status] || "•"; - const subj = (r.subject || "").replace(/\s+/g, " ").slice(0, 64); - // Tag the owning agent when it isn't the default (Claude), so a mixed - // fleet reads clearly: "session·codex". - const agent = r.agentType && r.agentType !== "claude" ? `·${r.agentType}` : ""; - const alias = r.proxName ? ` prox:easel:${r.proxName}` : ""; - return `${mark} ${r.host}:${r.name}${alias} [${r.status}] ${r.kind}${agent} ·${age(r.updated)} ${subj}`; -} - // ── resolve a `host:name` / bare-name / fuzzy handle to rock rows ──────────── function resolve(rocks, handle) { if (!handle) return rocks; @@ -225,23 +211,42 @@ end tell`; } // ── tools ───────────────────────────────────────────────────────────────────── -async function toolList({ host, status, kind, agent } = {}) { +// Finished rocks that have not moved in a day are ledger residue, not fleet +// state. The default list hides them and says how many it hid; `all` shows +// every row and an explicit `status` filter is never second-guessed. +const STALE_MS = 24 * 3600 * 1000; +const FINISHED = new Set(["complete", "interrupted", "blank"]); +const ROCK_FIELDS = ["host", "name", "status", "kind", "age", "subject", "alias"]; + +async function toolList({ host, status, kind, agent, all } = {}) { let rocks = await allRocks(); if (host) rocks = rocks.filter((r) => r.host.toLowerCase() === host.toLowerCase()); if (status) rocks = rocks.filter((r) => r.status === status); if (kind) rocks = rocks.filter((r) => r.kind === kind); if (agent) rocks = rocks.filter((r) => (r.agentType || "claude").toLowerCase() === agent.toLowerCase()); - if (!rocks.length) return [{ type: "text", text: "(no prompt rocks match — is SlabMenubar running? try again in a few seconds)" }]; - // group by host, self first - rocks.sort((a, b) => (a.self === b.self ? a.host.localeCompare(b.host) : a.self ? -1 : 1) || 0); - const byHost = new Map(); - for (const r of rocks) (byHost.get(r.host) || byHost.set(r.host, []).get(r.host)).push(r); - const L = [`${rocks.length} prompt rock(s) across ${byHost.size} machine(s):`]; - for (const [hst, rs] of byHost) { - L.push(`\n${hst} (${rs.length}):`); - for (const r of rs.sort((a, b) => (b.updated || 0) - (a.updated || 0))) L.push(" " + line(r)); + let hidden = 0; + if (!all && !status) { + const now = Date.now(); + const live = rocks.filter((r) => !(FINISHED.has(r.status) && now - (r.updated || 0) > STALE_MS)); + hidden = rocks.length - live.length; + rocks = live; } - return [{ type: "text", text: L.join("\n") }]; + // self first, then by host, newest first within a host + rocks.sort((a, b) => (a.self === b.self ? 0 : a.self ? -1 : 1) || a.host.localeCompare(b.host) || (b.updated || 0) - (a.updated || 0)); + const rows = rocks.map((r) => ({ + host: r.host, + name: r.name, + status: r.status, + kind: r.agentType && r.agentType !== "claude" ? `${r.kind}·${r.agentType}` : r.kind, + age: age(r.updated), + subject: clip(r.subject, 64), + alias: r.proxName ? `prox:easel:${r.proxName}` : "", + })); + const notes = []; + if (hidden) notes.push(`${hidden} finished rock(s) idle >24h hidden — all:true to include`); + if (!rows.length && !hidden) notes.push("no rocks in the ledger — is SlabMenubar running? try again in a few seconds"); + notes.push("one rock in full: prox_find "); + return [{ type: "text", text: toon("rocks", rows, ROCK_FIELDS, { note: notes.join(" · ") }) }]; } async function toolFind({ handle }) { @@ -623,14 +628,15 @@ const TOOLS = [ { name: "prox_list", description: - "List the prompt rocks across the Slab fleet — every live Claude, Codex, or Easel session and headless agent the menubar advertises, as host:name with its status, kind, owning interface, age, and subject. Reads the local fleet ledger cache (no SSH).", + "List the prompt rocks across the Slab fleet — every live Claude, Codex, or Easel session and headless agent the menubar advertises — as one compact table: host, name, status, kind, age, subject, alias. Finished rocks idle for more than a day are hidden by default (the footer counts them). Reads the local fleet ledger cache (no SSH).", inputSchema: { type: "object", properties: { host: { type: "string", description: "Only rocks on this machine (e.g. neo, blueberry, panda)." }, - status: { type: "string", description: "Filter by status: working | awaiting | complete | rendering | blank | interrupted." }, + status: { type: "string", description: "Filter by status: working | awaiting | complete | rendering | blank | interrupted. Disables the stale-rock hiding." }, kind: { type: "string", description: "Filter by kind: session | agent." }, agent: { type: "string", description: "Filter by owning interface: claude | codex | easel." }, + all: { type: "boolean", description: "Include finished rocks idle for more than 24h (hidden by default)." }, }, }, }, diff --git a/slab/bin/puppet-mcp.mjs b/slab/bin/puppet-mcp.mjs index 1cefc0fe89..d77776a485 100755 --- a/slab/bin/puppet-mcp.mjs +++ b/slab/bin/puppet-mcp.mjs @@ -20,6 +20,7 @@ import { homedir } from "node:os"; import { join } from "node:path"; import { httpPort, serveHttp, serveStdio } from "../../toolchain/mcp/http-front.mjs"; import { PUPPET_GUIDANCE } from "../lib/computer-use-guidance.mjs"; +import { clip, toon } from "../../shared/toon.mjs"; import { termListAsync as termList, typeTextAsync as typeText, sendKeysAsync as sendKeys } from "./macos.mjs"; const HOME = homedir(); @@ -66,8 +67,19 @@ function rpc(req, { timeoutMs = 20000 } = {}) { const text = (t) => [{ type: "text", text: typeof t === "string" ? t : JSON.stringify(t, null, 2) }]; // ── CDP verbs (via the warm daemon) ───────────────────────────────────────── -async function toolList() { - return text(await rpc({ cmd: "list" })); +async function toolList({ full = false } = {}) { + const machines = await rpc({ cmd: "list" }); + if (full) return text(machines); + const rows = Object.entries(machines).map(([machine, state]) => ({ + machine, connected: state.connected, lazy: state.lazy, + pages: state.pages?.length || 0, error: state.lastError, + })); + const pages = Object.entries(machines).flatMap(([machine, state]) => + (state.pages || []).map(page => ({ machine, id: page.id, title: clip(page.title, 80), url: page.url }))); + return text([ + toon("machines", rows, ["machine", "connected", "lazy", "pages", "error"]), + toon("pages", pages, ["machine", "id", "title", "url"], { note: "Use exact page id as target; full:true for raw state." }), + ].join("\n")); } async function toolEval({ machine, js, target }) { return text(await rpc({ cmd: "eval", machine, args: { expr: js, target } })); @@ -144,8 +156,8 @@ const SEMANTIC_TOOLS = ["snapshot", "click", "fill", "wait"].map(action => ({ const TOOLS = [ ...SEMANTIC_TOOLS, { name: "puppet_list", act: false, - description: "List registered machines with CDP connection state and open page targets (from the puppet daemon). Read-only.", - inputSchema: { type: "object", properties: {} } }, + description: "List machines and exact browser page IDs in compact tables. Read-only; full:true returns raw JSON state.", + inputSchema: { type: "object", properties: { full: { type: "boolean", description: "Return complete JSON state instead of compact tables." } } } }, { name: "puppet_eval", act: false, description: "Evaluate a JavaScript expression in the active page of a machine's browser (via CDP) and return the value. Reads/inspects page state; can also mutate the DOM.", inputSchema: { type: "object", properties: { machine: { type: "string" }, js: { type: "string", description: "Expression to evaluate (awaited if it returns a promise)." }, target: { type: "string", description: "Optional target url/id substring; defaults to most-recent http(s) page." } }, required: ["machine", "js"] } }, diff --git a/slab/bin/puppet.mjs b/slab/bin/puppet.mjs index 064dbae806..55f4935373 100755 --- a/slab/bin/puppet.mjs +++ b/slab/bin/puppet.mjs @@ -171,6 +171,9 @@ export class Machine { } async connectLoop() { + // Native-only fleet entries share this registry. They have no browser to + // reconnect, so don't wake every three seconds to parse an undefined URL. + if (!this.spec.cdpUrl && !this.spec.acquireCmd) return; for (;;) { try { await this.connect(); @@ -211,6 +214,9 @@ export class Machine { } async ensureConnected() { + if (!this.spec.cdpUrl && !this.spec.acquireCmd) { + throw new Error(`${this.name} has no browser configured; native tools remain available`); + } if (this.connected) { this.markActive(); return; diff --git a/slab/lib/frame-local.mjs b/slab/lib/frame-local.mjs new file mode 100644 index 0000000000..ddc914ae1c --- /dev/null +++ b/slab/lib/frame-local.mjs @@ -0,0 +1,36 @@ +// Same native request/done protocol as the SSH agent, without local processes. +// The caller holds the machine lease until both sidecars have been read. +import { access, mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises"; +import { join } from "node:path"; +import { randomUUID } from "node:crypto"; +import { setTimeout as delay } from "node:timers/promises"; + +export async function localFrame(stateDir, mode, { timeoutMs = 15000 } = {}) { + await mkdir(stateDir, { recursive: true }); + const done = join(stateDir, "frame.done"); + await unlink(done).catch(error => { if (error.code !== "ENOENT") throw error; }); + const temp = join(stateDir, `frame-${randomUUID()}.req`); + try { + await writeFile(temp, mode); + // The native watcher must never consume a partially written request. + await rename(temp, join(stateDir, "frame.req")); + } finally { + await unlink(temp).catch(error => { if (error.code !== "ENOENT") throw error; }); + } + const deadline = performance.now() + timeoutMs; + for (;;) { + try { await access(done); break; } + catch (error) { if (error.code !== "ENOENT") throw error; } + // Do not read old sidecars or resend input when the outcome is unknown. + if (performance.now() >= deadline) throw new Error("local frame timeout; outcome unknown, request not retried"); + await delay(10); + } + const [json, jpg] = await Promise.all([ + readFile(join(stateDir, "frame.out.json"), "utf8"), + readFile(join(stateDir, "frame.out.jpg")).catch(error => { + if (error.code === "ENOENT") return Buffer.alloc(0); + throw error; + }), + ]); + return { json, jpg }; +} diff --git a/slab/lib/puppet-semantic.mjs b/slab/lib/puppet-semantic.mjs index a5199fbbcd..e68a23b0a1 100644 --- a/slab/lib/puppet-semantic.mjs +++ b/slab/lib/puppet-semantic.mjs @@ -19,14 +19,14 @@ function timeout(value=5000) { } export class SemanticBrowser { - constructor(endpoint) { this.endpoint=endpoint;this.connection=null;this.pages=new Map();this.queues=new Map(); } + constructor(endpoint) { this.endpoint=endpoint;this.connection=null;this.inspector=null;this.pages=new Map();this.queues=new Map(); } async browser() { if(!this.connection) { this.connection=(async()=>{ const {chromium}=await import("playwright-core"); // Attach without changing the user's media/focus/download preferences. const browser=await chromium.connectOverCDP(this.endpoint(),{noDefaults:true,timeout:10000}); - browser.on("disconnected",()=>{this.connection=null;this.pages.clear();}); + browser.on("disconnected",()=>{this.connection=null;this.inspector=null;this.pages.clear();}); return browser; })(); this.connection.catch(()=>{this.connection=null;}); @@ -35,15 +35,21 @@ export class SemanticBrowser { } async close() { if(this.connection) await (await this.connection).close().catch(()=>{}); - this.connection=null;this.pages.clear(); + this.connection=null;this.inspector=null;this.pages.clear(); } async page(target) { if(typeof target!=="string" || !target) throw new Error("An exact browser target ID is required (puppet_list pages)"); const browser=await this.browser(); - const inspector=await browser.newBrowserCDPSession(); + // Keep the inspector warm, but still validate the exact target every time: + // another client's page-close event can lag behind isClosed(). + if (!this.inspector) { + const pending = browser.newBrowserCDPSession(); + this.inspector = pending; + pending.catch(() => { if (this.inspector === pending) this.inspector = null; }); + } + const inspector=await this.inspector; try { await inspector.send("Target.getTargetInfo",{targetId:target}); } catch { throw new Error("Browser target is gone or does not match exactly; no action sent"); } - finally { await inspector.detach(); } const cached=this.pages.get(target); if(cached && !cached.isClosed()) return cached; for(const context of browser.contexts()) for(const page of context.pages()) { diff --git a/slab/test/computer-use-semantic.test.mjs b/slab/test/computer-use-semantic.test.mjs index e86989a45d..a50251ee5b 100644 --- a/slab/test/computer-use-semantic.test.mjs +++ b/slab/test/computer-use-semantic.test.mjs @@ -34,6 +34,20 @@ test("semantic browser fixture", {timeout:30000}, async t => { assert.equal(result.performed,true);assert.equal(await page.locator("input").inputValue(),"hello"); assert.equal((await service.run("wait",{target,locator:{text:"Changed"},state:"visible"})).verified,true); }); + await t.test("cached exact targets avoid inspector setup and reject closed pages", async()=>{ + const browser = await service.browser(); + const original = browser.newBrowserCDPSession; + browser.newBrowserCDPSession = () => { throw new Error("unexpected inspector setup"); }; + try { assert.equal(await service.page(target), service.pages.get(target)); } + finally { browser.newBrowserCDPSession = original; } + const extra = await context.newPage(); + const session = await context.newCDPSession(extra); + const { targetInfo } = await session.send("Target.getTargetInfo"); + await session.detach(); + const cached = await service.page(targetInfo.targetId); + await cached.close(); + await assert.rejects(service.page(targetInfo.targetId), /gone|exactly/); + }); await t.test("a waiting client does not block another client's action", async()=>{ await page.setContent(`Pending`); const waiting=service.run("wait",{target,locator:{text:"Ready"},timeout:1500}); diff --git a/slab/test/computer-use-transport.test.mjs b/slab/test/computer-use-transport.test.mjs index 1c2fb10dcc..65ec54eb77 100644 --- a/slab/test/computer-use-transport.test.mjs +++ b/slab/test/computer-use-transport.test.mjs @@ -75,6 +75,13 @@ function browser() { machine.call = async (method, params) => { calls.push({method, params}); return {data: 'fresh-image'}; }; return {machine, calls}; } +test('native-only Puppet machines never enter the browser reconnect loop', async () => { + const machine = new Machine('native', { local: true }); + machine.connect = () => { throw new Error('must not connect'); }; + await machine.connectLoop(); + assert.equal(machine.info().lastError, null); + await assert.rejects(machine.ensureConnected(), /no browser configured/); +}); test('Puppet uses recent in-memory JPEG without a CDP screenshot', async () => { const {machine, calls} = browser(); machine.liveFrames.set('session', {data: 'live-image', at: Date.now()}); @@ -101,3 +108,26 @@ test('Puppet invalidates only the acted-on browser session', () => { machine.invalidateFrame('Page.screencastFrameAck', 'other'); assert.equal(machine.liveFrames.has('other'), true); }); + +test('Puppet compact list retains exact page IDs and URLs with a full-state escape hatch', async t => { + const dir = await mkdtemp(join(tmpdir(), 'puppet-list-')); + const socket = join(dir, 'puppet.sock'); + const state = { fixture: { connected: true, lazy: false, managedLifecycle: false, lastActiveAt: null, lastError: null, + targets: ['https://example.test/path?q=a,b'], + pages: [{ id: 'exact-page-id', title: 'Page, title', url: 'https://example.test/path?q=a,b' }] } }; + const server = net.createServer(sock => sock.once('data', () => sock.end(JSON.stringify({ result: state }) + '\n'))); + await new Promise(resolve => server.listen(socket, resolve)); + t.after(async () => { await new Promise(resolve => server.close(resolve)); await rm(dir, { recursive: true, force: true }); }); + const call = full => new Promise((resolve, reject) => { + const child = execFile(process.execPath, ['slab/bin/puppet-mcp.mjs'], { + env: { ...process.env, SLAB_PUPPET_SOCK: socket }, + }, (error, stdout) => error ? reject(error) : resolve(JSON.parse(stdout).result.content[0].text)); + child.stdin.end(JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: 'puppet_list', arguments: { full } } }) + '\n'); + }); + const compact = await call(false), full = await call(true); + assert.match(compact, /pages\[1\]\{machine,id,title,url\}/); + assert.match(compact, /fixture,exact-page-id,"Page, title","https:\/\/example.test\/path\?q=a,b"/); + assert.deepEqual(JSON.parse(full), state); + assert.ok(Buffer.byteLength(compact) < Buffer.byteLength(full)); + t.diagnostic(`Puppet fixture list bytes: ${Buffer.byteLength(full)} → ${Buffer.byteLength(compact)}`); +}); diff --git a/slab/test/frame-local.test.mjs b/slab/test/frame-local.test.mjs new file mode 100644 index 0000000000..02db83ac88 --- /dev/null +++ b/slab/test/frame-local.test.mjs @@ -0,0 +1,46 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import { mkdtemp, readFile, writeFile, rm, readdir, unlink } from "node:fs/promises"; +import { join } from "node:path"; +import { tmpdir } from "node:os"; +import { setTimeout as delay } from "node:timers/promises"; +import { localFrame } from "../lib/frame-local.mjs"; + +test("local Frame waits for completion, never returning previous pixels", async t => { + const dir = await mkdtemp(join(tmpdir(), "frame-local-")); + t.after(() => rm(dir, { recursive: true, force: true })); + await writeFile(join(dir, "frame.done"), ""); + await writeFile(join(dir, "frame.out.json"), '{"capture":"old"}'); + await writeFile(join(dir, "frame.out.jpg"), "old"); + const pending = localFrame(dir, "window noocr session=fixture", { timeoutMs: 1000 }); + let request; + for (let i = 0; i < 100; i++) { + request = await readFile(join(dir, "frame.req"), "utf8").catch(() => null); + if (request) break; + await delay(5); + } + assert.equal(request, "window noocr session=fixture"); + await writeFile(join(dir, "frame.out.json"), '{"capture":"ok"}'); + const jpg = Buffer.from([255, 216, 0, 10, 255, 217]); + await writeFile(join(dir, "frame.out.jpg"), jpg); + await writeFile(join(dir, "frame.done"), ""); + const result = await pending; + assert.equal(JSON.parse(result.json).capture, "ok"); + assert.deepEqual(result.jpg, jpg); + assert.deepEqual((await readdir(dir)).filter(x => /^frame-/.test(x)), []); +}); + +test("local Frame times out rather than reading stale output or replaying input", async t => { + const dir = await mkdtemp(join(tmpdir(), "frame-timeout-")); + t.after(() => rm(dir, { recursive: true, force: true })); + await writeFile(join(dir, "frame.out.json"), '{"capture":"ok"}'); + await writeFile(join(dir, "frame.out.jpg"), "stale"); + const result = localFrame(dir, "press=12,34,1", { timeoutMs: 40 }); + const rejection = assert.rejects(result, /timeout.*not retried/); + for (let i = 0; i < 100; i++) { + try { await unlink(join(dir, "frame.req")); break; } catch {} + await delay(2); + } + await rejection; + assert.equal((await readdir(dir)).includes("frame.req"), false, "no replay after native consumption"); +}); diff --git a/toolchain/fleet/fleet-mcp.mjs b/toolchain/fleet/fleet-mcp.mjs index 334a4c5ee3..4e18e91fca 100644 --- a/toolchain/fleet/fleet-mcp.mjs +++ b/toolchain/fleet/fleet-mcp.mjs @@ -24,6 +24,7 @@ import { homedir } from "node:os"; import { dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; import { httpPort, serveHttp, serveStdio } from "../mcp/http-front.mjs"; +import { toon } from "../../shared/toon.mjs"; const HOME = homedir(); const REPO = resolve(dirname(fileURLToPath(import.meta.url)), "../.."); @@ -120,24 +121,25 @@ function statusGlyph(live) { } // ── tool implementations ──────────────────────────────────────────────────── +// One TOON table, sorted by role then name. Capabilities are space-joined so +// the cell never needs quoting; `live` is up / down / ? / self. async function toolList() { - const { machines, path } = loadRegistry(); + const { machines } = loadRegistry(); const nodes = await tailscaleStatus(); - const lines = [`fleet (${Object.keys(machines).length} machines) — source: ${path}`, ""]; - // group by designation for legibility - const groups = {}; - for (const [name, m] of Object.entries(machines)) (groups[m.designation || "unclassified"] ||= []).push([name, m]); - for (const [designation, entries] of Object.entries(groups)) { - lines.push(`── ${designation} ──`); - for (const [name, m] of entries) { - const live = liveFor(name, m, nodes); - const caps = (m.capabilities || []).join(",") || "—"; - const flag = m._review ? " ⚠review" : ""; - lines.push(` ${statusGlyph(live)} ${(m.emoji || "").padEnd(2)} ${name} — [${caps}]${flag}`); - } - lines.push(""); - } - return [{ type: "text", text: lines.join("\n").trimEnd() }]; + const rows = Object.entries(machines).map(([name, m]) => { + const live = liveFor(name, m, nodes); + return { + name, + role: m.designation || "unclassified", + live: live.online === true ? (live.self ? "self" : "up") : live.online === false ? "down" : "?", + caps: (m.capabilities || []).join(" "), + flags: m._review ? "review" : "", + }; + }).sort((a, b) => a.role.localeCompare(b.role) || a.name.localeCompare(b.name)); + const text = toon("machines", rows, ["name", "role", "live", "caps", "flags"], { + note: "live via tailscale · roles: fleet_designations · one machine in full: fleet_machine ", + }); + return [{ type: "text", text }]; } async function toolMachine({ name }) { @@ -201,7 +203,7 @@ async function toolCleaner({ apply = true, thinSnapshots = false, remoteBacked = const TOOLS = [ { name: "fleet_list", - description: "List every machine @jeffrey has access to, grouped by fleet designation, with a live online/offline glyph (🟢 online · ⚪ offline · ❔ unknown) and a one-line capability summary. The single source of truth for 'what machines do I have?'. Merges the private vault registry with live `tailscale status`.", + description: "List every machine @jeffrey has access to as one compact table: name, role (fleet designation), live (up/down/?/self via tailscale), capability tags, flags. The single source of truth for 'what machines do I have?'. Merges the private vault registry with live `tailscale status`; fleet_machine gives one machine in full.", inputSchema: { type: "object", properties: {} }, }, {