From 467553fedb6656f0c98254af1946ded20ac9852a Mon Sep 17 00:00:00 2001 From: "prompt.ac/@jeffrey" Date: Sat, 5 Sep 2026 13:30:38 -0400 Subject: [PATCH] Add live Mediascholar progress page --- package.json | 2 +- slab/bin/prox-worker.mjs | 193 ++++++++++- slab/test/mediascholar-status.test.mjs | 63 ++++ slab/test/prox-worker.test.mjs | 56 +++- .../netlify/functions/mediascholar-status.mjs | 117 +++++++ .../papers.aesthetic.computer/index.html | 2 +- .../scholar/index.html | 316 ++++++++++++++++++ 7 files changed, 741 insertions(+), 8 deletions(-) create mode 100644 slab/test/mediascholar-status.test.mjs create mode 100644 system/netlify/functions/mediascholar-status.mjs create mode 100644 system/public/papers.aesthetic.computer/scholar/index.html diff --git a/package.json b/package.json index 5ba08fa1ad..fc6ac71a10 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "papers": "node papers/cli.mjs", "papers:publish": "node papers/cli.mjs publish", "mediascholar": "node papers/bin/mediascholar.mjs", - "test:mediascholar": "node --test papers/test/mediascholar.test.mjs slab/test/prox-worker.test.mjs slab/test/mediascholar-credential-proxy.test.mjs slab/test/mediascholar-mcp-integration.test.mjs", + "test:mediascholar": "node --test papers/test/mediascholar.test.mjs slab/test/prox-worker.test.mjs slab/test/mediascholar-credential-proxy.test.mjs slab/test/mediascholar-mcp-integration.test.mjs slab/test/mediascholar-status.test.mjs", "user": "f() { echo -n \"https://cloud.digitalocean.com/spaces/user-aesthetic-computer?path=\"; curl -s \"https://aesthetic.computer/user?from=$1\" | jq -r '.sub'; }; f", "filter": "cd shared; node --experimental-repl-await -i -e \"import('./filter.mjs').then(m => global.filter = m.filter)\"", "chat": "echo '\n💬 Chat\n';", diff --git a/slab/bin/prox-worker.mjs b/slab/bin/prox-worker.mjs index 4ac8d42957..5652a39079 100644 --- a/slab/bin/prox-worker.mjs +++ b/slab/bin/prox-worker.mjs @@ -18,6 +18,7 @@ const JOBS = Object.freeze({ mediascholar: "mediascholar.service" }); export function workerConfig(env = process.env) { const ledgerDir = resolve(env.PROX_WORKER_LEDGER_DIR || join(homedir(), ".config", "slab", "ledger")); + const mediascholarHome = resolve(env.MEDIASCHOLAR_HOME || join(homedir(), ".local", "share", "mediascholar")); return { host: String(env.PROX_WORKER_HOST || hostname().split(".")[0]).toLowerCase(), bind: env.PROX_WORKER_BIND || "", @@ -27,6 +28,11 @@ export function workerConfig(env = process.env) { localFile: resolve(env.PROX_WORKER_LOCAL_FILE || join(ledgerDir, "local.json")), refreshMs: Number(env.PROX_WORKER_REFRESH_MS) || 3000, systemctl: env.PROX_WORKER_SYSTEMCTL || "systemctl", + mediascholarHome, + mediascholarRunner: resolve(env.PROX_WORKER_MEDIASCHOLAR_RUNNER + || join(homedir(), ".local", "lib", "mediascholar", "papers", "bin", "mediascholar.mjs")), + mediascholarEnabled: !new Set(["0", "false", "no", "off", ""]) + .has(String(env.MEDIASCHOLAR_ENABLED || "").toLowerCase()), }; } @@ -104,6 +110,184 @@ async function systemctl(config, args) { return { stdout: stdout.trim(), stderr: stderr.trim() }; } +function parseProperties(stdout) { + return Object.fromEntries(stdout.split("\n").filter(Boolean).map((line) => { + const at = line.indexOf("="); + return at < 0 ? [line, ""] : [line.slice(0, at), line.slice(at + 1)]; + })); +} + +async function unitProperties(config, unit) { + try { + const { stdout } = await systemctl(config, [ + "show", unit, "--property=ActiveState,SubState,Result,ExecMainStatus", + ]); + return parseProperties(stdout); + } catch { + return {}; + } +} + +async function nextMediascholarTimer(config) { + try { + const { stdout } = await systemctl(config, [ + "list-timers", "mediascholar.timer", "mediascholar-bootstrap.timer", + "--all", "--output=json", "--no-pager", + ]); + const timers = JSON.parse(stdout || "[]"); + const next = timers.map((timer) => Number(timer.next)).filter(Number.isFinite).sort((a, b) => a - b)[0]; + return next ? new Date(next / 1000).toISOString() : null; + } catch { + return null; + } +} + +const clipped = (value, max) => String(value || "").replace(/\s+/g, " ").trim().slice(0, max); + +function publicTopic(topic) { + if (!topic || typeof topic !== "object") return null; + const signals = Array.isArray(topic.signals) ? topic.signals.flatMap((signal) => { + try { + const url = new URL(signal?.url); + if (!new Set(["http:", "https:"]).has(url.protocol)) return []; + return [{ + title: clipped(signal.title, 240), + url: url.toString(), + kind: clipped(signal.kind, 80), + relevance: clipped(signal.relevance, 500), + }]; + } catch { + return []; + } + }).slice(0, 16) : []; + return { + title: clipped(topic.title, 140), + question: clipped(topic.question, 500), + claim: clipped(topic.claim, 700), + whyNow: clipped(topic.whyNow, 700), + terms: Array.isArray(topic.terms) ? topic.terms.map((term) => clipped(term, 80)).filter(Boolean).slice(0, 12) : [], + signals, + }; +} + +async function recentMediascholarRuns(config) { + const runsDir = join(config.mediascholarHome, "runs"); + const names = (await readdir(runsDir, { withFileTypes: true }).catch(() => [])) + .filter((entry) => entry.isDirectory()).map((entry) => entry.name).sort().reverse().slice(0, 12); + const runs = []; + for (const name of names) { + const directory = join(runsDir, name); + const run = await readJson(join(directory, "run.json")); + if (!run || typeof run !== "object") continue; + runs.push({ + id: clipped(run.id || name, 64), + status: new Set([ + "starting", "synthesizing-topic", "authoring-paper", "candidate", "failed", "dry-run-complete", + ]).has(run.status) ? run.status : "unknown", + providerInvoked: Boolean(run.provider), + startedAt: clipped(run.startedAt, 32) || null, + completedAt: clipped(run.completedAt, 32) || null, + topic: publicTopic(await readJson(join(directory, "topic.json"))), + }); + } + return runs; +} + +async function mediascholarDoctor(config) { + try { + const { stdout } = await pexec("/usr/bin/env", [ + "node", config.mediascholarRunner, "doctor", "--json", + ], { timeout: 10_000, maxBuffer: 1024 * 1024 }); + return JSON.parse(stdout); + } catch { + return null; + } +} + +export async function buildMediascholarStatus(config) { + const [doctor, mainUnit, bootstrapUnit, nextCheckAt, runs] = await Promise.all([ + mediascholarDoctor(config), + unitProperties(config, "mediascholar.service"), + unitProperties(config, "mediascholar-bootstrap.service"), + nextMediascholarTimer(config), + recentMediascholarRuns(config), + ]); + const current = runs[0] || null; + const running = mainUnit.ActiveState === "active"; + const requiredTools = doctor?.paperTools ? Object.values(doctor.paperTools) : []; + const toolsReady = requiredTools.length > 0 && requiredTools.every(Boolean); + const admission = doctor?.admission; + const reasons = Array.isArray(admission?.reasons) ? admission.reasons : []; + const gates = { + processor: admission ? (reasons.some((reason) => reason.startsWith("load ")) ? "waiting" : "ready") : "unknown", + memory: admission ? (reasons.some((reason) => reason.startsWith("available memory ")) ? "waiting" : "ready") : "unknown", + disk: admission ? (reasons.some((reason) => reason.startsWith("free disk ")) ? "waiting" : "ready") : "unknown", + }; + + let state = "idle"; + let phase = "admission"; + let headline = "Ready for the next run"; + let detail = "The next unattended run will begin on its timer."; + if (running) { + state = "working"; + phase = current?.status || "starting"; + headline = phase === "synthesizing-topic" ? "Synthesizing a new-media inquiry" + : phase === "authoring-paper" ? `Writing ${current?.topic?.title || "a Botted Paper"}` + : "Starting a Botted Paper"; + detail = phase === "authoring-paper" + ? "Research, drafting, building, and visual QA are running inside the paper stack." + : "The topic is being formed without a human-supplied prompt."; + } else if (current?.status === "candidate") { + state = "review"; + phase = "candidate"; + headline = `Candidate ready: ${current.topic?.title || "Botted Paper"}`; + detail = "The paper is held for human review and cannot publish itself."; + } else if (current?.status === "failed") { + state = "idle"; + phase = "recovery"; + headline = "The last run stopped safely"; + detail = "No draft was published; the timer can try again after review."; + } else if (!toolsReady || !config.mediascholarEnabled) { + state = "waiting"; + phase = "bootstrap"; + headline = "Waiting for a quiet window"; + detail = gates.processor === "waiting" + ? "Jasellite is busy, so setup is yielding to its existing work." + : "The guarded paper-tool setup is waiting for its next check."; + } else if (admission && !admission.accepted) { + state = "waiting"; + headline = "Waiting for a quiet window"; + detail = "The worker is yielding to Jasellite's existing work."; + } + + return { + version: 1, + system: "mediascholar", + updatedAt: new Date().toISOString(), + state, + phase, + headline, + detail, + nextCheckAt, + gates, + current, + activity: { + runs: runs.length, + providerRuns: runs.filter((run) => run.providerInvoked).length, + candidates: runs.filter((run) => run.status === "candidate").map((run) => ({ + id: run.id, + title: run.topic?.title || "Botted Paper", + completedAt: run.completedAt, + status: "awaiting review", + })), + }, + safeguards: { + autoPublish: false, + bootstrapResult: clipped(bootstrapUnit.Result, 32) || "unknown", + }, + }; +} + async function jobAction(config, body) { const job = String(body.job || "").toLowerCase(); const action = String(body.action || "status").toLowerCase(); @@ -124,10 +308,7 @@ async function jobAction(config, body) { const { stdout } = await systemctl(config, [ "show", unit, "--property=ActiveState,SubState,Result,ExecMainStatus,MemoryCurrent,CPUUsageNSec", ]); - const properties = Object.fromEntries(stdout.split("\n").filter(Boolean).map((line) => { - const at = line.indexOf("="); - return at < 0 ? [line, ""] : [line.slice(0, at), line.slice(at + 1)]; - })); + const properties = parseProperties(stdout); return { status: 200, body: { ok: true, host: config.host, job, action, unit, properties } }; } catch (error) { return { status: 503, body: { ok: false, error: String(error.stderr || error.message).trim().slice(-500) } }; @@ -162,6 +343,10 @@ export function createWorkerServer(config, ip) { respond(res, 405, { ok: false, error: "method not allowed" }); return; } + if (url.pathname === "/status/mediascholar") { + respond(res, 200, await buildMediascholarStatus(config)); + return; + } const ledger = await buildLedger(config, ip); await atomicJson(config.localFile, ledger); respond(res, 200, ledger); diff --git a/slab/test/mediascholar-status.test.mjs b/slab/test/mediascholar-status.test.mjs new file mode 100644 index 0000000000..c155e013cd --- /dev/null +++ b/slab/test/mediascholar-status.test.mjs @@ -0,0 +1,63 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import test from "node:test"; + +import { handler, sanitizeMediascholarStatus } from "../../system/netlify/functions/mediascholar-status.mjs"; + +test("public Mediascholar status strips private worker fields", () => { + const status = sanitizeMediascholarStatus({ + state: "working", + phase: "authoring-paper", + headline: "Writing a Botted Paper", + detail: "Drafting safely", + updatedAt: "2026-09-05T17:00:00Z", + nextCheckAt: "2026-09-05T18:00:00Z", + gates: { processor: "ready", memory: "ready", disk: "ready" }, + current: { + id: "run-1", + status: "authoring-paper", + startedAt: "2026-09-05T17:00:00Z", + worktree: "/home/jas/private", + provider: "private-provider", + topic: { + title: "A live inquiry", + question: "How do computational media expose their own conditions?", + claim: "They make infrastructure available as an aesthetic material.", + whyNow: "The substrate is newly inspectable.", + terms: ["infrastructure"], + signals: [ + { title: "Source", url: "https://example.com/paper", kind: "paper", relevance: "Direct evidence" }, + { title: "Local", url: "file:///home/jas/note", kind: "note", relevance: "Private" }, + ], + }, + }, + activity: { runs: 1, providerRuns: 1, candidates: [] }, + }); + const text = JSON.stringify(status); + assert.equal(status.current.topic.signals.length, 1); + assert.equal(text.includes("/home/jas"), false); + assert.equal(text.includes("private-provider"), false); + assert.equal(status.safeguards.autoPublish, false); +}); + +test("status endpoint degrades safely when no worker route is configured", async () => { + const prior = process.env.MEDIASCHOLAR_WORKER_URL; + delete process.env.MEDIASCHOLAR_WORKER_URL; + try { + const response = await handler({ httpMethod: "GET" }); + const body = JSON.parse(response.body); + assert.equal(response.statusCode, 200); + assert.equal(body.state, "unavailable"); + assert.equal(body.safeguards.autoPublish, false); + } finally { + if (prior === undefined) delete process.env.MEDIASCHOLAR_WORKER_URL; + else process.env.MEDIASCHOLAR_WORKER_URL = prior; + } +}); + +test("scholar page polls the public status endpoint and names the release gate", async () => { + const html = await readFile(new URL("../../system/public/papers.aesthetic.computer/scholar/index.html", import.meta.url), "utf8"); + assert.match(html, /fetch\("\/api\/mediascholar-status"/); + assert.match(html, /Candidates cannot publish themselves\./); + assert.doesNotMatch(html, /100\.\d+\.\d+\.\d+/); +}); diff --git a/slab/test/prox-worker.test.mjs b/slab/test/prox-worker.test.mjs index afe0370354..f78c74c60e 100644 --- a/slab/test/prox-worker.test.mjs +++ b/slab/test/prox-worker.test.mjs @@ -4,7 +4,7 @@ import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; -import { buildLedger, createWorkerServer, workerConfig } from "../bin/prox-worker.mjs"; +import { buildLedger, buildMediascholarStatus, createWorkerServer, workerConfig } from "../bin/prox-worker.mjs"; const listen = (server) => new Promise((resolve, reject) => { server.once("error", reject); @@ -30,7 +30,7 @@ test("headless Prox sanitizes its ledger and exposes only allowlisted systemd jo const calls = join(root, "systemctl.calls"); const fakeSystemctl = join(root, "systemctl"); - await writeFile(fakeSystemctl, `#!/bin/sh\nprintf '%s\\n' "$*" >> '${calls}'\ncase "$*" in\n *show*) printf 'ActiveState=inactive\\nSubState=dead\\nResult=success\\nExecMainStatus=0\\nMemoryCurrent=0\\nCPUUsageNSec=0\\n' ;;\nesac\n`); + await writeFile(fakeSystemctl, `#!/bin/sh\nprintf '%s\\n' "$*" >> '${calls}'\ncase "$*" in\n *show*) printf 'ActiveState=inactive\\nSubState=dead\\nResult=success\\nExecMainStatus=0\\nMemoryCurrent=0\\nCPUUsageNSec=0\\n' ;;\n *list-timers*) printf '[]\\n' ;;\nesac\n`); await chmod(fakeSystemctl, 0o700); const config = workerConfig({ @@ -71,3 +71,55 @@ test("headless Prox sanitizes its ledger and exposes only allowlisted systemd jo }).then((response) => response.json()); assert.equal(status.properties.ActiveState, "inactive"); }); + +test("headless Prox exposes a path-free public Mediascholar status", async (t) => { + const root = await mkdtemp(join(tmpdir(), "prox-scholar-")); + t.after(() => rm(root, { recursive: true, force: true })); + const home = join(root, "mediascholar"); + const runDir = join(home, "runs", "20260905-170000-000Z"); + await mkdir(runDir, { recursive: true }); + await writeFile(join(runDir, "run.json"), JSON.stringify({ + id: "20260905-170000-000Z", + status: "authoring-paper", + startedAt: "2026-09-05T17:00:00.000Z", + worktree: "/private/worktree", + provider: "secret-provider", + })); + await writeFile(join(runDir, "topic.json"), JSON.stringify({ + title: "Synthetic media question", + question: "What changes when a medium can inspect its own circulation?", + claim: "Self-inspection becomes a material property of the medium.", + whyNow: "Machine-readable cultural infrastructure is becoming ordinary.", + terms: ["circulation", "reflexivity"], + signals: [ + { title: "Public source", url: "https://example.org/source", kind: "paper", relevance: "Evidence" }, + { title: "Private file", url: "file:///private/note", kind: "note", relevance: "Do not expose" }, + ], + })); + const fakeSystemctl = join(root, "systemctl"); + await writeFile(fakeSystemctl, `#!/bin/sh\ncase "$*" in\n *show*mediascholar.service*) printf 'ActiveState=active\\nSubState=running\\nResult=success\\nExecMainStatus=0\\n' ;;\n *show*) printf 'ActiveState=inactive\\nSubState=dead\\nResult=success\\nExecMainStatus=0\\n' ;;\n *list-timers*) printf '[]\\n' ;;\nesac\n`); + await chmod(fakeSystemctl, 0o700); + + const config = workerConfig({ + PROX_WORKER_BIND: "127.0.0.1", + PROX_WORKER_LEDGER_DIR: join(root, "ledger"), + PROX_WORKER_SYSTEMCTL: fakeSystemctl, + PROX_WORKER_MEDIASCHOLAR_RUNNER: join(root, "missing-runner.mjs"), + MEDIASCHOLAR_HOME: home, + MEDIASCHOLAR_ENABLED: "1", + }); + const status = await buildMediascholarStatus(config); + assert.equal(status.state, "working"); + assert.equal(status.phase, "authoring-paper"); + assert.equal(status.current.topic.signals.length, 1); + assert.equal(status.current.topic.signals[0].url, "https://example.org/source"); + assert.equal(JSON.stringify(status).includes("/private/"), false); + assert.equal(JSON.stringify(status).includes("secret-provider"), false); + + const server = createWorkerServer(config, "127.0.0.1"); + const port = await listen(server); + t.after(() => close(server)); + const response = await fetch(`http://127.0.0.1:${port}/status/mediascholar`); + assert.equal(response.status, 200); + assert.equal((await response.json()).system, "mediascholar"); +}); diff --git a/system/netlify/functions/mediascholar-status.mjs b/system/netlify/functions/mediascholar-status.mjs new file mode 100644 index 0000000000..ac3b6ef2d5 --- /dev/null +++ b/system/netlify/functions/mediascholar-status.mjs @@ -0,0 +1,117 @@ +const DEFAULT_STATUS = Object.freeze({ + version: 1, + system: "mediascholar", + updatedAt: null, + state: "unavailable", + phase: "connection", + headline: "Live status is unavailable", + detail: "Mediascholar may still be running; the public status link could not be reached.", + nextCheckAt: null, + gates: { processor: "unknown", memory: "unknown", disk: "unknown" }, + current: null, + activity: { runs: 0, providerRuns: 0, candidates: [] }, + safeguards: { autoPublish: false }, +}); + +const clipped = (value, max) => String(value || "").replace(/\s+/g, " ").trim().slice(0, max); +const iso = (value) => Number.isFinite(Date.parse(value)) ? new Date(value).toISOString() : null; + +function safeUrl(value) { + try { + const url = new URL(value); + return new Set(["http:", "https:"]).has(url.protocol) ? url.toString() : null; + } catch { + return null; + } +} + +function safeTopic(topic) { + if (!topic || typeof topic !== "object") return null; + return { + title: clipped(topic.title, 140), + question: clipped(topic.question, 500), + claim: clipped(topic.claim, 700), + whyNow: clipped(topic.whyNow, 700), + terms: Array.isArray(topic.terms) ? topic.terms.map((item) => clipped(item, 80)).filter(Boolean).slice(0, 12) : [], + signals: Array.isArray(topic.signals) ? topic.signals.flatMap((signal) => { + const url = safeUrl(signal?.url); + return url ? [{ + title: clipped(signal.title, 240), + url, + kind: clipped(signal.kind, 80), + relevance: clipped(signal.relevance, 500), + }] : []; + }).slice(0, 16) : [], + }; +} + +export function sanitizeMediascholarStatus(value) { + if (!value || typeof value !== "object") return { ...DEFAULT_STATUS }; + const state = new Set(["waiting", "working", "idle", "review", "unavailable"]).has(value.state) + ? value.state : "unavailable"; + const current = value.current && typeof value.current === "object" ? { + id: clipped(value.current.id, 64), + status: clipped(value.current.status, 40), + startedAt: iso(value.current.startedAt), + completedAt: iso(value.current.completedAt), + topic: safeTopic(value.current.topic), + } : null; + const candidates = Array.isArray(value.activity?.candidates) ? value.activity.candidates.map((candidate) => ({ + id: clipped(candidate.id, 64), + title: clipped(candidate.title, 140) || "Botted Paper", + completedAt: iso(candidate.completedAt), + status: "awaiting review", + })).slice(0, 12) : []; + const gate = (name) => new Set(["ready", "waiting", "unknown"]).has(value.gates?.[name]) + ? value.gates[name] : "unknown"; + return { + version: 1, + system: "mediascholar", + updatedAt: iso(value.updatedAt), + state, + phase: clipped(value.phase, 40) || "unknown", + headline: clipped(value.headline, 180) || DEFAULT_STATUS.headline, + detail: clipped(value.detail, 300) || DEFAULT_STATUS.detail, + nextCheckAt: iso(value.nextCheckAt), + gates: { processor: gate("processor"), memory: gate("memory"), disk: gate("disk") }, + current, + activity: { + runs: Math.max(0, Math.min(10_000, Number(value.activity?.runs) || 0)), + providerRuns: Math.max(0, Math.min(10_000, Number(value.activity?.providerRuns) || 0)), + candidates, + }, + safeguards: { autoPublish: false }, + }; +} + +function respond(statusCode, body) { + return { + statusCode, + headers: { + "Content-Type": "application/json; charset=utf-8", + "Access-Control-Allow-Origin": "*", + "Access-Control-Allow-Methods": "GET, OPTIONS", + "Cache-Control": "public, max-age=5, stale-while-revalidate=15", + "X-Content-Type-Options": "nosniff", + }, + body: JSON.stringify(body), + }; +} + +export async function handler(event) { + if (event.httpMethod === "OPTIONS") return respond(204, {}); + if (event.httpMethod !== "GET") return respond(405, { error: "GET only" }); + const workerUrl = String(process.env.MEDIASCHOLAR_WORKER_URL || "").replace(/\/$/, ""); + if (!workerUrl) return respond(200, DEFAULT_STATUS); + try { + const response = await fetch(`${workerUrl}/status/mediascholar`, { + headers: { accept: "application/json" }, + signal: AbortSignal.timeout(4000), + }); + if (!response.ok) throw new Error(`worker returned ${response.status}`); + return respond(200, sanitizeMediascholarStatus(await response.json())); + } catch (error) { + console.warn(`mediascholar-status: ${clipped(error.message, 160)}`); + return respond(200, DEFAULT_STATUS); + } +} diff --git a/system/public/papers.aesthetic.computer/index.html b/system/public/papers.aesthetic.computer/index.html index 41701d05a1..1e5f778588 100644 --- a/system/public/papers.aesthetic.computer/index.html +++ b/system/public/papers.aesthetic.computer/index.html @@ -1609,7 +1609,7 @@
- aesthetic.computer · platter · sleuth · source · + aesthetic.computer · platter · mediascholar · sleuth · source ·
+ + -- 2.51.2