Something went wrong. Try again.
A local-first event pipeline for independent agents, built on Jazz.
Something went wrong. Try again.
6.8 kB · 174 lines
TypeScript
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175import { spawn } from "node:child_process";import fs from "node:fs/promises";import path from "node:path";import { afterEach, describe, expect, test } from "vitest";import { outputContractForDeclaration, outputContractIdentityJson } from "../src/agents/output-contracts.js";import type { ThoughtAgentDeclaration } from "../src/agents/types.js";import { JazzThoughtStore } from "../src/jazz/store.js";import { temporaryProject } from "./helpers.js";
const roots: string[] = [];
afterEach(async () => { await Promise.all(roots.splice(0).map((root) => fs.rm(root, { recursive: true, force: true })));});
describe("judgment and training export commands", () => { test("records a correction and writes JSONL with a dataset manifest", async () => { const project = await temporaryProject(); roots.push(project); const store = await JazzThoughtStore.open({ projectRoot: project, appId: "thoughtstream-local", runtimeRevision: "test" }); const source = await store.appendEvent({ type: "stream.thought.runtime.notice", schemaVersion: 1, source: "system:test", sourceKind: "system", externalId: "cli-source", idempotencyKey: "cli-source", occurredAt: "2026-07-15T00:00:00.000Z", actor: "test", correlationId: "cli-source", privacy: "public-source", payload: { text: "source" }, createdByRuntime: "test", }); const outputContract = outputContractForDeclaration({} as ThoughtAgentDeclaration); const original = { summary: "wrong", tags: ["wrong"], importance: "normal" as const, confidence: 0.5 }; const output = await store.appendEvent({ type: "stream.thought.derived.topics", schemaVersion: 1, source: "agent:test", sourceKind: "agent", externalId: "cli-output", idempotencyKey: "cli-output", occurredAt: "2026-07-15T00:00:01.000Z", actor: "test", rootEventId: source.event.id, parentEventId: source.event.id, correlationId: "cli-run", privacy: "public-source", payload: { outputContract: outputContractIdentityJson(outputContract), structuredOutput: original, }, createdByRuntime: "test", }); await store.upsertRun({ id: "cli-run", executionKey: "cli-execution", triggerEventId: source.event.id, agentId: "cli-agent", agentVersion: 1, status: "completed", inputEventIds: [source.event.id], outputEventIds: [output.event.id], attempt: 1, provider: "tinker", model: "Qwen/Qwen3.5-4B", promptHash: "cli-prompt", contextManifest: { eventIds: [source.event.id], agentRole: "standard", outputContract: outputContractIdentityJson(outputContract), }, result: original, createdAt: "2026-07-15T00:00:00.000Z", completedAt: "2026-07-15T00:00:02.000Z", updatedAt: "2026-07-15T00:00:02.000Z", }); await store.close(); const replacement = path.join(project, "replacement.json"); await fs.writeFile(replacement, JSON.stringify({ summary: "correct", tags: ["right"], importance: "normal", confidence: 0.9, })); const judgment = await run([ "judgment", "cli-run", "--kind", "correct", "--criterion", "fidelity", "--replacement", replacement, "--external-export-eligible", ], project); expect(judgment.code).toBe(0); expect(JSON.parse(judgment.stdout)).toMatchObject({ judgment: { schemaVersion: 2, payload: { kind: "correct", qualityEligible: true, externalExportEligible: true }, }, }); const destination = path.join(project, "exports", "training.jsonl"); const exported = await run(["training-export", "--output", destination], project); expect(exported.code).toBe(0); expect(JSON.parse(exported.stdout)).toMatchObject({ trainingExport: { examples: 1, datasetId: expect.stringMatching(/^sha256:/) } }); expect(JSON.parse((await fs.readFile(destination, "utf8")).trim())).toMatchObject({ kind: "correct", chosen: { summary: "correct" }, rejected: { summary: "wrong" }, }); expect(JSON.parse(await fs.readFile(`${destination}.manifest.json`, "utf8"))).toMatchObject({ examples: 1 }); }, 15_000);
test("requires both private-export acknowledgement flags and a file destination", async () => { const project = await temporaryProject(); roots.push(project); const incomplete = await run([ "training-export", "--include-sensitive-private", "--output", path.join(project, "private.jsonl"), ], project); expect(incomplete.code).not.toBe(0); expect(incomplete.stderr).toContain("requires both --include-sensitive-private and --authorize-sensitive-private-export");
const stdoutAttempt = await run([ "training-export", "--include-sensitive-private", "--authorize-sensitive-private-export", ], project); expect(stdoutAttempt.code).not.toBe(0); expect(stdoutAttempt.stderr).toContain("requires --output and is never written to stdout");
const privateDestination = path.join(project, "private-training", "stream.jsonl"); const unacknowledged = await run([ "private-training-export", "--output", privateDestination, ], project); expect(unacknowledged.code).not.toBe(0); expect(unacknowledged.stderr).toContain("--acknowledge-sensitive-private-training"); expect(unacknowledged.stdout).toBe("");
const privateExport = await run([ "private-training-export", "--output", privateDestination, "--acknowledge-sensitive-private-training", ], project); expect(privateExport.code).toBe(0); expect(JSON.parse(privateExport.stdout)).toMatchObject({ privateTrainingExport: { output: privateDestination, examples: 0, exactPrivateProvenance: true, externalExportAuthorityRequired: false, }, }); expect(await fs.readFile(privateDestination, "utf8")).toBe(""); expect((await fs.stat(privateDestination)).mode & 0o777).toBe(0o600); }, 15_000);});
async function run(args: string[], project: string): Promise<{ code: number | null; stdout: string; stderr: string }> { return await new Promise((resolve, reject) => { const child = spawn(process.execPath, ["--import", "tsx", "src/cli.ts", ...args], { cwd: path.resolve(import.meta.dirname, ".."), env: { ...process.env, THOUGHTSTREAM_ROOT: project }, stdio: ["ignore", "pipe", "pipe"], }); let stdout = ""; let stderr = ""; child.stdout.on("data", (chunk) => { stdout += String(chunk); }); child.stderr.on("data", (chunk) => { stderr += String(chunk); }); child.once("error", reject); child.once("exit", (code) => resolve({ code, stdout, stderr })); });}