diff --git a/tests/fixtures/schema_eval/results/local/results.jsonl b/tests/fixtures/schema_eval/results/local/results.jsonl new file mode 100644 index 000000000..1349e9314 --- /dev/null +++ b/tests/fixtures/schema_eval/results/local/results.jsonl @@ -0,0 +1,10 @@ +{"content_preservation": {"found": ["apple", "banana"], "fraction": 1.0, "missing": []}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "bounded_snacks", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"snacks\": [\n \"apple\",\n \"banana\"\n ]\n}", "usage": {"input_tokens": 58, "output_tokens": 25, "total_tokens": 83}} +{"content_preservation": {"found": ["Mira Patel", "incident coordinator", "database failover"], "fraction": 1.0, "missing": []}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "contact_card", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"name\": \"Mira Patel\",\n \"role\": \"incident coordinator\"\n,\n \"summary\": \"Tracking database failover status for the evening handoff.\"\n}", "usage": {"input_tokens": 52, "output_tokens": 42, "total_tokens": 94}} +{"content_preservation": {"found": ["security", "launch"], "fraction": 0.6666666666666666, "missing": ["delaying"]}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "decision_flags", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"decision\": \"delay_launch\",\n \"approved\": true,\n \"tags\": [\n \"security\",\n \"launch\",\n \"billing\"\n ]\n}", "usage": {"input_tokens": 51, "output_tokens": 44, "total_tokens": 95}} +{"content_preservation": {"found": ["Miller Family Trust Amendment", "Jordan Miller", "Priya Shah", "Evan Lee", "brokerage account", "third anniversary"], "fraction": 1.0, "missing": []}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "document_analysis_trust_excerpt", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"overview\": \"Miller Family Trust Amendment\",\n \"parties\": [\n {\n \"name\": \"Jordan Miller\",\n \"role\": \"Settlor\"\n ,\n \"formal_term\": \"Settlor\"\n ,\n \"appointment_tier\": \"primary\"\n ,\n \"context\": \"appointed primary Trustee\"\n }\n ],\n \"key_provisions\": [\n {\n \"type\": \"distribution\",\n \"text\": \"The Trustee may distribute the brokerage account for health and education expenses.\"\n ,\n \"applies_to\": \"Trustee\"\n }\n ],\n \"assets\": [\n {\n \"name\": \"brokerage account\",\n \"asset_type\": \"account\"\n ,\n \"disposition\": \"may be distributed for health and education expenses\"\n }\n ],\n \"conditions\": [\n {\n \"trigger\": \"third anniversary of the Settlor's death\",\n \"effect\": \"mandatory review\"\n ,\n \"date_or_timing\": \"third anniversary of the Settlor's death\"\n }\n ],\n \"important_dates\": [\n {\n \"date\": \"March 4, 2026\",\n \"meaning\": \"Execution date\"\n }\n ]\n ,\n \"summary\": \"The Miller Family Trust Amendment, executed on March 4, 2026, by Settlor Jordan Miller, appoints Priya Shah as primary Trustee and Evan Lee as successor. The Trustee may distribute the brokerage account for health and education expenses. A mandatory review is triggered on the third anniversary of the Settlor's death.\"\n}\n", "usage": {"input_tokens": 111, "output_tokens": 398, "total_tokens": 509}} +{"content_preservation": {"found": ["Alice Smith", "Bob Chen", "solstone", "pytest tests/test_cluster.py", "solstone/think/cluster.py", "https://example.test/spec"], "fraction": 1.0, "missing": []}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "screen_record_editor_meeting", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"narrative\": \"At 09:00:05, a Zoom meeting is in progress with Alice Smith and Bob Chen visible as participants. The shared screen displays the solstone repository within VS Code. At 09:01:10, a terminal command `pytest tests/test_cluster.py` fails due to an AssertionError in solstone/think/cluster.py. By 09:02:00, a browser window opens to https://example.test/spec for the formatter contract.\",\n \"entities\": [\n {\n \"type\": \"Person\",\n \"name\": \"Alice Smith\"\n ,\n \"role\": \"attendee\"\n ,\n \"context\": \"Zoom meeting participant\"\n },\n {\n \"type\": \"Person\",\n \"name\": \"Bob Chen\"\n ,\n \"role\": \"attendee\"\n ,\n \"context\": \"Zoom meeting participant\"\n },\n {\n \"type\": \"Tool\",\n \"name\": \"VS Code\"\n ,\n \"role\": \"mentioned\"\n ,\n \"context\": \"displaying solstone repository\"\n },\n {\n \"type\": \"Project\",\n \"name\": \"solstone\"\n ,\n \"role\": \"mentioned\"\n ,\n \"context\": \"repository and file path\"\n },\n {\n \"type\": \"FilePath\",\n \"name\": \"solstone/think/cluster.py\"\n ,\n \"role\": \"mentioned\"\n ,\n \"context\": \"location of AssertionError\"\n },\n {\n \"type\": \"Tool\",\n \"name\": \"pytest\"\n ,\n \"role\": \"mentioned\"\n ,\n \"context\": \"command execution\"\n },\n {\n \"type\": \"URL\",\n \"name\": \"https://example.test/spec\"\n ,\n \"role\": \"mentioned\"\n ,\n \"context\": \"browser destination for formatter contract\"\n }\n ]\n}", "usage": {"input_tokens": 120, "output_tokens": 466, "total_tokens": 586}} +{"content_preservation": {"found": ["Michael Bauer <> Jer Miller Meeting", "11:00", "13:00", "82 agents failed", "sol://facets/awareness/news/20260708", "no follow-up items found"], "fraction": 1.0, "missing": []}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "morning_briefing_20260708_trimmed", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"metadata\": {\n \"generated\": \"2026-07-09T04:47:41\",\n \"model\": \"scratch-model\",\n \"sources\": {\n \"segments\": 0,\n \"anticipated_activities\": 9,\n \"facet_newsletters\": 6,\n \"followups\": 0,\n \"steward_health\": \"present\"\n },\n \"gaps\": [\n \"no facet newsletter available for vconic\",\n \"no follow-up items found\",\n \"no decision items found\"\n ],\n \"coverage_preamble\": \"Built from 0 indexed source paths, 9 anticipated activities today, 17 forward-looking anticipated activities, 6 facet newsletters, 0 follow-ups, 0 decision results. Gaps: no facet newsletter available for vconic; no follow-up items found; no decision items found.\"\n },\n \"your_day\": [\n {\n \"time\": \"11:00\",\n \"text\": \"Michael Bauer <> Jer Miller Meeting [meeting, solpbc]\"\n },\n {\n \"time\": \"13:00\",\n \"text\": \"Travel Prep [reminder, personal]\"\n }\n ],\n \"yesterday\": [],\n \"needs_attention\": [\n {\n \"text\": \"Pipeline issue: 82 agents failed during yesterday's processing.\",\n \"source_id\": \"sol://steward/pipeline\"\n }\n ],\n \"forward_look\": [],\n \"reading\": [\n {\n \"facet\": \"awareness\",\n \"summary\": \"Michael Morgan confirmed guardrails; reply required. Source: sol://facets/awareness/news/20260708\"\n }\n ]\n}", "usage": {"input_tokens": 401, "output_tokens": 414, "total_tokens": 815}} +{"content_preservation": {"found": ["Slack", "#kg-schema-finish", "Mira Patel", "3:12 PM", "Schema drift", "Noah Lee", "3:13 PM", "edge rebuild"], "fraction": 0.8888888888888888, "missing": ["merge convergence eval gate"]}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "messaging_frame_team_chat", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"app\": \"Slack\",\n \"thread\": \"#kg-schema-finish\",\n \"view\": \"conversation\",\n \"messages\": [\n {\n \"sender\": \"Mira Patel\",\n \"timestamp\": \"3:12 PM\",\n \"subject\": null,\n \"text\": \"Schema drift is fixed, but edge rebuild still needs a second idempotency run.\"\n },\n {\n \"sender\": \"Noah Lee\",\n \"timestamp\": \"3:13 PM\",\n \"subject\": null,\n \"text\": \"I can compare the aggregate hash after rebuild.\"\n }\n ]\n}", "usage": {"input_tokens": 141, "output_tokens": 150, "total_tokens": 291}} +{"content_preservation": {"found": ["Google Calendar", "July 13-17, 2026", "Google Meet", "Michael Bauer", "Jer Miller", "review edges layer convergence"], "fraction": 0.6666666666666666, "missing": ["Michael Bauer <> Jer Miller KG Validation", "11:00 AM", "12:00 PM"]}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "calendar_frame_live_validation", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"app\": \"Google Calendar\",\n \"view\": \"week\",\n \"range\": \"July 13-17, 2026\",\n \"events\": [\n {\n \"title\": \"KG Validation\",\n \"start\": \"2026-07-16T11:00:00Z\",\n \"end\": \"2026-07-16T12:00:00Z\",\n \"location\": \"Zoom\",\n \"conferencing\": \"Google Meet link visible\",\n \"guests\": [\n \"Michael Bauer\",\n \"Jer Miller\"\n ],\n \"status\": \"accepted\",\n \"recurrence\": null,\n \"calendar\": \"solpbc\",\n \"description\": \"review edges layer convergence and eval gate closure.\"\n }\n ],\n \"availability\": [\n \"Free: 09:00-10:30\",\n \"Busy: 13:00-14:00\"\n ]\n,\n \"notes\": \"Extracted calendar UI details from the frame text without inventing attendees or availability.\"\n}", "usage": {"input_tokens": 179, "output_tokens": 273, "total_tokens": 452}} +{"content_preservation": {"found": ["Jer Miller", "Michael Bauer", "rerun the edge rebuild", "review the eval summary", "Friday", "keep relation target names", "works-with"], "fraction": 0.875, "missing": ["target name so we can repair it later"]}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "story_relations_bauer_kg_review", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"body\": \"Jer Miller and Michael Bauer reviewed the knowledge graph schema enrichment pass. Jer committed to rerun the edge rebuild after merging the alias fix. Michael committed to review the eval summary by Friday. The group decided to keep relation target names when the entity observer cannot resolve a target id, as Michael suggested keeping the name so it can be repaired later. Jer works with Michael Bauer on the solstone KG validation effort.\",\n \"topics\": [\n \"Knowledge Graph\",\n \"Schema Enrichment\",\n \"Edge Rebuild\",\n \"Alias Fix\",\n \"Entity Observer\",\n \"Target Names\",\n \"Solstone KG Validation\"\n ],\n \"confidence\": 0.9,\n \"commitments\": [\n {\n \"owner\": \"Jer Miller\",\n \"action\": \"rerun the edge rebuild\",\n \"counterparty\": \"after merging the alias fix\"\n ,\n \"when\": \"after merging the alias fix\"\n ,\n \"context\": \"Knowledge Graph schema enrichment pass\"\n },\n {\n \"owner\": \"Michael Bauer\",\n \"action\": \"review the eval summary\",\n \"counterparty\": \"by Friday\"\n ,\n \"when\": \"by Friday\"\n ,\n \"context\": \"Knowledge Graph schema enrichment pass\"\n }\n ],\n \"closures\": [\n {\n \"owner\": \"Jer Miller\",\n \"action\": \"reviewed the knowledge graph schema enrichment pass\"\n ,\n \"counterparty\": \"Michael Bauer\"\n ,\n \"resolution\": \"done\"\n ,\n \"context\": \"Knowledge Graph schema enrichment pass\"\n }\n ],\n \"decisions\": [\n {\n \"owner\": \"The group\",\n \"action\": \"keep relation target names when the entity observer cannot resolve a target id\",\n \"counterparty\": \"N/A\"\n ,\n \"context\": \"Knowledge Graph schema enrichment pass\"\n }\n ],\n \"relations\": [\n {\n \"from\": \"Jer Miller\",\n \"to\": \"Michael Bauer\",\n \"kind\": \"works-with\",\n \"note\": \"on the solstone KG validation effort\"\n ,\n \"quote\": \"Jer works with Michael Bauer on the solstone KG validation effort.\"\n }\n ]\n}", "usage": {"input_tokens": 137, "output_tokens": 545, "total_tokens": 682}} +{"content_preservation": {"found": ["alice_johnson", "add", "works with Bob Lee", "works-with", "Bob Lee"], "fraction": 0.8333333333333334, "missing": ["keep"]}, "finish_reason": "stop", "model": "local/qwen3.5-4b", "name": "entity_observer_relation_ops", "schema_validity": {"errors": [], "valid": true}, "text": "{\n \"entities\": [\n {\n \"entity_id\": \"alice_johnson\",\n \"operations\": [\n {\n \"op\": \"add\",\n \"target_index\": 1,\n \"content\": \"Works with Bob Lee on the release checklist.\"\n ,\n \"target_quote\": \"Runs release QA.\",\n \"reasoning\": \"Alice Johnson works with Bob Lee on the release checklist.\"\n ,\n \"relation\": {\n \"kind\": \"works-with\",\n \"target_name\": \"bob_lee\"\n ,\n \"note\": \"Preserves relationship to Bob Lee.\"\n }\n }\n ]\n }\n]\n,\n\"summary\": \"Added observation for Alice Johnson regarding her collaboration with Bob Lee on the release checklist, preserving the existing roadmap observation.\"\n}", "usage": {"input_tokens": 131, "output_tokens": 189, "total_tokens": 320}} diff --git a/tests/fixtures/schema_eval/results/local/summary.txt b/tests/fixtures/schema_eval/results/local/summary.txt new file mode 100644 index 000000000..dfbae33fe --- /dev/null +++ b/tests/fixtures/schema_eval/results/local/summary.txt @@ -0,0 +1,3 @@ +cases: 10 +schema_valid: 10/10 +average_content_preservation: 0.893