From cf027c9898435f8a8705d311acdcedaa03da5647 Mon Sep 17 00:00:00 2001 From: dawn <90008@klbr.net> Date: Tue, 30 Jun 2026 21:02:21 +0000 Subject: [PATCH] fix evidence rendered ref diagnostics --- .beads/interactions.jsonl | 1 + .beads/issues.jsonl | 3 ++- klbr-bench/src/longmemeval.rs | 61 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++-- klbr-core/src/evidence.rs | 56 +++++++++++++++++++++++++++++++++++++++++++++++++++++++- klbr-core/src/pipeline.rs | 82 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++------- 5 file(s) changed, 192 insertion(s)(+), 11 deletion(s)(-) diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -76,3 +76,4 @@ {"id":"int-788f0749","kind":"field_change","created_at":"2026-06-30T17:01:55.237649146Z","actor":"dawn","issue_id":"klbr-e28","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Completed: diagnosed renderer truncation of anchor/local answer bodies, landed prioritized packet rendering, added regression tests, and verified same-seed official QA improved from broken 0.32 and old 0.44 to 0.56."}} {"id":"int-eba49ab6","kind":"field_change","created_at":"2026-06-30T17:48:55.880302146Z","actor":"dawn","issue_id":"klbr-d76","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Not planned for now: user wants folgezettel usefulness evaluated from real-world usage traces rather than a synthetic agentic eval. Revisit only after real usage exposes concrete failures or measurable trace-review needs."}} {"id":"int-ce8a7dc1","kind":"field_change","created_at":"2026-06-30T20:47:27.910556331Z","actor":"dawn","issue_id":"klbr-k7d","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Completed larger query-time LongMemEval QA run, inspected miss buckets, and recorded follow-up work."}} +{"id":"int-01ad9c52","kind":"field_change","created_at":"2026-06-30T21:01:48.359954498Z","actor":"dawn","issue_id":"klbr-2jh","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Completed structural inspection and diagnostic fix; filed aggregate value representation follow-up."}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -41,7 +41,8 @@ {"_type":"issue","id":"klbr-wmz.2","title":"Route runtime passive recall through the canonical memory pipeline","description":"Bench retrieval now goes through MemoryPipeline, but runtime passive recall in klbr-core/src/agent.rs still embeds the prompt, calls MemoryStore::get_searchable, runs retrieval::retrieve_exact over legacy memories, then injects Context memory packets. That bypasses fts, exact refs, markdown notes, graph expansion, and the canonical lane/lifecycle path the docs describe.","design":"Avoid duplicating retrieval logic in agent.rs. Either make MemoryPipeline usable by AgentRuntime or extract a shared retrieval facade that both MemoryPipeline and runtime passive recall call.","acceptance_criteria":"Runtime passive recall uses the same lane-aware canonical retrieval and context packet assembly policy as the production pipeline; recalled packets can include fts/exact/dense/graph candidates from refs and markdown notes; archived/tombstoned/suppressed refs do not leak; klbr-core/src/instructions.md matches the actual memory packet format; tests or a focused integration fixture cover passive recall from a markdown note and from an explicit ref.","notes":"Runtime passive recall now calls MemoryPipeline::retrieve_evidence and injects shared EvidencePacket XML via Context::inject_evidence_packets; no-model runtime packet fixture covers turn-window expansion. Remaining acceptance is blocked on klbr-wmz.1 because dense search still starts from legacy memory rows before ref mapping.","status":"closed","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-26T17:53:19Z","created_by":"dawn","updated_at":"2026-06-26T19:44:55Z","started_at":"2026-06-26T19:37:32Z","closed_at":"2026-06-26T19:44:55Z","close_reason":"Completed: runtime passive recall now uses MemoryPipeline/EvidencePlanner packets, instructions document current packet XML, and fixtures cover turn-window recall plus explicit markdown refs; dense canonical dependency completed in klbr-wmz.1.","labels":["architecture","memory","retrieval","runtime"],"dependencies":[{"issue_id":"klbr-wmz.2","depends_on_id":"klbr-wmz","type":"parent-child","created_at":"2026-06-26T20:53:19Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-wmz.2","depends_on_id":"klbr-wmz.1","type":"blocks","created_at":"2026-06-26T22:37:53Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-wmz.1","title":"Move dense retrieval onto canonical refs and embedding_items","description":"Current source still has dense retrieval on the legacy memory-row surface: klbr-core/src/pipeline.rs search_dense calls MemoryStore::get_searchable and retrieval::retrieve_exact, while fts/exact retrieval uses canonical refs and promptable_text. The schema already has refs, promptable_text, markdown_note_chunks, and embedding_items, so dense retrieval should not be the odd path out.","design":"Prefer a ref-native embedding index backed by embedding_items. Backfill embeddings from promptable_text, keep memory-id aliases as compatibility aliases, and make klbr/full versus dense-only profiles exercise the same canonical identity layer as fts and exact retrieval.","acceptance_criteria":"Dense candidate generation works over canonical ref ids for memories, turn chunks, markdown note chunks, episode notes, profile notes, and procedural notes; lane and lifecycle filtering come from refs/ref_metadata instead of memory tags alone; benchmark traces return canonical refs for dense hits; regression tests cover a markdown-note-only hit and a tombstoned/suppressed ref not leaking through dense search.","status":"closed","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-26T17:53:11Z","created_by":"dawn","updated_at":"2026-06-26T19:43:27Z","started_at":"2026-06-26T19:38:02Z","closed_at":"2026-06-26T19:43:27Z","close_reason":"Completed: dense candidate generation now lazily backfills embedding_items from active promptable refs, scores canonical ref embeddings directly, and tests markdown-note dense hits plus suppressed-ref filtering.","labels":["architecture","memory","refs","retrieval"],"dependencies":[{"issue_id":"klbr-wmz.1","depends_on_id":"klbr-wmz","type":"parent-child","created_at":"2026-06-26T20:53:11Z","created_by":"dawn","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"klbr-wmz","title":"Finish memory architecture follow-through","description":"Tracks the remaining memory architecture work identified from docs/memory-arch.md, docs/memory-benches.md, docs/memory-implementation-status.md, and current klbr-core/klbr-bench source. Current status says pipeline, typed memory packets, markdown notes, edge mirroring, lifecycle projection, and benchmark runner integration exist; this epic is for gaps still present in source/docs.","acceptance_criteria":"Close when the child issues are complete, docs/memory-implementation-status.md is updated from current verification, and the architecture docs no longer point at missing or stale follow-up work.","status":"closed","priority":1,"issue_type":"epic","owner":"90008@klbr.net","created_at":"2026-06-26T17:52:54Z","created_by":"dawn","updated_at":"2026-06-26T23:45:35Z","closed_at":"2026-06-26T23:45:35Z","close_reason":"All 19 memory architecture child issues are closed; docs/status were updated from current verification; remaining official evaluator run is tracked separately as external blocked klbr-1yn.","labels":["architecture","memory"],"dependency_count":0,"dependent_count":0,"comment_count":0} -{"_type":"issue","id":"klbr-2jh","title":"Inspect aggregate and value-synthesis LongMemEval misses","description":"The sample-100 query-time LongMemEval run showed 12 official failures where the answer ref was rendered but the answer value was not visible, plus 10 value-visible reader or synthesis failures. Inspect aggregate_count, aggregate_sum, lookup, and update_resolution misses to decide whether packet rendering, fact extraction, or final reader prompting needs the next fix. Use measured miss buckets rather than broad context fanout.","status":"open","priority":2,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-30T20:47:37Z","created_by":"dawn","updated_at":"2026-06-30T20:47:37Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"klbr-jae","title":"Design aggregate evidence value representation","description":"LongMemEval aggregate misses show that a prompt fix would be benchmark-shaped and brittle. The structural gap is evidence representation: EvidenceFactRow currently stores only the first numeric value from a chunk, often capturing list ordinals, dates, or distractors instead of the values needed for counts/sums. Design and implement a general value representation for memory evidence that can keep multiple numeric mentions with nearby unit/currency/date/entity context, then only enable deterministic aggregate synthesis when the rendered facts are grounded and unambiguous.","status":"open","priority":2,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-30T21:01:47Z","created_by":"dawn","updated_at":"2026-06-30T21:01:47Z","dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"klbr-2jh","title":"Inspect aggregate and value-synthesis LongMemEval misses","description":"The sample-100 query-time LongMemEval run showed 12 official failures where the answer ref was rendered but the answer value was not visible, plus 10 value-visible reader or synthesis failures. Inspect aggregate_count, aggregate_sum, lookup, and update_resolution misses to decide whether packet rendering, fact extraction, or final reader prompting needs the next fix. Use measured miss buckets rather than broad context fanout.","notes":"Inspected sample-100 aggregate/value misses without prompt changes. User explicitly rejected prompt-fix/hardcoding route. Findings: many aggregate misses are not simple retrieval failures; literal answer_value_visible is misleading for derived answers like 5+3=8 or component expenses summing to 85, and short numeric answers can produce false positives from unrelated dates/list ordinals. Also found a real diagnostic bug: assemble_context marked every selected packet ref as used/rendered even when render_evidence_packet omitted that ref under packet budget. Fixed core ref accounting so render_evidence_packet_with_refs reports emitted body/fact/timeline refs and assemble_context.used_refs tracks only those. Tightened bench diagnostics so answer_value_visible requires an answer-bearing ref to have been emitted, and critical_fact_row_visible requires a fact row for an answer-bearing ref. Targeted retrieval-only rerun for 46a3abf7 now reports answer_bearing_ref_selected=1.0 but rendered/in_context/value_visible/critical_fact_row_visible=0.0, correctly rebucketing it as selected_not_rendered instead of value-visible reader failure. Tests: cargo fmt --check, cargo test -p klbr-core, cargo test -p klbr-bench, git diff --check.","status":"closed","priority":2,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-30T20:47:37Z","created_by":"dawn","updated_at":"2026-06-30T21:01:48Z","started_at":"2026-06-30T20:49:47Z","closed_at":"2026-06-30T21:01:48Z","close_reason":"Completed structural inspection and diagnostic fix; filed aggregate value representation follow-up.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-4ls","title":"Design time-aware passive memory ranking","description":"LongMemEval temporal misses exposed that BenchQuery.reference_time is currently not used by dense/sparse candidate ranking, and temporal evidence is selected mostly by semantic/entity similarity. Design and implement a measured ranking policy that uses query reference_time and evidence timestamps to support relative-date, elapsed-time, and event-order questions without hard-coded dataset cues or broad context fanout.","notes":"sample-100 query-time run finished with temporal-reasoning accuracy 0.5556 over 27 examples. On the 25 overlapping sample-25 ids, query-date rendering recovered b46e15ee and gpt4_b5700ca9, both relative-date examples. Remaining temporal failures are still mostly order_or_rank: 12 temporal misses total, including 5 rendered-value-not-visible, 3 selected-not-rendered, 2 not-selected, and 2 value-visible reader/synthesis. Design should use BenchQuery.reference_time plus evidence timestamps during candidate or packet ranking, not dataset cue hacks.","status":"open","priority":2,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-30T17:49:11Z","created_by":"dawn","updated_at":"2026-06-30T20:47:38Z","dependencies":[{"issue_id":"klbr-4ls","depends_on_id":"klbr-k7d","type":"blocks","created_at":"2026-06-30T20:49:19Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-k7d","title":"Run larger LongMemEval QA and inspect remaining misses","description":"Run a larger same-protocol LongMemEval-S QA sample after the anchor-window evidence rendering fix, compare against the sample-25 artifacts, inspect remaining miss categories while the run is in flight, and collect broader retrieval/memory-eval research perspectives before deciding next implementation work.","notes":"sample-100 query-time run finished in benchmarks/runs/longmemeval-s/klbr-full/2026-06-30_qa_sample100_seed4937553249516211047_query_time_qa. Official LongMemEval-S QA accuracy: 0.61 over 100 evaluated, 0 skipped. Retrieval metrics: RecallAny@5 0.98, RecallAll@5 0.85, PacketRecallAny@5 0.98, PacketRecallAll@5 0.78, answer_bearing_ref_selected 0.81, answer_bearing_ref_in_context 0.74, answer_value_visible 0.47. Per type accuracy: single-session-assistant 0.9091, knowledge-update 0.8667, single-session-user 0.7143, temporal-reasoning 0.5556, multi-session 0.4074, single-session-preference 0.3333. Miss buckets among 39 official failures: 12 answer ref not selected, 5 selected not rendered, 12 rendered but answer value not visible, 10 value-visible reader or synthesis failures. Misses by type: 16 multi-session, 12 temporal-reasoning, 4 single-session-user, 4 single-session-preference, 2 knowledge-update, 1 single-session-assistant. Misses by op: 17 order_or_rank, 6 aggregate_count, 5 aggregate_sum, 4 preference_recommendation, 4 lookup, 3 update_resolution. Compared on the 25 overlapping sample-25 ids, official labels moved 14/25 to 15/25: recovered 6b168ec8, b46e15ee, gpt4_b5700ca9; regressed 51a45a95 and b5ef892d. The query reference date rendering specifically helped relative-date examples, but sample100 still supports separate follow-up for time-aware ranking and aggregate/value synthesis.","status":"closed","priority":2,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-30T17:31:26Z","created_by":"dawn","updated_at":"2026-06-30T20:47:28Z","started_at":"2026-06-30T17:31:31Z","closed_at":"2026-06-30T20:47:28Z","close_reason":"Completed larger query-time LongMemEval QA run, inspected miss buckets, and recorded follow-up work.","dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"klbr-54l.6","title":"Probe passive writer retention for atomic facts and events","description":"Research on memory systems suggests passive QA can fail because write-time compression loses entity/slot/value/time facts before retrieval ever has a chance. Klbr now writes episode_event_card artifacts and generic fact_rows, but the next passive-recall improvement pass should add writer-side probes for counts, updates, preference constraints, and temporal facts so we can tell whether the passive writer retained the answer-bearing atom before tuning retrieval.","acceptance_criteria":"Fixtures or diagnostics check that answer-bearing entity/slot/value/time atoms exist in stored promptable artifacts before retrieval; failures are reported separately from packet/ranking misses; at least count, update_resolution, temporal order, and preference-constraint cases are covered.","status":"closed","priority":2,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-30T11:13:57Z","created_by":"dawn","updated_at":"2026-06-30T12:55:18Z","closed_at":"2026-06-30T12:55:18Z","close_reason":"trace rows include writer_retention probes for answer refs and episode cards; same-seed sample reports writer vs retrieval misses separately","labels":["bench","longmemeval","memory","writer"],"dependencies":[{"issue_id":"klbr-54l.6","depends_on_id":"klbr-54l","type":"parent-child","created_at":"2026-06-30T14:13:56Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-54l.6","depends_on_id":"klbr-54l.1","type":"blocks","created_at":"2026-06-30T14:14:18Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} diff --git a/klbr-bench/src/longmemeval.rs b/klbr-bench/src/longmemeval.rs --- a/klbr-bench/src/longmemeval.rs +++ b/klbr-bench/src/longmemeval.rs @@ -649,6 +649,36 @@ } #[test] + fn answer_value_visible_metric_requires_rendered_answer_ref() { + let mut item = test_question("q-visible", "multi-session"); + item.answer = serde_json::Value::String("3".to_string()); + let answer_refs = HashSet::from(["ref_answer".to_string()]); + let no_answer_refs_rendered = HashSet::from(["ref_other".to_string()]); + let answer_ref_rendered = HashSet::from(["ref_answer".to_string()]); + + assert!(!answer_value_visible_for_rendered_answer_refs( + &item, + "random list item", + &answer_refs, + &no_answer_refs_rendered, + )); + assert!(answer_value_visible_for_rendered_answer_refs( + &item, + "three tanks", + &answer_refs, + &answer_ref_rendered, + )); + assert!(!critical_fact_row_visible_for_answer_refs( + "three tanks", + &answer_refs, + )); + assert!(critical_fact_row_visible_for_answer_refs( + "three tanks", + &answer_refs, + )); + } + + #[test] fn default_pipeline_run_dir_lives_under_benchmarks_runs() { let out_dir = default_pipeline_run_dir("longmemeval-s", "raw-turns/fts-only"); let parts = out_dir @@ -2228,8 +2258,14 @@ .iter() .any(|ref_id| used_refs.contains(ref_id)); let answer_bearing_ref_in_context = answer_bearing_ref_rendered; - let answer_value_visible = answer_value_visible_in_context(item, &context.content); - let critical_fact_row_visible = answer_value_visible && context.content.contains(", + used_refs: &HashSet, +) -> bool { + answer_ref_ids + .iter() + .any(|ref_id| used_refs.contains(ref_id)) + && answer_value_visible_in_context(item, context) +} + +fn critical_fact_row_visible_for_answer_refs( + context: &str, + answer_ref_ids: &HashSet, +) -> bool { + answer_ref_ids + .iter() + .any(|ref_id| context.contains(&format!(", +} + pub fn render_evidence_packet(packet: &EvidencePacket, max_tokens: usize) -> String { + render_evidence_packet_with_refs(packet, max_tokens).content +} + +pub fn render_evidence_packet_with_refs( + packet: &EvidencePacket, + max_tokens: usize, +) -> RenderedEvidencePacket { let max_chars = max_tokens.saturating_mul(4); let lanes = packet .lanes @@ -506,6 +519,7 @@ xml_escape(packet.session_id.as_deref().unwrap_or("")), packet.estimated_tokens.min(max_tokens), ); + let mut rendered_refs = Vec::new(); let closing_chars = " \n".len(); let anchor_body_index = packet .refs @@ -518,6 +532,9 @@ { if push_packet_body(&mut out, packet, index, max_chars, closing_chars) { priority_body_indices.push(index); + if let Some(ref_id) = packet.refs.get(index) { + push_rendered_ref(&mut rendered_refs, ref_id); + } } } if !packet.fact_rows.is_empty() { @@ -566,6 +583,7 @@ break; } out.push_str(&line); + push_rendered_ref(&mut rendered_refs, &fact.ref_id); } out.push_str(close); } @@ -603,6 +621,7 @@ break; } out.push_str(&line); + push_rendered_ref(&mut rendered_refs, &event.ref_id); } out.push_str(close); } @@ -614,9 +633,15 @@ if !push_packet_body(&mut out, packet, index, max_chars, closing_chars) { break; } + if let Some(ref_id) = packet.refs.get(index) { + push_rendered_ref(&mut rendered_refs, ref_id); + } } out.push_str(" \n"); - out + RenderedEvidencePacket { + content: out, + rendered_refs, + } } fn format_timestamp_utc(timestamp: i64) -> Option { @@ -626,6 +651,12 @@ Utc.timestamp_opt(timestamp, 0) .single() .map(|dt| dt.to_rfc3339_opts(SecondsFormat::Secs, true)) +} + +fn push_rendered_ref(rendered_refs: &mut Vec, ref_id: &str) { + if !rendered_refs.iter().any(|seen| seen == ref_id) { + rendered_refs.push(ref_id.to_string()); + } } fn push_packet_body( @@ -1195,6 +1226,29 @@ assert!(rendered.contains("date=\"1970-01-01T00:00:42Z\"")); assert!(rendered.contains("value=\"27:45\"")); assert!(rendered.contains(" remaining_tokens { break; } remaining_tokens = remaining_tokens.saturating_sub(rendered_tokens); - for ref_id in &packet.refs { - if !used_refs.contains(ref_id) { - used_refs.push(ref_id.clone()); + for ref_id in rendered_packet.rendered_refs { + if !used_refs.contains(&ref_id) { + used_refs.push(ref_id); } } packets.push_str(&rendered); @@ -2050,6 +2051,73 @@ assert!(context .content .contains("reference_date=\"1970-01-01T00:00:03Z\"")); + Ok(()) + } + + #[tokio::test] + async fn assemble_context_tracks_only_refs_with_emitted_evidence() -> Result<()> { + let tmp = NamedTempFile::new()?; + let store = MemoryStore::open(tmp.path().to_str().unwrap(), 4)?; + let llm = LlmClient::new(crate::models::ModelsConfig::default()); + let pipeline = MemoryPipeline::new(store, llm, MemoryConfig::default()); + let budget = ContextBudget { + max_tokens: 96, + top_k: 1, + graph_depth: 0, + }; + let packet = EvidencePacket { + packet_id: "pkt_budget".to_string(), + session_id: Some("s1".to_string()), + anchor_ref: "ref_anchor".to_string(), + packet_kind: EvidencePacketKind::TurnWindow, + refs: vec!["ref_anchor".to_string(), "ref_omitted".to_string()], + bodies: vec![ + "anchor evidence ".repeat(80), + "omitted support says the hidden answer is Target".to_string(), + ], + fact_rows: vec![], + timeline: vec![], + signals: EvidencePacketSignals { + sources: vec!["fts".to_string()], + seed_count: 1, + best_source: "fts".to_string(), + best_score: 0.0, + per_source: vec![EvidenceSourceSignal { + source: "fts".to_string(), + ref_id: "ref_anchor".to_string(), + rank: 1, + score: 0.0, + }], + }, + lanes: vec![MemoryLane::Episodic], + estimated_tokens: 240, + }; + let retrieval = PipelineRetrievalTrace { + query_id: "q-budget".to_string(), + routed_lanes: vec![MemoryLane::Episodic], + route: LaneRoute::default(), + op_plan: OpPlan::default(), + stage_one: StageOneTrace::default(), + exact_refs: vec![], + candidates: vec![], + packets: vec![packet], + packet_omissions: vec![], + coverage: EvidenceCoverageTrace::default(), + packet_rerank: None, + }; + let query = BenchQuery { + query_id: "q-budget".to_string(), + text: "what is the hidden answer?".to_string(), + reference_time: None, + op_hint: None, + }; + + let context = pipeline.assemble_context(query, &retrieval, budget).await?; + + assert!(context.content.contains("refs=\"ref_anchor ref_omitted\"")); + assert!(context.content.contains("