diff --git a/.beads/interactions.jsonl b/.beads/interactions.jsonl index a3acd8c..e663209 100644 --- a/.beads/interactions.jsonl +++ b/.beads/interactions.jsonl @@ -51,3 +51,4 @@ {"id":"int-c8b11ed4","kind":"field_change","created_at":"2026-06-28T11:20:38.81144388Z","actor":"dawn","issue_id":"klbr-a9b","extra":{"field":"status","new_value":"closed","old_value":"open","reason":"Completed first-pass generic tree-sitter file-local code retrieval, navigation, symbolic edits, tests, and docs."}} {"id":"int-ca7d767e","kind":"field_change","created_at":"2026-06-28T16:16:07.053939933Z","actor":"dawn","issue_id":"klbr-8dq","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Completed: LongMemEval pipeline run now defaults --out under benchmarks/runs."}} {"id":"int-fb2d145f","kind":"field_change","created_at":"2026-06-28T16:16:07.11528074Z","actor":"dawn","issue_id":"klbr-oba","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Completed: added reproducible random LongMemEval sampling with question_type stratification."}} +{"id":"int-fdca5c0e","kind":"field_change","created_at":"2026-06-28T19:54:14.601069054Z","actor":"dawn","issue_id":"klbr-f0q.1","extra":{"field":"status","new_value":"closed","old_value":"in_progress","reason":"Implemented OpPlan/QueryOp classification, serialized op_plan into retrieval traces, preserved lookup defaults, added deterministic planner and pipeline trace tests."}} diff --git a/.beads/issues.jsonl b/.beads/issues.jsonl index 1789c2c..236054f 100644 --- a/.beads/issues.jsonl +++ b/.beads/issues.jsonl @@ -1,3 +1,8 @@ +{"_type":"issue","id":"klbr-f0q.1","title":"Add operation-aware query planning","description":"Introduce a small OpPlan/QueryOp classifier for memory QA queries: lookup, aggregate_count, aggregate_sum, aggregate_avg, order_or_rank, update_resolution, preference_recommendation, and abstain_or_false_premise_check. Use it to switch only high-value classes away from plain lookup behavior.","design":"Keep the label set intentionally small and rule-based first; avoid growing English lexical hacks into the main ranking decision boundary.","acceptance_criteria":"Production memory retrieval traces expose the selected query op and plan; existing lookup behavior remains unchanged by default; aggregate/order/update/preference queries can be detected deterministically in tests.","status":"closed","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T19:44:46Z","created_by":"dawn","updated_at":"2026-06-28T19:54:15Z","started_at":"2026-06-28T19:45:04Z","closed_at":"2026-06-28T19:54:15Z","close_reason":"Implemented OpPlan/QueryOp classification, serialized op_plan into retrieval traces, preserved lookup defaults, added deterministic planner and pipeline trace tests.","dependencies":[{"issue_id":"klbr-f0q.1","depends_on_id":"klbr-f0q","type":"parent-child","created_at":"2026-06-28T22:44:46Z","created_by":"dawn","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"klbr-f0q.2","title":"Add structured synthesis for planned QA operations","description":"Add a fact-table synthesis path for aggregate count/sum/avg, ordering, previous/latest update resolution, and preference-grounded recommendations. Do not route ordinary conversation lookup through the structured path unless the planner selects it.","design":"Stage A extracts/normalizes fact rows with refs; Stage B computes or renders the answer according to the OpPlan.","acceptance_criteria":"Planned synthesis computes counts/averages/order/update answers from cited fact rows; preference recommendations require personal support and avoid generic distractor answers; lookup questions keep current direct reader path.","status":"open","priority":1,"issue_type":"feature","owner":"90008@klbr.net","created_at":"2026-06-28T19:44:46Z","created_by":"dawn","updated_at":"2026-06-28T19:44:46Z","dependencies":[{"issue_id":"klbr-f0q.2","depends_on_id":"klbr-f0q","type":"parent-child","created_at":"2026-06-28T22:44:46Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-f0q.2","depends_on_id":"klbr-f0q.4","type":"blocks","created_at":"2026-06-28T22:44:56Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"klbr-f0q.3","title":"Implement collect-mode evidence assembly","description":"For aggregate, ordering, and update-resolution plans, retrieve and rank evidence by fact-group coverage instead of fixed top-k alone. Add sufficiency/gap diagnosis and clustered packet assembly by entity/task/session where feasible.","design":"Use relevance plus marginal coverage gain, with hard budget/stopping controls based on fact-group sufficiency rather than packet count.","acceptance_criteria":"Collect mode can cover multi-session fact groups without one high-scoring session monopolizing context; packet traces expose coverage/gap state; focused tests cover six-way count/order style cases.","notes":"Started implementation: collect-mode plans now widen candidate/packet limits and use collect-aware packet capping that prefers distinct sessions before second packets from the same session. Remaining: explicit fact-group sufficiency and gap diagnosis.","status":"in_progress","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T19:44:46Z","created_by":"dawn","updated_at":"2026-06-28T19:54:15Z","started_at":"2026-06-28T19:54:15Z","dependencies":[{"issue_id":"klbr-f0q.3","depends_on_id":"klbr-f0q","type":"parent-child","created_at":"2026-06-28T22:44:46Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-f0q.3","depends_on_id":"klbr-f0q.1","type":"blocks","created_at":"2026-06-28T22:44:56Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"klbr-f0q.4","title":"Render chronology and fact rows in evidence packets","description":"Extend evidence packet rendering to include compact structured timeline/fact rows alongside raw ref-grounded bodies, and make rendering report selected/rendered/dropped refs or fact rows.","design":"Structured rows must retain supporting ref ids; raw bodies stay as the provenance source, not replaced by summaries.","acceptance_criteria":"Rendered context includes chronology/facts for temporal/aggregate/update packets while preserving body refs; benchmarks can distinguish selected refs from rendered refs and visible critical facts.","status":"open","priority":1,"issue_type":"feature","owner":"90008@klbr.net","created_at":"2026-06-28T19:44:46Z","created_by":"dawn","updated_at":"2026-06-28T19:44:46Z","dependencies":[{"issue_id":"klbr-f0q.4","depends_on_id":"klbr-f0q","type":"parent-child","created_at":"2026-06-28T22:44:46Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-f0q.4","depends_on_id":"klbr-f0q.3","type":"blocks","created_at":"2026-06-28T22:44:56Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} +{"_type":"issue","id":"klbr-f0q","title":"Planner-grade evidence assembly","description":"Implement docs/planner-grade-evidence-assembly.md: make memory QA operation-aware instead of relying on larger top-k/budget. Scope includes query operation planning, collect-mode evidence assembly, chronology/fact packet rendering, structured synthesis for aggregate/temporal/preference questions, and benchmark metrics/fixtures.","design":"Follow the implementation order in docs/planner-grade-evidence-assembly.md: classify operation first, add collect-mode evidence planning, expose facts/timeline while keeping raw refs, add structured synthesis only for op labels that need it, then guard with metrics and fixtures.","acceptance_criteria":"Planner-grade evidence assembly is implemented behind conservative defaults or config/CLI toggles, covered by production-pipeline tests, documented, and validated with focused cargo tests plus a bench smoke when local services are available.","status":"in_progress","priority":1,"issue_type":"epic","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T19:44:25Z","created_by":"dawn","updated_at":"2026-06-28T19:45:04Z","started_at":"2026-06-28T19:45:04Z","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-a9b.8","title":"Extract file-local symbols from tree-sitter tags","description":"Build symbol extraction on top of generic tree-sitter language bundles. Prefer tags/locals/code-intel APIs when a language provides them, and keep the internal symbol model independent of any one grammar. Convert definition/reference captures or equivalent structure output into name, kind, role, name range, body range, parent/name path when derivable, docs when provided, and source snippet boundaries.","acceptance_criteria":"Fixture files in multiple representative languages produce stable symbol lists with definitions, ranges, names, and basic nesting/name paths where the language bundle supports it; extraction tolerates parse errors by returning partial symbols plus parse status rather than panicking; unsupported query features degrade cleanly.","status":"closed","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T10:46:33Z","created_by":"dawn","updated_at":"2026-06-28T11:06:00Z","started_at":"2026-06-28T11:02:44Z","closed_at":"2026-06-28T11:06:00Z","close_reason":"Implemented generic file-local symbol extraction through tree-sitter-tags over language-pack query bundles, including definition/reference roles, capture-derived kinds, ranges, docs, containment name paths, and focused tests.","labels":["code-intel","retrieval","tools","tree-sitter"],"dependencies":[{"issue_id":"klbr-a9b.8","depends_on_id":"klbr-a9b","type":"parent-child","created_at":"2026-06-28T13:46:32Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-a9b.8","depends_on_id":"klbr-a9b.6","type":"blocks","created_at":"2026-06-28T13:46:50Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":3,"comment_count":0} {"_type":"issue","id":"klbr-a9b.1","title":"Define tree-sitter code-intel backend boundary","description":"Decide the first-pass tree-sitter backend shape before code lands: supported languages, crate choices, tool naming, data structures, and explicit non-goals. The boundary should keep this file-local and tree-sitter-first, with no LSP, diagnostics tool, project-wide semantic resolution, or automatic check/format hook in the first slice.","design":"first pass is tree-sitter-first, file-local, and generic across languages. the implementation should not hardcode rust/typescript as the architecture; concrete languages are only fixtures. the core model is source text plus path/language hint -\u003e generic parser/query registry -\u003e tags/locals captures or generic code-intel output -\u003e file-local symbols with kind/name/name_path/ranges/docs/parse status. prefer tree-sitter-language-pack or an equivalent bundle registry so languages can be added by parser/query availability rather than bespoke code. tool behavior follows serena: OK on successful edits; concise errors for unsupported language, missing/ambiguous symbol, parse/range failure, or invalid replacement. no lsp backend, no diagnostics tool, no automatic cargo/check/format hook, no verification hints, and no project-wide semantic resolution in this first implementation. project-wide indexing can be a later layer over the same file-local extractor.","acceptance_criteria":"A short design note is recorded in the issue or docs; the first implementation surface lists supported languages, commands/tools, and unsupported behavior; later implementation issues can proceed without re-litigating LSP/project-wide scope.","status":"closed","priority":1,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T10:46:31Z","created_by":"dawn","updated_at":"2026-06-28T10:59:12Z","started_at":"2026-06-28T10:55:29Z","closed_at":"2026-06-28T10:55:43Z","close_reason":"Recorded tree-sitter-first, file-local first-pass boundary and explicit non-goals in issue design.","labels":["code-intel","design","tools","tree-sitter"],"dependencies":[{"issue_id":"klbr-a9b.1","depends_on_id":"klbr-a9b","type":"parent-child","created_at":"2026-06-28T13:46:30Z","created_by":"dawn","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"klbr-a9b.6","title":"Add tree-sitter parser and language registry","description":"Add the klbr-core substrate for selecting a tree-sitter parser and query bundle from a file path or optional language hint. This should be generic: use tree-sitter-language-pack or an equivalent registry/loader rather than handcoding a small set of language crates. Concrete languages in tests are only fixtures proving the generic path.","acceptance_criteria":"Given a supported file path or language hint and source text, klbr can parse it with tree-sitter, report parse errors/MISSING nodes as parse status, and expose language metadata plus tags/locals/query availability when present; unsupported or unavailable languages return a concise unsupported-language error without panicking.","status":"closed","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T10:46:31Z","created_by":"dawn","updated_at":"2026-06-28T11:02:31Z","started_at":"2026-06-28T10:55:54Z","closed_at":"2026-06-28T11:02:31Z","close_reason":"Implemented generic tree-sitter language detection, query support, and parse-status substrate via tree-sitter-language-pack; focused code_intel tests and klbr-core check pass.","labels":["code-intel","tools","tree-sitter"],"dependencies":[{"issue_id":"klbr-a9b.6","depends_on_id":"klbr-a9b","type":"parent-child","created_at":"2026-06-28T13:46:31Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-a9b.6","depends_on_id":"klbr-a9b.1","type":"blocks","created_at":"2026-06-28T13:46:50Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":1,"comment_count":0} @@ -22,6 +27,7 @@ {"_type":"issue","id":"klbr-wmz.2","title":"Route runtime passive recall through the canonical memory pipeline","description":"Bench retrieval now goes through MemoryPipeline, but runtime passive recall in klbr-core/src/agent.rs still embeds the prompt, calls MemoryStore::get_searchable, runs retrieval::retrieve_exact over legacy memories, then injects Context memory packets. That bypasses fts, exact refs, markdown notes, graph expansion, and the canonical lane/lifecycle path the docs describe.","design":"Avoid duplicating retrieval logic in agent.rs. Either make MemoryPipeline usable by AgentRuntime or extract a shared retrieval facade that both MemoryPipeline and runtime passive recall call.","acceptance_criteria":"Runtime passive recall uses the same lane-aware canonical retrieval and context packet assembly policy as the production pipeline; recalled packets can include fts/exact/dense/graph candidates from refs and markdown notes; archived/tombstoned/suppressed refs do not leak; klbr-core/src/instructions.md matches the actual memory packet format; tests or a focused integration fixture cover passive recall from a markdown note and from an explicit ref.","notes":"Runtime passive recall now calls MemoryPipeline::retrieve_evidence and injects shared EvidencePacket XML via Context::inject_evidence_packets; no-model runtime packet fixture covers turn-window expansion. Remaining acceptance is blocked on klbr-wmz.1 because dense search still starts from legacy memory rows before ref mapping.","status":"closed","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-26T17:53:19Z","created_by":"dawn","updated_at":"2026-06-26T19:44:55Z","started_at":"2026-06-26T19:37:32Z","closed_at":"2026-06-26T19:44:55Z","close_reason":"Completed: runtime passive recall now uses MemoryPipeline/EvidencePlanner packets, instructions document current packet XML, and fixtures cover turn-window recall plus explicit markdown refs; dense canonical dependency completed in klbr-wmz.1.","labels":["architecture","memory","retrieval","runtime"],"dependencies":[{"issue_id":"klbr-wmz.2","depends_on_id":"klbr-wmz","type":"parent-child","created_at":"2026-06-26T20:53:19Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-wmz.2","depends_on_id":"klbr-wmz.1","type":"blocks","created_at":"2026-06-26T22:37:53Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-wmz.1","title":"Move dense retrieval onto canonical refs and embedding_items","description":"Current source still has dense retrieval on the legacy memory-row surface: klbr-core/src/pipeline.rs search_dense calls MemoryStore::get_searchable and retrieval::retrieve_exact, while fts/exact retrieval uses canonical refs and promptable_text. The schema already has refs, promptable_text, markdown_note_chunks, and embedding_items, so dense retrieval should not be the odd path out.","design":"Prefer a ref-native embedding index backed by embedding_items. Backfill embeddings from promptable_text, keep memory-id aliases as compatibility aliases, and make klbr/full versus dense-only profiles exercise the same canonical identity layer as fts and exact retrieval.","acceptance_criteria":"Dense candidate generation works over canonical ref ids for memories, turn chunks, markdown note chunks, episode notes, profile notes, and procedural notes; lane and lifecycle filtering come from refs/ref_metadata instead of memory tags alone; benchmark traces return canonical refs for dense hits; regression tests cover a markdown-note-only hit and a tombstoned/suppressed ref not leaking through dense search.","status":"closed","priority":1,"issue_type":"feature","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-26T17:53:11Z","created_by":"dawn","updated_at":"2026-06-26T19:43:27Z","started_at":"2026-06-26T19:38:02Z","closed_at":"2026-06-26T19:43:27Z","close_reason":"Completed: dense candidate generation now lazily backfills embedding_items from active promptable refs, scores canonical ref embeddings directly, and tests markdown-note dense hits plus suppressed-ref filtering.","labels":["architecture","memory","refs","retrieval"],"dependencies":[{"issue_id":"klbr-wmz.1","depends_on_id":"klbr-wmz","type":"parent-child","created_at":"2026-06-26T20:53:11Z","created_by":"dawn","metadata":"{}"}],"dependency_count":0,"dependent_count":1,"comment_count":0} {"_type":"issue","id":"klbr-wmz","title":"Finish memory architecture follow-through","description":"Tracks the remaining memory architecture work identified from docs/memory-arch.md, docs/memory-benches.md, docs/memory-implementation-status.md, and current klbr-core/klbr-bench source. Current status says pipeline, typed memory packets, markdown notes, edge mirroring, lifecycle projection, and benchmark runner integration exist; this epic is for gaps still present in source/docs.","acceptance_criteria":"Close when the child issues are complete, docs/memory-implementation-status.md is updated from current verification, and the architecture docs no longer point at missing or stale follow-up work.","status":"closed","priority":1,"issue_type":"epic","owner":"90008@klbr.net","created_at":"2026-06-26T17:52:54Z","created_by":"dawn","updated_at":"2026-06-26T23:45:35Z","closed_at":"2026-06-26T23:45:35Z","close_reason":"All 19 memory architecture child issues are closed; docs/status were updated from current verification; remaining official evaluator run is tracked separately as external blocked klbr-1yn.","labels":["architecture","memory"],"dependency_count":0,"dependent_count":0,"comment_count":0} +{"_type":"issue","id":"klbr-f0q.5","title":"Add planner-grade benchmark metrics and fixtures","description":"Add rendered-evidence metrics and deterministic production-pipeline fixtures for count, average truncation, ordered temporal sessions, previous-vs-latest, and preference distractor regressions.","design":"Metrics should separate retrieval selection, packet rendering, fact-row visibility, and final answer correctness rather than collapsing them into one recall score.","acceptance_criteria":"Bench reports split selected/rendered/visible evidence metrics; tests cover the concrete failure families from the 20-sample LongMemEval brief; docs list smoke commands for planner/fact-table runs.","notes":"Partial metrics support landed early: LongMemEval reports op_plan_counts plus answer_bearing_ref_selected, answer_bearing_ref_rendered, and answer_value_visible. Remaining: critical_fact_row_visible, collect_mode_fact_group_recall, preference/update-specific correctness metrics, and deterministic fixtures for all failure families.","status":"open","priority":2,"issue_type":"task","owner":"90008@klbr.net","created_at":"2026-06-28T19:44:47Z","created_by":"dawn","updated_at":"2026-06-28T19:54:15Z","dependencies":[{"issue_id":"klbr-f0q.5","depends_on_id":"klbr-f0q","type":"parent-child","created_at":"2026-06-28T22:44:46Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-f0q.5","depends_on_id":"klbr-f0q.2","type":"blocks","created_at":"2026-06-28T22:44:56Z","created_by":"dawn","metadata":"{}"}],"dependency_count":1,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-oba","title":"add stratified random longmemeval sampling","description":"support reproducible random subset runs for the LongMemEval pipeline, stratified by question_type so small incremental qa runs still cover categories.","status":"closed","priority":2,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T16:09:37Z","created_by":"dawn","updated_at":"2026-06-28T16:16:07Z","started_at":"2026-06-28T16:10:01Z","closed_at":"2026-06-28T16:16:07Z","close_reason":"Completed: added reproducible random LongMemEval sampling with question_type stratification.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-8dq","title":"default longmemeval qa output under benchmarks/runs","description":"make the qa bench less annoying to run by defaulting its output directory under benchmarks/runs when --out is not supplied.","status":"closed","priority":2,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T15:55:48Z","created_by":"dawn","updated_at":"2026-06-28T16:16:07Z","started_at":"2026-06-28T15:56:03Z","closed_at":"2026-06-28T16:16:07Z","close_reason":"Completed: LongMemEval pipeline run now defaults --out under benchmarks/runs.","dependency_count":0,"dependent_count":0,"comment_count":0} {"_type":"issue","id":"klbr-a9b.7","title":"Document and test tree-sitter code tools","description":"Add focused fixtures, regression tests, and docs for the tree-sitter-first code tools. Tests should cover multiple representative languages through the generic registry, symbol extraction, overview/find/read flows, symbolic edits, terse OK/error behavior, and the explicitly skipped diagnostics/project-wide/LSP scope.","acceptance_criteria":"Core tests cover supported generic file-local behavior and failure modes; docs explain parser/query-bundle availability, the first-pass boundary, and future expansion path; the epic's verification commands are recorded after implementation lands.","status":"closed","priority":2,"issue_type":"task","assignee":"dawn","owner":"90008@klbr.net","created_at":"2026-06-28T10:46:32Z","created_by":"dawn","updated_at":"2026-06-28T11:20:25Z","started_at":"2026-06-28T11:18:27Z","closed_at":"2026-06-28T11:20:25Z","close_reason":"Added code-intel docs, registration coverage, and recorded focused verification commands.","labels":["code-intel","docs","tests","tools","tree-sitter"],"dependencies":[{"issue_id":"klbr-a9b.7","depends_on_id":"klbr-a9b","type":"parent-child","created_at":"2026-06-28T13:46:32Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-a9b.7","depends_on_id":"klbr-a9b.2","type":"blocks","created_at":"2026-06-28T13:46:51Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-a9b.7","depends_on_id":"klbr-a9b.3","type":"blocks","created_at":"2026-06-28T13:46:50Z","created_by":"dawn","metadata":"{}"},{"issue_id":"klbr-a9b.7","depends_on_id":"klbr-a9b.5","type":"blocks","created_at":"2026-06-28T13:46:58Z","created_by":"dawn","metadata":"{}"}],"dependency_count":3,"dependent_count":0,"comment_count":0} diff --git a/docs/planner-grade-evidence-assembly.md b/docs/planner-grade-evidence-assembly.md new file mode 100644 index 0000000..f62ff7c --- /dev/null +++ b/docs/planner-grade-evidence-assembly.md @@ -0,0 +1,314 @@ +# klbr’s next fix is planner-grade evidence assembly + +## diagnosis + +the good news is that the earlier “right session, wrong chunk” problem looks substantially less central now. your internal benchmark gap report already described the original failure as a selection-granularity bug, where session recall was high but qa accuracy lagged because answer-bearing neighboring turns were getting dropped; that report also notes the move to `EvidencePacket` planning as the production fix. your memory system report, meanwhile, describes `MemoryPipeline` as the shared facade for observing sessions, retrieving evidence, assembling context, and answering through the real stack rather than a benchmark-only shortcut. in other words, klbr is no longer mostly failing because it cannot *find* the right area; it is now more often failing because it does not always *plan the right evidence set and reasoning mode* for the question type. fileciteturn0file0 fileciteturn0file1 + +that lines up with the new higher-budget run pattern you pasted: increasing `top-k` and read budget improved packet/session coverage a bit, but did not lift official qa, and some cases even regressed. that is exactly what you expect when the remaining bottleneck is no longer coarse retrieval, but a mix of missing “collect-all” behavior for temporal/aggregate questions, weak update resolution, and distractor drift in preference questions. long-term memory research keeps seeing this same shape: long-horizon assistants struggle not just with recall, but with temporal reasoning, multi-session aggregation, and user-specific constraint retention; single-pass retrieval is especially brittle on multi-hop evidence needs. citeturn12academia2turn5academia2turn10academia3turn12academia0 + +so my headline recommendation is pretty simple: **do not keep turning the “more packets, more budget” knob and hoping the reader gets smarter. add an operation-aware query planner that changes retrieval, packing, and synthesis behavior based on the question.** fixed top-k is fine for ordinary lookup questions. it is the wrong default for totals, averages, counts, “previous/latest,” and ordered event lists. citeturn12academia2turn5academia0turn5academia2 + +## make retrieval operation-aware and iterative + +for longmemeval-style qa, klbr should stop treating every question as “retrieve some relevant packets and let the reader freestyle.” LongMemEval itself is built around distinct abilities such as multi-session reasoning, temporal reasoning, and knowledge updates, and its own paper reports gains from session decomposition and time-aware query expansion. newer systems aimed at the same class of problem go even further: EviMem adds explicit evidence-gap diagnosis for iterative retrieval, S2G-RAG adds a structured sufficiency judge that decides whether more evidence is needed, and TReMu improves multi-session temporal reasoning with timeline summarization plus symbolic temporal computation. citeturn12academia2turn5academia2turn5academia0turn10academia3 + +the planner i would add is intentionally small. it does not need to be a giant taxonomy. it only needs enough labels to switch klbr into the correct retrieval mode: + +- `lookup` +- `aggregate_count` +- `aggregate_sum` +- `aggregate_avg` +- `order_or_rank` +- `update_resolution` +- `preference_recommendation` +- `abstain_or_false_premise_check` + +that label set is small enough to implement now, but expressive enough to catch the failure clusters in your brief. `aggregate_*`, `order_or_rank`, and `update_resolution` should all trigger **collect mode** rather than fixed top-k mode. `preference_recommendation` should trigger **constraint-preserving mode**, where user-grounded preference evidence is required and generic world-knowledge packets are demoted unless the user explicitly asked for general suggestions. citeturn12academia2turn10academia3turn12academia0turn12academia1 + +i would implement collect mode as an iterative loop over packet candidates, not as “just raise top-k to 20.” the core idea is that the planner keeps a small structured state of what it has already covered and what still appears missing. + +```rust +enum QueryOp { + Lookup, + AggregateCount, + AggregateSum, + AggregateAvg, + OrderOrRank, + UpdateResolution, + PreferenceRecommendation, + AbstainCheck, +} + +struct OpPlan { + op: QueryOp, + target_entities: Vec, + target_slots: Vec, // e.g. ["gpa", "museum_visit", "personal_best"] + time_scope: Option, + require_personal_support: bool, + collect_all: bool, + max_passes: usize, +} +``` + +```rust +async fn retrieve_with_plan(query: &str, budget: ContextBudget) -> Result { + let plan = classify_query_op(query).await?; + let mut frontier = initial_seed_search(query, &plan, budget).await?; + let mut packets = build_and_rank_packets(query, frontier.clone(), budget)?; + + for _pass in 0..plan.max_passes { + if evidence_sufficient(&plan, &packets) { + break; + } + let gaps = diagnose_missing_groups(&plan, &packets)?; + if gaps.is_empty() { + break; + } + frontier.extend(search_gap_queries(&plan, &gaps, budget).await?); + packets = build_and_rank_packets(query, frontier.clone(), budget)?; + } + + Ok(finalize_trace(plan, frontier, packets)) +} +``` + +the important part is `diagnose_missing_groups`. for aggregate and order questions, the planner should track **fact groups**, not only packet scores. a fact group can be as simple as: + +```rust +struct FactGroupKey { + entity: String, // "science museum", "charity 5k", "undergrad gpa" + slot: String, // "visit", "personal_best", "gpa" + session_id: String, + timestamp: Option, +} +``` + +then `evidence_sufficient` becomes sane: + +- for `lookup`, one grounded fact group is often enough; +- for `aggregate_*`, you want enough distinct fact groups to compute the operation; +- for `order_or_rank`, you want a set of grounded events with sortable timestamps or chronology markers; +- for `update_resolution`, you want at least two grounded facts for the same slot with different times, so “previous” and “latest” can be distinguished; +- for `preference_recommendation`, you want at least one positive personal signal and no unresolved contradictions dominating the set. citeturn5academia0turn5academia2turn10academia3turn12academia1 + +for fusion, i would keep the current packet-level approach, but change its objective in collect mode. the top-level ranking should not be “best packet globally” alone; it should be **best packet by relevance plus marginal coverage gain**. that keeps one session from monopolizing the context and helps multi-session totals/order questions surface all answer-bearing groups. + +```rust +fn collect_mode_score(packet: &EvidencePacket, covered: &HashSet) -> f32 { + let relevance = packet_fusion_score(packet); + let novelty = packet + .fact_groups + .iter() + .filter(|g| !covered.contains(g)) + .count() as f32; + let temporal_bonus = if packet_has_time_signal(packet) { 0.5 } else { 0.0 }; + relevance + 0.8 * novelty + temporal_bonus +} +``` + +that is the real fix for the six-way cases in your brief. the planner has to behave like it knows the question is asking for a *set* or an *ordering*, not a single remembered anecdote. citeturn5academia2turn10academia3turn1academia1turn10academia0 + +## make packets chronology-native instead of prose-only + +yes, klbr should expose an explicit chronology field to the reader, separate from raw packet text. temporal memory systems keep converging on this: TiMem makes temporal structure a first-class organizing principle; MemWeaver combines structured temporally grounded memory with supporting passage evidence; TReMu improves multi-session temporal reasoning by summarizing timelines and then applying a more explicit reasoning stage. if the question says “previous,” “latest,” “before,” “after,” or “in what order,” leaving chronology implicit inside free-form prose is just making the reader do unnecessary guesswork. citeturn10academia0turn1academia1turn10academia3 + +the cleanest packet shape for klbr is mixed: keep raw ref-grounded bodies, but add a compact structured timeline and a compact structured fact layer. do not dump the full session. do not rely only on prose. do both. + +```xml + + + + + + + + + + + + + … + … + + +``` + +for ordinary lookup questions, the reader can still mostly ignore `timeline` and `facts`. for aggregate/update/order questions, the downstream synthesis stage can use them directly. this also solves the “gold ref selected but decisive value hidden after rendering” problem much more cleanly, because the crucial datum is no longer buried only in one body span. citeturn2academia0turn9academia0turn9academia3 + +packing should also change for temporal multi-session questions. today the temptation is to take the highest-scoring independent packets. for counts and chronology, that is the wrong shape. instead, use **clustered evidence packs**: + +- cluster by entity or task; +- within each cluster, keep all sessions that contribute distinct fact groups; +- sort those sessions by time; +- render a tiny cluster summary plus only the narrow raw bodies needed for provenance. + +that is much closer to the way MemWeaver and temporal-hierarchical memory systems treat long-horizon evidence: structured temporal scaffolding plus a compact supporting passage layer. citeturn1academia1turn10academia0 + +## answer from extracted facts, not raw packets alone + +for benchmark-style qa, especially on `aggregate_*`, `order_or_rank`, and `update_resolution`, i think the answer should be synthesized in two stages. not for every question. not for all runtime chat. but definitely for this class of benchmark and for matching runtime prompts that smell like benchmark-style memory queries. retrieval research and temporal qa work both support the idea that structured fact extraction can materially help when the answer depends on multiple facts, time relations, or arithmetic rather than a single span. targeted temporal fact extraction, relation-aware extraction systems, and temporal neuro-symbolic reasoning all exist because “reader sees passages and just vibes it out” is not reliable enough for this kind of task. citeturn9academia0turn9academia3turn10academia3 + +the practical design i would ship is: + +```text +stage a: packet -> fact rows with refs +stage b: fact rows -> deterministic compute/render +``` + +```rust +struct ExtractedFact { + ref_id: String, + session_id: Option, + timestamp: Option, + entity: String, + slot: String, + value_text: String, + value_number: Option, + unit: Option, + polarity: FactPolarity, +} +``` + +```rust +enum SynthesisMode { + DirectLookup, + Count, + Sum, + Average, + OrderByTime, + ResolvePreviousVsLatest, + PreferenceGroundedSuggestion, +} +``` + +```rust +fn synthesize_answer(plan: &OpPlan, facts: &[ExtractedFact]) -> AnswerDraft { + match plan.op { + QueryOp::Lookup => direct_lookup(facts), + QueryOp::AggregateCount => render_count(distinct_events(facts)), + QueryOp::AggregateSum => render_sum(numeric_values(facts)), + QueryOp::AggregateAvg => render_average(numeric_values(facts)), + QueryOp::OrderOrRank => render_time_order(sort_by_time(facts)), + QueryOp::UpdateResolution => render_previous_or_latest(facts, plan.time_scope.as_ref()), + QueryOp::PreferenceRecommendation => render_preference_grounded_suggestion(facts), + QueryOp::AbstainCheck => abstain_if_insufficient(facts), + } +} +``` + +this is how i would handle the three concrete failure classes from your brief: + +for the count/average/order cases, the system should extract one row per relevant event and then compute from the rows. if the fact table does not have enough rows, it should return to collect mode instead of answering early. that is the whole point of the sufficiency loop. citeturn5academia0turn5academia2turn10academia3 + +for the update-resolution case, the synthesis step should sort facts for the same `(entity, slot)` by time and explicitly choose `previous` versus `latest/current`. that prevents “previous personal best” from accidentally resolving to the newest time just because it is more salient in the prose. temporal reasoning papers keep showing that models need help with exactly this distinction. citeturn10academia3turn9academia2 + +for the preference-recommendation regression, the synthesis step must be stricter than “here are some denver things.” it should first extract the personal preference target and any positive or negative signals, then render a suggestion that is *explicitly grounded in those signals*. work on cognitive memory and conversational recommendation both points the same way: latent user constraints and contrasting likes/dislikes are easy to wash out if the system collapses everything into one generic user representation or lets broad distractors dominate retrieval. PersistBench also shows how broad memory injection can create cross-domain leakage rather than useful personalization. citeturn12academia0turn12academia1turn12academia3 + +so i would add one very opinionated rule for `PreferenceRecommendation`: + +```rust +fn render_preference_grounded_suggestion(facts: &[ExtractedFact]) -> AnswerDraft { + let prefs = extract_preference_profile(facts); + if prefs.personal_support_count == 0 { + return AnswerDraft::idk(); + } + + // generic world-knowledge suggestions are allowed only as decoration, + // never as the core answer. + let target = choose_target_from_personal_history(&prefs)?; + AnswerDraft::grounded(format!( + "based on your earlier likes/dislikes, the best fit is {}", + target + )) +} +``` + +that prevents extra budget from turning the answer into “tourism slurry.” the system can still mention optional expansions later, but the first sentence has to preserve the user-specific answer shape. citeturn12academia0turn12academia1turn12academia3 + +## guard the fix with rendered-evidence metrics and deterministic fixtures + +your current packet metrics are already better than plain session recall, but they still need one more split: **selected**, **rendered**, and **usable**. WhenLoss is useful here conceptually because it shows how much confusion comes from collapsing distinct failure modes together; in your case, “gold ref selected” and “gold value actually visible after packet rendering/truncation” are also different failure modes and should be measured separately. citeturn2academia0 + +i would add these metrics: + +- `answer_bearing_ref_selected`: a gold ref made it into retrieved packets. +- `answer_bearing_ref_rendered`: that ref survived packet selection and context rendering. +- `answer_value_visible`: the decisive answer literal or normalized value is present in rendered context. +- `critical_fact_row_visible`: the decisive row survived into `` or ``. +- `collect_mode_fact_group_recall`: fraction of required fact groups covered before synthesis. +- `preference_target_preserved`: for recommendation questions, whether the final answer names a user-grounded target rather than generic category suggestions. +- `update_resolution_correct`: whether previous/current/latest were disambiguated correctly for the same slot. + +implementation-wise, make the renderer return span metadata instead of just a string. + +```rust +struct RenderedPacket { + xml: String, + rendered_refs: Vec, + rendered_fact_rows: Vec, + dropped_refs: Vec, +} +``` + +then the benchmark can finally distinguish “retrieval had it” from “rendering hid it” from “reader still blew it.” + +the regression suite should include deterministic fixtures that hit the exact failure families you are now seeing, and they should all go through the production `MemoryPipeline`, not a sidecar benchmark path. i would add at least these six: + +- `count_six_way_books_and_audiobooks`: six answer sessions, count/total question, asserts collect mode keeps retrieving until all six fact groups are present or reports insufficiency. +- `average_hidden_tail_value`: two gpa values where one falls near the render budget boundary, asserts `answer_value_visible` and `critical_fact_row_visible` catch truncation. +- `museum_order_six_sessions`: six museum visits across sessions, asserts ordered output matches earliest→latest using timeline rows. +- `previous_vs_latest_personal_best`: two `charity 5k` personal best values with different dates, asserts `previous` returns the older value and `latest` returns the newer one. +- `preference_denver_music_not_generic_travel`: personal history supports a music-shaped answer, and extra generic denver packets exist; asserts the first sentence names a user-grounded target instead of generic city recommendations. +- `preference_with_negative_constraint`: user likes one category and explicitly dislikes another, asserts recommendation honors contrasting preferences. + +a good smoke command for the real bench path is still a stratified longmemeval run on the official cleaned set, because that is the smallest public slice that catches different question types while staying comparable: + +```bash +rtk cargo run -p klbr-bench -- run \ + --suite longmemeval-s \ + --data benchmarks/inputs/datasets/longmemeval_s_cleaned.json \ + --profile klbr-full \ + --top-k 5 \ + --budget-read 5000 \ + --graph-depth 1 \ + --sample 20 \ + --sample-seed 16670756171641590060 \ + --out /tmp/klbr-lme-s-op-aware-smoke +``` + +after the planner lands, i would add a second smoke for the new path: + +```bash +rtk cargo run -p klbr-bench -- run \ + --suite longmemeval-s \ + --data benchmarks/inputs/datasets/longmemeval_s_cleaned.json \ + --profile klbr-full \ + --top-k 5 \ + --budget-read 5000 \ + --graph-depth 1 \ + --sample 20 \ + --sample-seed 16670756171641590060 \ + --planner op-aware \ + --synthesis fact-table \ + --out /tmp/klbr-lme-s-op-aware-fact-table +``` + +## implementation order and risk containment + +i would land this in four small passes, not one giant refactor. + +first, add `OpPlan` classification and only switch behavior for the obviously high-value classes: `aggregate_*`, `order_or_rank`, `update_resolution`, and `preference_recommendation`. leave ordinary lookup alone. that gives you the biggest benchmark gain per line of code. LongMemEval, EviMem, and TReMu all point to temporal and multi-session reasoning as the place where this extra structure matters most. citeturn12academia2turn5academia2turn10academia3 + +second, add `` and `` to packet rendering, but keep raw `` spans. that preserves provenance and exact refs while giving the downstream stage something more machine-readable than prose. this is also where i would cluster temporal packets by entity/task rather than ranking all packets independently. temporal-hierarchical and hybrid graph+passage systems consistently benefit from that kind of structure. citeturn10academia0turn1academia1 + +third, add the structured synthesis stage only for the planner labels that need it. do **not** force every runtime memory question through extract-then-compute. the benchmark cases want it; ordinary conversation often does not. that keeps runtime behavior sane and avoids turning every answer into a mini database job. citeturn9academia0turn9academia3turn10academia3 + +fourth, keep lexical retrieval on a short leash. sqlite fts5 and tokenizer-level features are worth keeping; hand-written english-ish lexical heuristics are not where you should be putting more policy. fts5 already gives you phrase queries, `NEAR`, bm25, unicode-aware tokenization, trigram substring matching, and synonym-capable custom tokenizers. BGE-M3 also already supports dense, sparse, and multilingual retrieval in one family. so the containment rule should be: **lexical is allowed to generate candidates and soft signals, but not to become the main decision boundary for planning or suppression.** keep only the truly local discourse shortcuts hardcoded, and let the op planner plus metadata do the heavy lifting. citeturn7view0turn8view0turn8view1turn8view2turn8view3turn6academia1 + +the biggest risks are pretty predictable. collect mode can bloat context if you let it turn into “retrieve everything,” so the stopping condition has to be fact-group sufficiency, not packet count. timeline extraction can introduce its own errors, so every structured row must keep its supporting ref. preference mode can become overly conservative and say “i don’t know” too often if it requires too much personal support; but that is still better than regressing into generic recommendations. and multilingual behavior will get worse, not better, if you keep growing english cue lists; tokenizer-level lexical retrieval and multilingual sparse/dense channels are the safer bet there. cross-domain leakage is also a real risk when extra budget adds semantically adjacent but task-irrelevant memories, which is exactly why preference and temporal questions need stricter planner rules than plain lookup. citeturn12academia3turn12academia0turn6academia1turn8view0turn8view1 + +if i had to compress all of this into one sentence: **the next strong version of klbr should stop being a very good packet retriever with a hopeful reader, and become a query-planned evidence system that knows when it is doing lookup, collection, chronology, update resolution, or preference grounding.** that is the cleanest way to turn your current “budget helps coverage but not answers” plateau into actual qa gains. fileciteturn0file0 fileciteturn0file1 citeturn12academia2turn5academia2turn10academia3turn10academia0 \ No newline at end of file diff --git a/klbr-bench/src/longmemeval.rs b/klbr-bench/src/longmemeval.rs index f634ba5..d55180c 100644 --- a/klbr-bench/src/longmemeval.rs +++ b/klbr-bench/src/longmemeval.rs @@ -49,7 +49,10 @@ pub struct LongMemEvalQuestion { #[derive(Debug, Clone, Serialize)] struct PacketSelectionMetrics { packet_k: usize, + answer_bearing_ref_selected: bool, + answer_bearing_ref_rendered: bool, answer_bearing_ref_in_context: bool, + answer_value_visible: bool, gold_session_in_context: bool, gold_session_plus_neighbor_in_context: bool, packet_recall_any_at_k: f64, @@ -542,6 +545,21 @@ mod tests { assert!(suite_protocol_note("longmemeval-s").is_none()); } + #[test] + fn answer_value_visible_metric_uses_rendered_context_text() { + let mut item = test_question("q-visible", "multi-session"); + item.answer = serde_json::Value::String("3.83".to_string()); + + assert!(answer_value_visible_in_context( + &item, + "the average gpa is 3.83" + )); + assert!(!answer_value_visible_in_context( + &item, + "the packet ref was selected but the value was truncated" + )); + } + #[test] fn default_pipeline_run_dir_lives_under_benchmarks_runs() { let out_dir = default_pipeline_run_dir("longmemeval-s", "raw-turns/fts-only"); @@ -1367,7 +1385,10 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { let mut recall_all_at_5 = 0usize; let mut answerable = 0usize; let mut last_stats = None; + let mut answer_bearing_ref_selected = 0usize; + let mut answer_bearing_ref_rendered = 0usize; let mut answer_bearing_ref_in_context = 0usize; + let mut answer_value_visible = 0usize; let mut gold_session_in_context = 0usize; let mut gold_session_plus_neighbor_in_context = 0usize; let mut packet_recall_any_at_k = 0usize; @@ -1375,6 +1396,7 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { let mut same_session_dedupe_suppression_count = 0usize; let mut packet_tokens_total_sum = 0usize; let mut packet_tokens_mean_sum = 0.0f64; + let mut op_plan_counts = BTreeMap::::new(); let total = limit.unwrap_or(data.len()).min(data.len()); for (idx, item) in data.iter().take(total).enumerate() { @@ -1439,6 +1461,9 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { reference_time: Some(parse_date_to_timestamp(&item.question_date)), }; let retrieval = pipeline.retrieve_evidence(query.clone(), budget).await?; + *op_plan_counts + .entry(retrieval.op_plan.op.as_str().to_string()) + .or_insert(0) += 1; let context = pipeline .assemble_context(query.clone(), &retrieval, budget) .await?; @@ -1508,6 +1533,15 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { if packet_metrics.answer_bearing_ref_in_context { answer_bearing_ref_in_context += 1; } + if packet_metrics.answer_bearing_ref_selected { + answer_bearing_ref_selected += 1; + } + if packet_metrics.answer_bearing_ref_rendered { + answer_bearing_ref_rendered += 1; + } + if packet_metrics.answer_value_visible { + answer_value_visible += 1; + } if packet_metrics.gold_session_in_context { gold_session_in_context += 1; } @@ -1600,7 +1634,10 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { "recall_any_at_5": if answerable == 0 { 0.0 } else { recall_any_at_5 as f64 / answerable as f64 }, "recall_all_at_5": if answerable == 0 { 0.0 } else { recall_all_at_5 as f64 / answerable as f64 }, "packet_metrics": { + "answer_bearing_ref_selected": if answerable == 0 { 0.0 } else { answer_bearing_ref_selected as f64 / answerable_denominator }, + "answer_bearing_ref_rendered": if answerable == 0 { 0.0 } else { answer_bearing_ref_rendered as f64 / answerable_denominator }, "answer_bearing_ref_in_context": if answerable == 0 { 0.0 } else { answer_bearing_ref_in_context as f64 / answerable_denominator }, + "answer_value_visible": if answerable == 0 { 0.0 } else { answer_value_visible as f64 / answerable_denominator }, "gold_session_in_context": if answerable == 0 { 0.0 } else { gold_session_in_context as f64 / answerable_denominator }, "gold_session_plus_neighbor_in_context": if answerable == 0 { 0.0 } else { gold_session_plus_neighbor_in_context as f64 / answerable_denominator }, "packet_recall_any_at_k": if answerable == 0 { 0.0 } else { packet_recall_any_at_k as f64 / answerable_denominator }, @@ -1609,6 +1646,7 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { "packet_tokens_total_mean": packet_tokens_total_sum as f64 / evaluated_denominator, "packet_tokens_mean": packet_tokens_mean_sum / evaluated_denominator, }, + "op_plan_counts": op_plan_counts.clone(), "official_eval": official_eval_result.clone(), "last_store_stats": last_stats, }); @@ -1619,7 +1657,7 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { fs::write( out_dir.join("report.md"), format!( - "# klbr memory pipeline benchmark\n\n- suite: `{}`\n- profile: `{}`\n- evaluated: `{}`\n- answerable: `{}`\n- sample: `{}`\n- RecallAny@5: `{:.4}`\n- RecallAll@5: `{:.4}`\n- PacketRecallAny@{}: `{:.4}`\n- PacketRecallAll@{}: `{:.4}`\n- answer_bearing_ref_in_context: `{:.4}`\n- gold_session_plus_neighbor_in_context: `{:.4}`\n- same_session_dedupe_suppression_count: `{}`\n- packet_tokens_mean: `{:.2}`\n- retrieval_only: `{}`\n- official_eval: `{}`\n", + "# klbr memory pipeline benchmark\n\n- suite: `{}`\n- profile: `{}`\n- evaluated: `{}`\n- answerable: `{}`\n- sample: `{}`\n- RecallAny@5: `{:.4}`\n- RecallAll@5: `{:.4}`\n- PacketRecallAny@{}: `{:.4}`\n- PacketRecallAll@{}: `{:.4}`\n- answer_bearing_ref_selected: `{:.4}`\n- answer_bearing_ref_rendered: `{:.4}`\n- answer_bearing_ref_in_context: `{:.4}`\n- answer_value_visible: `{:.4}`\n- gold_session_plus_neighbor_in_context: `{:.4}`\n- same_session_dedupe_suppression_count: `{}`\n- packet_tokens_mean: `{:.2}`\n- op_plan_counts: `{}`\n- retrieval_only: `{}`\n- official_eval: `{}`\n", suite, profile, evaluated, @@ -1631,10 +1669,14 @@ pub async fn run_pipeline_command(args: &[String]) -> Result<()> { if answerable == 0 { 0.0 } else { packet_recall_any_at_k as f64 / answerable_denominator }, top_k, if answerable == 0 { 0.0 } else { packet_recall_all_at_k as f64 / answerable_denominator }, + if answerable == 0 { 0.0 } else { answer_bearing_ref_selected as f64 / answerable_denominator }, + if answerable == 0 { 0.0 } else { answer_bearing_ref_rendered as f64 / answerable_denominator }, if answerable == 0 { 0.0 } else { answer_bearing_ref_in_context as f64 / answerable_denominator }, + if answerable == 0 { 0.0 } else { answer_value_visible as f64 / answerable_denominator }, if answerable == 0 { 0.0 } else { gold_session_plus_neighbor_in_context as f64 / answerable_denominator }, same_session_dedupe_suppression_count, packet_tokens_mean_sum / evaluated_denominator, + serde_json::to_string(&op_plan_counts).unwrap_or_else(|_| "{}".to_string()), retrieval_only, match &official_eval_result { Some(result) if result.success => "success", @@ -2029,11 +2071,22 @@ fn compute_packet_selection_metrics( .take(k) .filter_map(|packet| packet.session_id.clone()) .collect::>(); + let selected_refs = retrieval + .packets + .iter() + .flat_map(|packet| packet.refs.iter().cloned()) + .collect::>(); - let answer_bearing_ref_in_context = !answer_ref_ids.is_empty() + let answer_bearing_ref_selected = !answer_ref_ids.is_empty() + && answer_ref_ids + .iter() + .any(|ref_id| selected_refs.contains(ref_id)); + let answer_bearing_ref_rendered = !answer_ref_ids.is_empty() && answer_ref_ids .iter() .any(|ref_id| used_refs.contains(ref_id)); + let answer_bearing_ref_in_context = answer_bearing_ref_rendered; + let answer_value_visible = answer_value_visible_in_context(item, &context.content); let gold_session_in_context = !gold_sessions.is_empty() && gold_sessions .iter() @@ -2078,7 +2131,10 @@ fn compute_packet_selection_metrics( PacketSelectionMetrics { packet_k: k, + answer_bearing_ref_selected, + answer_bearing_ref_rendered, answer_bearing_ref_in_context, + answer_value_visible, gold_session_in_context, gold_session_plus_neighbor_in_context, packet_recall_any_at_k, @@ -2093,6 +2149,18 @@ fn compute_packet_selection_metrics( } } +fn answer_value_visible_in_context(item: &LongMemEvalQuestion, context: &str) -> bool { + let context_norm = normalize_answer_text(context); + if context_norm.is_empty() { + return false; + } + answer_strings(&item.answer) + .into_iter() + .map(|answer| normalize_answer_text(&answer)) + .filter(|answer| answer.chars().any(|ch| ch.is_alphanumeric())) + .any(|answer| context_norm.contains(&answer)) +} + async fn run_ingest(data_path: &str, db_dir_path: &str) -> Result<()> { let dataset_file = File::open(data_path)?; let dataset: Vec = serde_json::from_reader(dataset_file)?; diff --git a/klbr-core/src/evidence.rs b/klbr-core/src/evidence.rs index 2fafc82..972c72a 100644 --- a/klbr-core/src/evidence.rs +++ b/klbr-core/src/evidence.rs @@ -247,6 +247,17 @@ impl EvidencePlanner { candidates: &[EvidenceAtom], limit: usize, packet_token_budget: usize, + ) -> Result { + self.build_packets_with_collect_mode(query, candidates, limit, packet_token_budget, false) + } + + pub fn build_packets_with_collect_mode( + &self, + query: &str, + candidates: &[EvidenceAtom], + limit: usize, + packet_token_budget: usize, + collect_mode: bool, ) -> Result { let mut packet_positions: HashMap = HashMap::new(); let mut packets = Vec::new(); @@ -265,7 +276,11 @@ impl EvidencePlanner { } } - let ranked = rank_and_cap_packets(packets, limit); + let ranked = if collect_mode { + rank_and_cap_packets_collect_mode(packets, limit) + } else { + rank_and_cap_packets(packets, limit) + }; omissions.extend(ranked.omissions); Ok(PacketBuildOutcome { packets: ranked.packets, @@ -539,6 +554,102 @@ pub(crate) fn rank_and_cap_packets( } } +pub(crate) fn rank_and_cap_packets_collect_mode( + mut packets: Vec, + limit: usize, +) -> RankedPacketOutcome { + packets.sort_by(|left, right| { + packet_fusion_score(right) + .partial_cmp(&packet_fusion_score(left)) + .unwrap_or(std::cmp::Ordering::Equal) + .then_with(|| left.anchor_ref.cmp(&right.anchor_ref)) + }); + + let mut out = Vec::new(); + let mut deferred = Vec::new(); + let mut seen_bodies = HashSet::new(); + let mut seen_sessions = HashSet::new(); + let mut omissions = Vec::new(); + + for packet in packets { + if out.len() >= limit { + omissions.extend(packet_omissions("packet_limit", &packet)); + continue; + } + if packet + .session_id + .as_ref() + .is_some_and(|session_id| seen_sessions.contains(session_id)) + { + deferred.push(packet); + continue; + } + if select_packet(packet, &mut out, &mut seen_bodies, &mut omissions) { + if let Some(session_id) = out.last().and_then(|packet| packet.session_id.clone()) { + seen_sessions.insert(session_id); + } + } + } + + for packet in deferred { + if out.len() >= limit { + omissions.extend(packet_omissions("packet_limit", &packet)); + continue; + } + select_packet(packet, &mut out, &mut seen_bodies, &mut omissions); + } + + RankedPacketOutcome { + packets: out, + omissions, + } +} + +fn select_packet( + packet: EvidencePacket, + out: &mut Vec, + seen_bodies: &mut HashSet, + omissions: &mut Vec, +) -> bool { + if is_graph_only_packet(&packet) { + omissions.extend(packet_omissions("graph_only_uncorroborated", &packet)); + return false; + } + let body_key = packet.bodies.join("\n---\n"); + if seen_bodies.contains(&body_key) { + omissions.extend(packet_omissions("duplicate_body", &packet)); + return false; + } + if packet.packet_kind != EvidencePacketKind::ExactRef + && packet.session_id.is_some() + && out + .iter() + .filter(|previous: &&EvidencePacket| { + previous.packet_kind != EvidencePacketKind::ExactRef + && previous.session_id == packet.session_id + }) + .any(|previous| packet_ref_overlap(previous, &packet) >= 0.6) + { + omissions.extend(packet_omissions("same_session_overlap", &packet)); + return false; + } + seen_bodies.insert(body_key); + out.push(packet); + true +} + +fn packet_omissions(reason: &str, packet: &EvidencePacket) -> Vec { + packet + .refs + .iter() + .map(|ref_id| EvidenceOmittedRef { + packet_id: packet.packet_id.clone(), + ref_id: ref_id.clone(), + reason: reason.to_string(), + }) + .collect() +} + fn is_graph_only_packet(packet: &EvidencePacket) -> bool { let has_graph_signal = packet .signals @@ -837,6 +948,32 @@ mod tests { assert!(ranked.omissions.is_empty()); } + #[test] + fn collect_mode_prefers_distinct_sessions_before_second_packet_from_same_session() { + let ranked = rank_and_cap_packets_collect_mode( + vec![ + test_packet_with_refs("pkt_s1_best", &["ref_1"], "fts", 1, Some("s1")), + test_packet_with_refs("pkt_s1_second", &["ref_2"], "fts", 2, Some("s1")), + test_packet_with_refs("pkt_s2", &["ref_3"], "fts", 3, Some("s2")), + ], + 2, + ); + + assert_eq!( + ranked + .packets + .iter() + .map(|packet| packet.packet_id.as_str()) + .collect::>(), + vec!["pkt_s1_best", "pkt_s2"] + ); + assert!(ranked + .omissions + .iter() + .any(|omission| omission.packet_id == "pkt_s1_second" + && omission.reason == "packet_limit")); + } + fn test_packet( packet_id: &str, ref_id: &str, diff --git a/klbr-core/src/lib.rs b/klbr-core/src/lib.rs index 4bc0b12..ea9a2e3 100644 --- a/klbr-core/src/lib.rs +++ b/klbr-core/src/lib.rs @@ -10,6 +10,7 @@ pub mod memory; pub mod models; pub mod mvp; pub mod pipeline; +pub mod planner; pub mod retrieval; pub mod router; pub mod support; diff --git a/klbr-core/src/pipeline.rs b/klbr-core/src/pipeline.rs index dd1fb41..eb199aa 100644 --- a/klbr-core/src/pipeline.rs +++ b/klbr-core/src/pipeline.rs @@ -16,6 +16,7 @@ use crate::{ }, models::{LlmClient, Message, RerankResult}, mvp::{MemoryLayer, MemoryRecordInput, MemoryStatus}, + planner::{classify_query_op, OpPlan}, }; pub use crate::evidence::EvidenceAtom; @@ -106,6 +107,8 @@ pub struct PipelineRetrievalTrace { pub routed_lanes: Vec, #[serde(default)] pub route: LaneRoute, + #[serde(default)] + pub op_plan: OpPlan, pub exact_refs: Vec, pub candidates: Vec, #[serde(default)] @@ -318,12 +321,16 @@ impl MemoryPipeline { .collect::>(); let route = route_memory_query(&query.text, !exact_refs.is_empty()); let routed_lanes = route.lanes.clone(); + let op_plan = classify_query_op(&query.text); + let candidate_limit = op_plan.candidate_limit(budget.top_k); + let packet_limit = op_plan.packet_limit(budget.top_k); + let packet_token_budget = op_plan.packet_token_budget(budget.max_tokens); let mut candidates = Vec::new(); candidates.extend(self.resolve_exact_refs(&exact_refs)?); if route.archival_allowed && self.profile.lexical { candidates.extend( - self.search_sparse(&query.text, &routed_lanes, budget.top_k) + self.search_sparse(&query.text, &routed_lanes, candidate_limit) .await?, ); } @@ -333,7 +340,7 @@ impl MemoryPipeline { &query.text, query.reference_time, &routed_lanes, - budget.top_k, + candidate_limit, ) .await?, ); @@ -341,16 +348,19 @@ impl MemoryPipeline { if route.archival_allowed && self.profile.graph && budget.graph_depth > 0 { let seed_refs = canonical_refs(&self.memory, &exact_refs)?; - candidates.extend(self.expand_graph(&seed_refs, budget.top_k)?); + candidates.extend(self.expand_graph(&seed_refs, candidate_limit)?); } - let candidates = - rank_seed_candidates(candidates, budget.top_k.saturating_mul(4).max(budget.top_k)); - let mut packet_plan = self.evidence_planner().build_packets( + let candidates = rank_seed_candidates( + candidates, + packet_limit.saturating_mul(4).max(candidate_limit), + ); + let mut packet_plan = self.evidence_planner().build_packets_with_collect_mode( &query.text, &candidates, - budget.top_k, - budget.max_tokens.min(900), + packet_limit, + packet_token_budget, + op_plan.collect_all, )?; let constrained = constrain_packets_for_query(&query.text, packet_plan.packets); packet_plan.omissions.extend(constrained.omissions); @@ -361,6 +371,7 @@ impl MemoryPipeline { query_id: query.query_id, routed_lanes, route, + op_plan, exact_refs, candidates, packets, @@ -386,11 +397,12 @@ impl MemoryPipeline { { fallback_packets = self .evidence_planner() - .build_packets( + .build_packets_with_collect_mode( &query.text, &retrieved.candidates, - budget.top_k, - budget.max_tokens.min(900), + retrieved.op_plan.packet_limit(budget.top_k), + retrieved.op_plan.packet_token_budget(budget.max_tokens), + retrieved.op_plan.collect_all, )? .packets; let constrained = constrain_packets_for_query(&query.text, fallback_packets); @@ -440,10 +452,14 @@ impl MemoryPipeline { ) -> Result { self.memory.sync_reference_indexes()?; let route = route_memory_query(&query.text, false); + let op_plan = classify_query_op(&query.text); let lanes = route.lanes.clone(); + let candidate_limit = op_plan.candidate_limit(budget.top_k); + let packet_limit = op_plan.packet_limit(budget.top_k); + let packet_token_budget = op_plan.packet_token_budget(budget.max_tokens); let entries = if route.archival_allowed { self.memory - .active_promptable_refs(&lanes, budget.top_k.saturating_mul(8).max(24))? + .active_promptable_refs(&lanes, candidate_limit.saturating_mul(8).max(24))? } else { vec![] }; @@ -451,11 +467,12 @@ impl MemoryPipeline { .into_iter() .map(|entry| self.ref_search_entry_to_atom(entry)) .collect::>>()?; - let mut packet_plan = self.evidence_planner().build_packets( + let mut packet_plan = self.evidence_planner().build_packets_with_collect_mode( &query.text, &candidates, - budget.top_k, - budget.max_tokens.min(900), + packet_limit, + packet_token_budget, + op_plan.collect_all, )?; let constrained = constrain_packets_for_query(&query.text, packet_plan.packets); packet_plan.omissions.extend(constrained.omissions); @@ -466,6 +483,7 @@ impl MemoryPipeline { query_id: query.query_id.clone(), routed_lanes: lanes, route, + op_plan, exact_refs: vec![], candidates, packets, @@ -1599,6 +1617,7 @@ fn dense_embedding_text(body: &str) -> String { mod tests { use super::*; use crate::evidence::{EvidencePacketKind, EvidencePacketSignals, EvidenceSourceSignal}; + use crate::planner::QueryOp; use tempfile::NamedTempFile; #[tokio::test] @@ -1858,6 +1877,87 @@ mod tests { Ok(()) } + #[tokio::test] + async fn aggregate_query_trace_uses_collect_sized_plan() -> Result<()> { + let tmp = NamedTempFile::new()?; + let store = MemoryStore::open(tmp.path().to_str().unwrap(), 4)?; + let llm = LlmClient::new(crate::models::ModelsConfig::default()); + let pipeline = MemoryPipeline::new(store, llm, MemoryConfig::default()) + .with_profile("raw-turns/fts-only"); + let budget = ContextBudget { + max_tokens: 1_000, + top_k: 2, + graph_depth: 0, + }; + + for index in 0..6 { + pipeline + .observe_session(BenchSession { + session_id: format!("museum-session-{index}"), + timestamp: Some(index), + turns: vec![BenchTurn { + role: "user".to_string(), + content: format!( + "museum visit {index}: I went to the museum district stop" + ), + timestamp: Some(index), + }], + }) + .await?; + } + + let query = BenchQuery { + query_id: "q-museum-count".to_string(), + text: "how many museum visits did I mention?".to_string(), + reference_time: Some(10), + }; + let retrieval = pipeline.retrieve_evidence(query, budget).await?; + + assert_eq!(retrieval.op_plan.op, QueryOp::AggregateCount); + assert!(retrieval.op_plan.collect_all); + assert!(retrieval.candidates.len() > budget.top_k); + assert!(retrieval.packets.len() > budget.top_k); + Ok(()) + } + + #[tokio::test] + async fn lookup_query_trace_keeps_default_plan() -> Result<()> { + let tmp = NamedTempFile::new()?; + let store = MemoryStore::open(tmp.path().to_str().unwrap(), 4)?; + let llm = LlmClient::new(crate::models::ModelsConfig::default()); + let pipeline = MemoryPipeline::new(store, llm, MemoryConfig::default()) + .with_profile("raw-turns/fts-only"); + let budget = ContextBudget { + max_tokens: 1_000, + top_k: 2, + graph_depth: 0, + }; + + pipeline + .observe_session(BenchSession { + session_id: "lookup-session".to_string(), + timestamp: Some(1), + turns: vec![BenchTurn { + role: "user".to_string(), + content: "I bought coffee creamer at Target.".to_string(), + timestamp: Some(1), + }], + }) + .await?; + + let query = BenchQuery { + query_id: "q-lookup".to_string(), + text: "where did I buy coffee creamer?".to_string(), + reference_time: Some(2), + }; + let retrieval = pipeline.retrieve_evidence(query, budget).await?; + + assert_eq!(retrieval.op_plan.op, QueryOp::Lookup); + assert!(!retrieval.op_plan.collect_all); + assert!(retrieval.packets.len() <= budget.top_k); + Ok(()) + } + #[tokio::test] async fn false_recall_adversary_keeps_unmatched_packets_empty() -> Result<()> { let tmp = NamedTempFile::new()?; diff --git a/klbr-core/src/planner.rs b/klbr-core/src/planner.rs new file mode 100644 index 0000000..a6c9066 --- /dev/null +++ b/klbr-core/src/planner.rs @@ -0,0 +1,309 @@ +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, serde::Serialize, serde::Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum QueryOp { + Lookup, + AggregateCount, + AggregateSum, + AggregateAvg, + OrderOrRank, + UpdateResolution, + PreferenceRecommendation, + AbstainOrFalsePremiseCheck, +} + +impl QueryOp { + pub fn as_str(self) -> &'static str { + match self { + Self::Lookup => "lookup", + Self::AggregateCount => "aggregate_count", + Self::AggregateSum => "aggregate_sum", + Self::AggregateAvg => "aggregate_avg", + Self::OrderOrRank => "order_or_rank", + Self::UpdateResolution => "update_resolution", + Self::PreferenceRecommendation => "preference_recommendation", + Self::AbstainOrFalsePremiseCheck => "abstain_or_false_premise_check", + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct OpPlan { + pub op: QueryOp, + pub collect_all: bool, + pub require_personal_support: bool, + pub max_passes: usize, + pub reason: String, +} + +impl Default for OpPlan { + fn default() -> Self { + Self { + op: QueryOp::Lookup, + collect_all: false, + require_personal_support: false, + max_passes: 1, + reason: "lookup_default".to_string(), + } + } +} + +impl OpPlan { + pub fn candidate_limit(&self, top_k: usize) -> usize { + let base = top_k.max(1); + if self.collect_all { + base.saturating_mul(3).max(base) + } else if self.require_personal_support { + base.saturating_mul(2).max(base) + } else { + base + } + } + + pub fn packet_limit(&self, top_k: usize) -> usize { + let base = top_k.max(1); + if self.collect_all { + base.saturating_mul(2).max(base) + } else { + base + } + } + + pub fn packet_token_budget(&self, max_tokens: usize) -> usize { + let default = max_tokens.min(900); + if self.collect_all { + default.min(700) + } else { + default + } + } +} + +pub fn classify_query_op(query: &str) -> OpPlan { + let lower = normalize_query(query); + let tokens = lower.split_whitespace().collect::>(); + + if contains_any( + &lower, + &["don't know", "do not know", "unknown", "not sure"], + ) || contains_any(&lower, &["did i ever", "have i ever", "was there any"]) + { + return OpPlan { + op: QueryOp::AbstainOrFalsePremiseCheck, + collect_all: false, + require_personal_support: false, + max_passes: 1, + reason: "false_premise_or_abstain_cue".to_string(), + }; + } + + if contains_any( + &lower, + &[ + "previous", + "latest", + "current", + "most recent", + "last time", + "newer", + "older", + "before", + "after", + ], + ) { + return OpPlan { + op: QueryOp::UpdateResolution, + collect_all: true, + require_personal_support: false, + max_passes: 2, + reason: "update_or_temporal_resolution_cue".to_string(), + }; + } + + if contains_any( + &lower, + &[ + "in order", + "what order", + "which order", + "ordered", + "chronological", + "rank", + "ranked", + "earliest", + "oldest", + "newest", + "first", + "second", + "third", + ], + ) { + return OpPlan { + op: QueryOp::OrderOrRank, + collect_all: true, + require_personal_support: false, + max_passes: 2, + reason: "order_or_rank_cue".to_string(), + }; + } + + if contains_any(&lower, &["average", "mean"]) { + return OpPlan { + op: QueryOp::AggregateAvg, + collect_all: true, + require_personal_support: false, + max_passes: 2, + reason: "average_cue".to_string(), + }; + } + + if contains_any(&lower, &["total", "sum", "combined"]) && has_numeric_or_unit_cue(&tokens) { + return OpPlan { + op: QueryOp::AggregateSum, + collect_all: true, + require_personal_support: false, + max_passes: 2, + reason: "sum_cue".to_string(), + }; + } + + if contains_any( + &lower, + &[ + "how many", + "number of", + "count", + "how much", + "how often", + "times did", + ], + ) { + return OpPlan { + op: QueryOp::AggregateCount, + collect_all: true, + require_personal_support: false, + max_passes: 2, + reason: "count_cue".to_string(), + }; + } + + if contains_any( + &lower, + &[ + "recommend", + "recommendation", + "suggest", + "suggestion", + "best fit", + "would i like", + "should i try", + ], + ) || (contains_any( + &lower, + &["favorite", "prefer", "preference", "liked", "disliked"], + ) && contains_any(&lower, &["what", "which", "where"])) + { + return OpPlan { + op: QueryOp::PreferenceRecommendation, + collect_all: false, + require_personal_support: true, + max_passes: 1, + reason: "preference_recommendation_cue".to_string(), + }; + } + + OpPlan::default() +} + +fn normalize_query(query: &str) -> String { + query + .to_lowercase() + .replace(['?', '!', ',', '.', ';', ':', '"', '\''], " ") + .split_whitespace() + .collect::>() + .join(" ") +} + +fn contains_any(haystack: &str, needles: &[&str]) -> bool { + needles.iter().any(|needle| haystack.contains(needle)) +} + +fn has_numeric_or_unit_cue(tokens: &[&str]) -> bool { + tokens.iter().any(|token| { + token.chars().any(|c| c.is_ascii_digit()) + || matches!( + *token, + "weeks" + | "days" + | "hours" + | "minutes" + | "miles" + | "kilometers" + | "km" + | "gpa" + | "score" + | "scores" + | "cost" + | "costs" + | "price" + | "prices" + ) + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn planner_detects_aggregate_count() { + let plan = classify_query_op("how many competitive sports did I do?"); + + assert_eq!(plan.op, QueryOp::AggregateCount); + assert!(plan.collect_all); + assert_eq!(plan.candidate_limit(5), 15); + assert_eq!(plan.packet_limit(5), 10); + } + + #[test] + fn planner_detects_average() { + let plan = classify_query_op("what was the average of my two GPA values?"); + + assert_eq!(plan.op, QueryOp::AggregateAvg); + assert!(plan.collect_all); + } + + #[test] + fn planner_detects_temporal_update_resolution_before_ordering() { + let plan = classify_query_op("what was my previous personal best before the latest one?"); + + assert_eq!(plan.op, QueryOp::UpdateResolution); + assert!(plan.collect_all); + } + + #[test] + fn planner_detects_order_or_rank() { + let plan = classify_query_op("list the museums I visited in chronological order"); + + assert_eq!(plan.op, QueryOp::OrderOrRank); + assert!(plan.collect_all); + } + + #[test] + fn planner_detects_preference_recommendation() { + let plan = + classify_query_op("what music venue in Denver should I try based on what I liked?"); + + assert_eq!(plan.op, QueryOp::PreferenceRecommendation); + assert!(plan.require_personal_support); + assert!(!plan.collect_all); + assert_eq!(plan.candidate_limit(5), 10); + } + + #[test] + fn planner_leaves_plain_lookup_alone() { + let plan = classify_query_op("where did I buy coffee creamer?"); + + assert_eq!(plan.op, QueryOp::Lookup); + assert!(!plan.collect_all); + } +}